Record updated version number

* Remove mission text

* Change wording

* Add link to Read the Docs

* Add links to docs in empty log tables

* Remove beta block from Dashboard and Request Logs

* Make ActionButton onClick optional

.dockerignore

@@ -0,0 +1,5 @@
+**/node_modules/
+.git
+**/.venv/
+**/.env*
+**/.next/

.github/ISSUE_TEMPLATE/sweep-fast-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Fast Issue
+title: 'Sweep (fast): '
+description: For few-line fixes to be handled by Sweep, an AI-powered junior developer. Sweep will use GPT-3.5 to quickly create a PR for very small changes.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

.github/ISSUE_TEMPLATE/sweep-slow-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Slow Issue
+title: 'Sweep (slow): '
+description: For larger bugs, features, refactors, and tests to be handled by Sweep, an AI-powered junior developer. Sweep will perform a deeper search and more self-reviews but will take longer.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

.github/ISSUE_TEMPLATE/sweep-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Issue
+title: 'Sweep: '
+description: For small bugs, features, refactors, and tests to be handled by Sweep, an AI-powered junior developer.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

.gitignore

@@ -0,0 +1,6 @@
+.env
+.venv/
+*.pyc
+node_modules/
+*.tsbuildinfo
+dist/

.prettierignore

@@ -0,0 +1,2 @@
+*.schema.json
+app/pnpm-lock.yaml

README.md

@@ -1,16 +1,53 @@
-<!-- <img src="https://github.com/openpipe/openpipe/assets/41524992/ca59596e-eb80-40f9-921f-6d67f6e6d8fa" width="72px" /> -->
+ <p align="center">
+  <a href="https://openpipe.ai">
+    <img height="70" src="https://github.com/openpipe/openpipe/assets/41524992/70af25fb-1f90-42d9-8a20-3606e3b5aaba" alt="logo">
+  </a>
+</p>
+<h1 align="center">
+  OpenPipe
+</h1>
 
-# OpenPipe
+<p align="center">
+  <i>Turn expensive prompts into cheap fine-tuned models.</i>
+</p>
 
-OpenPipe is a flexible playground for comparing and optimizing LLM prompts. It lets you quickly generate, test and compare candidate prompts, and can automatically [translate](#-translate-between-model-apis) those prompts between models.
+<p align="center">
+  <a href="/LICENSE"><img alt="License Apache-2.0" src="https://img.shields.io/github/license/openpipe/openpipe?style=flat-square"></a>
+  <a href='http://makeapullrequest.com'><img alt='PRs Welcome' src='https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square'/></a>
+  <a href="https://github.com/openpipe/openpipe/graphs/commit-activity"><img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/openpipe/openpipe?style=flat-square"/></a>
+  <a href="https://github.com/openpipe/openpipe/issues"><img alt="GitHub closed issues" src="https://img.shields.io/github/issues-closed/openpipe/openpipe?style=flat-square"/></a>
+ <img src="https://img.shields.io/badge/Y%20Combinator-S23-orange?style=flat-square" alt="Y Combinator S23">
+</p>
 
-<img src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="demo">
+<p align="center">
+  <a href="https://app.openpipe.ai/">Hosted App</a> - <a href="#running-locally">Running Locally</a> - <a href="#sample-experiments">Experiments</a>
+</p>
+
+<br>
+Use powerful but expensive LLMs to fine-tune smaller and cheaper models suited to your exact needs. Evaluate model and prompt combinations in the playground. Query your past requests and export optimized training data. Try it out at https://app.openpipe.ai or <a href="#running-locally">run it locally</a>.
+<br>
+
+
+## Features
+
+ * <b>Experiment</b>
+   * Bulk-test wide-reaching scenarios using code templating.
+   * Seamlessly translate prompts across different model APIs.
+   * Tap into autogenerated scenarios for fresh test perspectives.
+
+ * <b>Fine-Tune (Beta)</b>
+   * Easy integration with OpenPipe's SDK in both Python and JS.
+   * Swiftly query logs using intuitive built-in filters.
+   * Export data in multiple training formats, including Alpaca and ChatGPT, with deduplication.
+   
+<img src="https://github.com/openpipe/openpipe/assets/41524992/eaa8b92d-4536-4f63-bbef-4b0b1a60f6b5" alt="fine-tune demo">
+
+<!-- <img height="400px" src="https://github.com/openpipe/openpipe/assets/41524992/66bb1843-cb72-4130-a369-eec2df3b8201" alt="playground demo"> -->
 
-You can use our hosted version of OpenPipe at https://openpipe.ai. You can also clone this repository and [run it locally](#running-locally).
 
 ## Sample Experiments
 
-These are simple experiments users have created that show how OpenPipe works. Feel free to fork them and start experimenting yourself.
+These are sample experiments users have created that show how OpenPipe works. Feel free to fork them and start experimenting yourself.
 
 - [Twitter Sentiment Analysis](https://app.openpipe.ai/experiments/62c20a73-2012-4a64-973c-4b665ad46a57)
 - [Reddit User Needs](https://app.openpipe.ai/experiments/22222222-2222-2222-2222-222222222222)
@@ -19,43 +56,25 @@ These are simple experiments users have created that show how OpenPipe works. Fe
 
 ## Supported Models
 
-- All models available through the OpenAI [chat completion API](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
+#### OpenAI
-- Llama2 [7b chat](https://replicate.com/a16z-infra/llama7b-v2-chat), [13b chat](https://replicate.com/a16z-infra/llama13b-v2-chat), [70b chat](https://replicate.com/replicate/llama70b-v2-chat).
+  - [GPT 3.5 Turbo](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
-- Anthropic's [Claude 1 Instant](https://www.anthropic.com/index/introducing-claude) and [Claude 2](https://www.anthropic.com/index/claude-2)
+  - [GPT 3.5 Turbo 16k](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
-
+  - [GPT 4](https://openai.com/gpt-4)
-## Features
+#### Llama2
-
+  - [7b chat](https://replicate.com/a16z-infra/llama7b-v2-chat)
-### 🔍 Visualize Responses
+  - [13b chat](https://replicate.com/a16z-infra/llama13b-v2-chat)
-
+  - [70b chat](https://replicate.com/replicate/llama70b-v2-chat)
-Inspect prompt completions side-by-side.
+#### Llama2 Fine-Tunes
-
+  - [Open-Orca/OpenOrcaxOpenChat-Preview2-13B](https://huggingface.co/Open-Orca/OpenOrcaxOpenChat-Preview2-13B)
-### 🧪 Bulk-Test
+  - [Open-Orca/OpenOrca-Platypus2-13B](https://huggingface.co/Open-Orca/OpenOrca-Platypus2-13B)
-
+  - [NousResearch/Nous-Hermes-Llama2-13b](https://huggingface.co/NousResearch/Nous-Hermes-Llama2-13b)
-OpenPipe lets you _template_ a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broad coverage of your problem space.
+  - [jondurbin/airoboros-l2-13b-gpt4-2.0](https://huggingface.co/jondurbin/airoboros-l2-13b-gpt4-2.0)
-
+  - [lmsys/vicuna-13b-v1.5](https://huggingface.co/lmsys/vicuna-13b-v1.5)
-### 📟 Translate between Model APIs
+  - [Gryphe/MythoMax-L2-13b](https://huggingface.co/Gryphe/MythoMax-L2-13b)
-
+  - [NousResearch/Nous-Hermes-llama-2-7b](https://huggingface.co/NousResearch/Nous-Hermes-llama-2-7b)
-Write your prompt in one format and automatically convert it to work with any other model.
+#### Anthropic
-
+  - [Claude 1 Instant](https://www.anthropic.com/index/introducing-claude)
-<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/1e19ccf2-96b6-4e93-a3a5-1449710d1b5b" alt="translate between models">
+  - [Claude 2](https://www.anthropic.com/index/claude-2)
-
-<br><br>
-
-### 🛠️ Refine Your Prompts Automatically
-
-Use a growing database of best-practice refinements to improve your prompts automatically.
-
-<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/87a27fe7-daef-445c-a5e2-1c82b23f9f99" alt="add function call">
-
-<br><br>
-
-### 🪄 Auto-generate Test Scenarios
-
-OpenPipe includes a tool to generate new test scenarios based on your existing prompts and scenarios. Just click "Autogenerate Scenario" to try it out!
-
-<img width="600" src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="auto-generate">
-
-<br><br>
 
 ## Running Locally
 

app/.env.example

@@ -32,5 +32,11 @@ NEXT_PUBLIC_HOST="http://localhost:3000"
 GITHUB_CLIENT_ID="your_client_id"
 GITHUB_CLIENT_SECRET="your_secret"
 
-OPENPIPE_BASE_URL="http://localhost:3000/api"
+OPENPIPE_BASE_URL="http://localhost:3000/api/v1"
 OPENPIPE_API_KEY="your_key"
+
+SENDER_EMAIL="placeholder"
+SMTP_HOST="placeholder"
+SMTP_PORT="placeholder"
+SMTP_LOGIN="placeholder"
+SMTP_PASSWORD="placeholder"

app/.eslintrc.cjs

@@ -6,7 +6,7 @@ const config = {
   overrides: [
     {
       extends: ["plugin:@typescript-eslint/recommended-requiring-type-checking"],
-      files: ["*.ts", "*.tsx"],
+      files: ["*.mts", "*.ts", "*.tsx"],
       parserOptions: {
         project: path.join(__dirname, "tsconfig.json"),
       },

app/.gitignore

@@ -44,3 +44,6 @@ yarn-error.log*
 
 # Sentry Auth Token
 .sentryclirc
+
+# custom openai intialization
+src/server/utils/openaiCustomConfig.json

app/.prettierignore

@@ -1,2 +0,0 @@
-*.schema.json
-pnpm-lock.yaml

app/@types/nextjs-routes.d.ts

@@ -12,19 +12,20 @@ declare module "nextjs-routes" {
 
   export type Route =
     | StaticRoute<"/account/signin">
-    | DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
+    | StaticRoute<"/admin/jobs">
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
     | StaticRoute<"/api/experiments/og-image">
-    | StaticRoute<"/api/openapi">
-    | StaticRoute<"/api/sentry-example-api">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
-    | DynamicRoute<"/data/[id]", { "id": string }>
+    | DynamicRoute<"/api/v1/[...trpc]", { "trpc": string[] }>
-    | StaticRoute<"/data">
+    | StaticRoute<"/api/v1/openapi">
-    | DynamicRoute<"/experiments/[id]", { "id": string }>
+    | StaticRoute<"/dashboard">
+    | DynamicRoute<"/experiments/[experimentSlug]", { "experimentSlug": string }>
     | StaticRoute<"/experiments">
+    | StaticRoute<"/fine-tunes">
     | StaticRoute<"/">
-    | StaticRoute<"/logged-calls">
+    | DynamicRoute<"/invitations/[invitationToken]", { "invitationToken": string }>
     | StaticRoute<"/project/settings">
+    | StaticRoute<"/request-logs">
     | StaticRoute<"/sentry-example-page">
     | StaticRoute<"/world-champs">
     | StaticRoute<"/world-champs/signup">;

app/Dockerfile

@@ -6,13 +6,13 @@ RUN yarn global add pnpm
 # DEPS
 FROM base as deps
 
-WORKDIR /app
+WORKDIR /code
 
-COPY prisma ./
+COPY app/prisma app/package.json ./app/
+COPY client-libs/typescript/package.json ./client-libs/typescript/
+COPY pnpm-lock.yaml pnpm-workspace.yaml ./
 
-COPY package.json pnpm-lock.yaml ./
+RUN cd app && pnpm install --frozen-lockfile
-
-RUN pnpm install --frozen-lockfile
 
 # BUILDER
 FROM base as builder
@@ -23,24 +23,25 @@ ARG NEXT_PUBLIC_SOCKET_URL
 ARG NEXT_PUBLIC_HOST
 ARG NEXT_PUBLIC_SENTRY_DSN
 ARG SENTRY_AUTH_TOKEN
-ARG NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS
 
-WORKDIR /app
+WORKDIR /code
-COPY --from=deps /app/node_modules ./node_modules
+COPY --from=deps /code/node_modules ./node_modules
+COPY --from=deps /code/app/node_modules ./app/node_modules
+COPY --from=deps /code/client-libs/typescript/node_modules ./client-libs/typescript/node_modules
 COPY . .
-RUN SKIP_ENV_VALIDATION=1 pnpm build
+RUN cd app && SKIP_ENV_VALIDATION=1 pnpm build
 
 # RUNNER
 FROM base as runner
-WORKDIR /app
+WORKDIR /code/app
 
 ENV NODE_ENV production
 ENV NEXT_TELEMETRY_DISABLED 1
 
-COPY --from=builder /app/ ./
+COPY --from=builder /code/ /code/
 
 EXPOSE 3000
 ENV PORT 3000
 
 # Run the "run-prod.sh" script
-CMD /app/run-prod.sh
+CMD /code/app/scripts/run-prod.sh

app/next.config.mjs

@@ -36,6 +36,8 @@ let config = {
     });
     return config;
   },
+
+  transpilePackages: ["openpipe"],
 };
 
 config = nextRoutes()(config);

app/package.json

@@ -1,5 +1,6 @@
 {
-  "name": "openpipe",
+  "name": "openpipe-app",
+  "private": true,
   "type": "module",
   "version": "0.1.0",
   "license": "Apache-2.0",
@@ -9,14 +10,15 @@
   },
   "scripts": {
     "build": "next build",
-    "dev:next": "next dev",
+    "dev:next": "TZ=UTC next dev",
     "dev:wss": "pnpm tsx --watch src/wss-server.ts",
-    "dev:worker": "NODE_ENV='development' pnpm tsx --watch src/server/tasks/worker.ts",
+    "worker": "NODE_ENV='development' pnpm tsx --watch src/server/tasks/worker.ts",
-    "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss' 'pnpm dev:worker'",
+    "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss' 'pnpm worker --watch'",
     "postinstall": "prisma generate",
     "lint": "next lint",
-    "start": "next start",
+    "start": "TZ=UTC next start",
-    "codegen": "tsx src/server/scripts/client-codegen.ts",
+    "codegen:clients": "tsx src/server/scripts/client-codegen.ts",
+    "codegen:db": "prisma generate && kysely-codegen --dialect postgres --out-file src/server/db.types.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
     "test": "pnpm vitest"
@@ -24,7 +26,6 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.5.8",
     "@apidevtools/json-schema-ref-parser": "^10.1.0",
-    "@babel/preset-typescript": "^7.22.5",
     "@babel/standalone": "^7.22.9",
     "@chakra-ui/anatomy": "^2.2.0",
     "@chakra-ui/next-js": "^2.1.4",
@@ -37,6 +38,7 @@
     "@monaco-editor/loader": "^1.3.3",
     "@next-auth/prisma-adapter": "^1.0.5",
     "@prisma/client": "^4.14.0",
+    "@sendinblue/client": "^3.3.1",
     "@sentry/nextjs": "^7.61.0",
     "@t3-oss/env-nextjs": "^0.3.1",
     "@tabler/icons-react": "^2.22.0",
@@ -46,6 +48,7 @@
     "@trpc/react-query": "^10.26.0",
     "@trpc/server": "^10.26.0",
     "@vercel/og": "^0.5.9",
+    "archiver": "^6.0.0",
     "ast-types": "^0.14.2",
     "chroma-js": "^2.4.2",
     "concurrently": "^8.2.0",
@@ -58,20 +61,26 @@
     "framer-motion": "^10.12.17",
     "gpt-tokens": "^1.0.10",
     "graphile-worker": "^0.13.0",
+    "human-id": "^4.0.0",
     "immer": "^10.0.2",
     "isolated-vm": "^4.5.0",
     "json-schema-to-typescript": "^13.0.2",
     "json-stringify-pretty-compact": "^4.0.0",
     "jsonschema": "^1.4.1",
     "kysely": "^0.26.1",
+    "kysely-codegen": "^0.10.1",
     "lodash-es": "^4.17.21",
     "lucide-react": "^0.265.0",
+    "marked": "^7.0.3",
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
     "next-query-params": "^4.2.3",
     "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
+    "nodemailer": "^6.9.4",
     "openai": "4.0.0-beta.7",
+    "openpipe": "^0.3.0",
+    "openpipe-dev": "workspace:^",
     "pg": "^8.11.2",
     "pluralize": "^8.0.0",
     "posthog-js": "^1.75.3",
@@ -92,6 +101,7 @@
     "replicate": "^0.12.3",
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
+    "stream-buffers": "^3.0.2",
     "superjson": "1.12.2",
     "trpc-openapi": "^1.2.0",
     "tsx": "^3.12.7",
@@ -104,6 +114,7 @@
   },
   "devDependencies": {
     "@openapi-contrib/openapi-schema-to-json-schema": "^4.0.5",
+    "@types/archiver": "^5.3.2",
     "@types/babel__core": "^7.20.1",
     "@types/babel__standalone": "^7.1.4",
     "@types/chroma-js": "^2.4.0",
@@ -113,12 +124,14 @@
     "@types/json-schema": "^7.0.12",
     "@types/lodash-es": "^4.17.8",
     "@types/node": "^18.16.0",
+    "@types/nodemailer": "^6.4.9",
     "@types/pg": "^8.10.2",
     "@types/pluralize": "^0.0.30",
     "@types/prismjs": "^1.26.0",
     "@types/react": "^18.2.6",
     "@types/react-dom": "^18.2.4",
     "@types/react-syntax-highlighter": "^15.5.7",
+    "@types/stream-buffers": "^3.0.4",
     "@types/uuid": "^9.0.2",
     "@typescript-eslint/eslint-plugin": "^5.59.6",
     "@typescript-eslint/parser": "^5.59.6",
@@ -128,6 +141,7 @@
     "eslint-plugin-unused-imports": "^2.0.0",
     "monaco-editor": "^0.40.0",
     "openapi-typescript": "^6.3.4",
+    "openapi-typescript-codegen": "^0.25.0",
     "prisma": "^4.14.0",
     "raw-loader": "^4.0.2",
     "typescript": "^5.0.4",

app/prisma/deleteOneFineTune.ts

@@ -0,0 +1,12 @@
+import { prisma } from "~/server/db";
+
+// delete most recent fineTune
+const mostRecentFineTune = await prisma.fineTune.findFirst({
+  orderBy: { createdAt: "desc" },
+});
+
+if (mostRecentFineTune) {
+  await prisma.fineTune.delete({
+    where: { id: mostRecentFineTune.id },
+  });
+}

app/prisma/migrations/20230811023536_standardize_on_input_and_output_tokens_as_names/migration.sql

@@ -1,15 +0,0 @@
-/*
-  Warnings:
-
-  - You are about to rename the column `completionTokens` to `outputTokens` on the `ModelResponse` table.
-  - You are about to rename the column `promptTokens` to `inputTokens` on the `ModelResponse` table.
-
-*/
-
--- Rename completionTokens to outputTokens
-ALTER TABLE "ModelResponse"
-RENAME COLUMN "completionTokens" TO "outputTokens";
-
--- Rename promptTokens to inputTokens
-ALTER TABLE "ModelResponse"
-RENAME COLUMN "promptTokens" TO "inputTokens";

app/prisma/migrations/20230811200114_rename_model_response_fields/migration.sql

@@ -0,0 +1,66 @@
+/*
+  Warnings:
+
+  - You are about to rename the column `completionTokens` to `outputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `promptTokens` to `inputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `startTime` on the `LoggedCall` table to `requestedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `startTime` on the `LoggedCallModelResponse` table to `requestedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `endTime` on the `LoggedCallModelResponse` table to `receivedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `error` on the `LoggedCallModelResponse` table to `errorMessage`. Ensure compatibility with application logic.
+  - You are about to rename the column `respStatus` on the `LoggedCallModelResponse` table to `statusCode`. Ensure compatibility with application logic.
+  - You are about to rename the column `totalCost` on the `LoggedCallModelResponse` table to `cost`. Ensure compatibility with application logic.
+  - You are about to rename the column `inputHash` on the `ModelResponse` table to `cacheKey`. Ensure compatibility with application logic.
+  - You are about to rename the column `output` on the `ModelResponse` table to `respPayload`. Ensure compatibility with application logic.
+
+*/
+-- DropIndex
+DROP INDEX "LoggedCall_startTime_idx";
+
+-- DropIndex
+DROP INDEX "ModelResponse_inputHash_idx";
+
+-- Rename completionTokens to outputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "completionTokens" TO "outputTokens";
+
+-- Rename promptTokens to inputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "promptTokens" TO "inputTokens";
+
+-- AlterTable
+ALTER TABLE "LoggedCall"
+RENAME COLUMN "startTime" TO "requestedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "startTime" TO "requestedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "endTime" TO "receivedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "error" TO "errorMessage";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "respStatus" TO "statusCode";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "totalCost" TO "cost";
+
+-- AlterTable
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "inputHash" TO "cacheKey";
+
+-- AlterTable
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "output" TO "respPayload";
+
+-- CreateIndex
+CREATE INDEX "LoggedCall_requestedAt_idx" ON "LoggedCall"("requestedAt");
+
+-- CreateIndex
+CREATE INDEX "ModelResponse_cacheKey_idx" ON "ModelResponse"("cacheKey");

app/prisma/migrations/20230811231429_add_model_to_logged_call/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LoggedCall" ADD COLUMN     "model" TEXT;

app/prisma/migrations/20230814233722_index_tags_by_project/migration.sql

@@ -0,0 +1,22 @@
+-- DropIndex
+DROP INDEX "LoggedCallTag_name_idx";
+DROP INDEX "LoggedCallTag_name_value_idx";
+
+-- AlterTable: Add projectId column without NOT NULL constraint for now
+ALTER TABLE "LoggedCallTag" ADD COLUMN "projectId" UUID;
+
+-- Set the default value
+UPDATE "LoggedCallTag" lct
+SET "projectId" = lc."projectId"
+FROM "LoggedCall" lc
+WHERE lct."loggedCallId" = lc.id;
+
+-- Now set the NOT NULL constraint
+ALTER TABLE "LoggedCallTag" ALTER COLUMN "projectId" SET NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_projectId_name_idx" ON "LoggedCallTag"("projectId", "name");
+CREATE INDEX "LoggedCallTag_projectId_name_value_idx" ON "LoggedCallTag"("projectId", "name", "value");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LoggedCallTag_loggedCallId_name_key" ON "LoggedCallTag"("loggedCallId", "name");

app/prisma/migrations/20230815234811_add_user_invitations/migration.sql

@@ -0,0 +1,25 @@
+-- CreateTable
+CREATE TABLE "UserInvitation" (
+    "id" UUID NOT NULL,
+    "projectId" UUID NOT NULL,
+    "email" TEXT NOT NULL,
+    "role" "ProjectUserRole" NOT NULL,
+    "invitationToken" TEXT NOT NULL,
+    "senderId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "UserInvitation_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "UserInvitation_invitationToken_key" ON "UserInvitation"("invitationToken");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "UserInvitation_projectId_email_key" ON "UserInvitation"("projectId", "email");
+
+-- AddForeignKey
+ALTER TABLE "UserInvitation" ADD CONSTRAINT "UserInvitation_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "UserInvitation" ADD CONSTRAINT "UserInvitation_senderId_fkey" FOREIGN KEY ("senderId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/migrations/20230818055303_add_slug_to_experiment/migration.sql

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2023 Viascom Ltd liab. Co
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+CREATE OR REPLACE FUNCTION nanoid(
+    size int DEFAULT 21,
+    alphabet text DEFAULT '_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+)
+    RETURNS text
+    LANGUAGE plpgsql
+    volatile
+AS
+$$
+DECLARE
+    idBuilder      text := '';
+    counter        int  := 0;
+    bytes          bytea;
+    alphabetIndex  int;
+    alphabetArray  text[];
+    alphabetLength int;
+    mask           int;
+    step           int;
+BEGIN
+    alphabetArray := regexp_split_to_array(alphabet, '');
+    alphabetLength := array_length(alphabetArray, 1);
+    mask := (2 << cast(floor(log(alphabetLength - 1) / log(2)) as int)) - 1;
+    step := cast(ceil(1.6 * mask * size / alphabetLength) AS int);
+
+    while true
+        loop
+            bytes := gen_random_bytes(step);
+            while counter < step
+                loop
+                    alphabetIndex := (get_byte(bytes, counter) & mask) + 1;
+                    if alphabetIndex <= alphabetLength then
+                        idBuilder := idBuilder || alphabetArray[alphabetIndex];
+                        if length(idBuilder) = size then
+                            return idBuilder;
+                        end if;
+                    end if;
+                    counter := counter + 1;
+                end loop;
+
+            counter := 0;
+        end loop;
+END
+$$;
+
+
+-- Make a short_nanoid function that uses the default alphabet and length of 15
+CREATE OR REPLACE FUNCTION short_nanoid()
+    RETURNS text
+    LANGUAGE plpgsql
+    volatile
+AS
+$$
+BEGIN
+    RETURN nanoid(15, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
+END
+$$;
+
+-- AlterTable
+ALTER TABLE "Experiment" ADD COLUMN     "slug" TEXT NOT NULL DEFAULT short_nanoid();
+
+-- For existing experiments, keep the existing id as the slug for backwards compatibility
+UPDATE "Experiment" SET "slug" = "id";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Experiment_slug_key" ON "Experiment"("slug");

app/prisma/migrations/20230822234224_add_fine_tunes/migration.sql

@@ -0,0 +1,48 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `input` on the `DatasetEntry` table. All the data in the column will be lost.
+  - You are about to drop the column `output` on the `DatasetEntry` table. All the data in the column will be lost.
+  - Added the required column `loggedCallId` to the `DatasetEntry` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "DatasetEntry" DROP COLUMN "input",
+DROP COLUMN "output",
+ADD COLUMN     "loggedCallId" UUID NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "DatasetEntry" ADD CONSTRAINT "DatasetEntry_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse" ALTER COLUMN "cost" SET DATA TYPE DOUBLE PRECISION;
+
+-- CreateEnum
+CREATE TYPE "FineTuneStatus" AS ENUM ('PENDING', 'TRAINING', 'AWAITING_DEPLOYMENT', 'DEPLOYING', 'DEPLOYED', 'ERROR');
+
+-- CreateTable
+CREATE TABLE "FineTune" (
+    "id" UUID NOT NULL,
+    "slug" TEXT NOT NULL,
+    "baseModel" TEXT NOT NULL,
+    "status" "FineTuneStatus" NOT NULL DEFAULT 'PENDING',
+    "trainingStartedAt" TIMESTAMP(3),
+    "trainingFinishedAt" TIMESTAMP(3),
+    "deploymentStartedAt" TIMESTAMP(3),
+    "deploymentFinishedAt" TIMESTAMP(3),
+    "datasetId" UUID NOT NULL,
+    "projectId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "FineTune_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "FineTune_slug_key" ON "FineTune"("slug");
+
+-- AddForeignKey
+ALTER TABLE "FineTune" ADD CONSTRAINT "FineTune_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "FineTune" ADD CONSTRAINT "FineTune_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/schema.prisma

@@ -11,7 +11,9 @@ datasource db {
 }
 
 model Experiment {
-    id    String @id @default(uuid()) @db.Uuid
+    id String @id @default(uuid()) @db.Uuid
+
+    slug  String @unique @default(dbgenerated("short_nanoid()"))
     label String
 
     sortIndex Int @default(0)
@@ -112,17 +114,17 @@ model ScenarioVariantCell {
 model ModelResponse {
     id String @id @default(uuid()) @db.Uuid
 
-    inputHash        String
+    cacheKey     String
-    requestedAt      DateTime?
+    requestedAt  DateTime?
-    receivedAt       DateTime?
+    receivedAt   DateTime?
-    output           Json?
+    respPayload  Json?
-    cost             Float?
+    cost         Float?
-    inputTokens      Int?
+    inputTokens  Int?
-    outputTokens     Int?
+    outputTokens Int?
-    statusCode       Int?
+    statusCode   Int?
-    errorMessage     String?
+    errorMessage String?
-    retryTime        DateTime?
+    retryTime    DateTime?
-    outdated         Boolean   @default(false)
+    outdated     Boolean   @default(false)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -131,7 +133,7 @@ model ModelResponse {
     scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
     outputEvaluations     OutputEvaluation[]
 
-    @@index([inputHash])
+    @@index([cacheKey])
 }
 
 enum EvalType {
@@ -179,6 +181,7 @@ model Dataset {
 
     name           String
     datasetEntries DatasetEntry[]
+    fineTunes      FineTune[]
 
     projectId String  @db.Uuid
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -190,8 +193,8 @@ model Dataset {
 model DatasetEntry {
     id String @id @default(uuid()) @db.Uuid
 
-    input  String
+    loggedCallId String @db.Uuid
-    output String?
+    loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
 
     datasetId String   @db.Uuid
     dataset   Dataset? @relation(fields: [datasetId], references: [id], onDelete: Cascade)
@@ -207,13 +210,15 @@ model Project {
     personalProjectUserId String? @unique @db.Uuid
     personalProjectUser   User?   @relation(fields: [personalProjectUserId], references: [id], onDelete: Cascade)
 
-    createdAt    DateTime      @default(now())
+    createdAt              DateTime         @default(now())
-    updatedAt    DateTime      @updatedAt
+    updatedAt              DateTime         @updatedAt
-    projectUsers ProjectUser[]
+    projectUsers           ProjectUser[]
-    experiments  Experiment[]
+    projectUserInvitations UserInvitation[]
-    datasets     Dataset[]
+    experiments            Experiment[]
-    loggedCalls  LoggedCall[]
+    datasets               Dataset[]
-    apiKeys      ApiKey[]
+    loggedCalls            LoggedCall[]
+    fineTunes              FineTune[]
+    apiKeys                ApiKey[]
 }
 
 enum ProjectUserRole {
@@ -256,7 +261,7 @@ model WorldChampEntrant {
 model LoggedCall {
     id String @id @default(uuid()) @db.Uuid
 
-    startTime DateTime
+    requestedAt DateTime
 
     // True if this call was served from the cache, false otherwise
     cacheHit Boolean
@@ -273,12 +278,14 @@ model LoggedCall {
     projectId String   @db.Uuid
     project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
-    tags LoggedCallTag[]
+    model           String?
+    tags            LoggedCallTag[]
+    datasetEntries  DatasetEntry[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    @@index([startTime])
+    @@index([requestedAt])
 }
 
 model LoggedCallModelResponse {
@@ -287,14 +294,14 @@ model LoggedCallModelResponse {
     reqPayload Json
 
     // The HTTP status returned by the model provider
-    respStatus  Int?
+    statusCode  Int?
     respPayload Json?
 
     // Should be null if the request was successful, and some string if the request failed.
-    error String?
+    errorMessage String?
 
-    startTime DateTime
+    requestedAt DateTime
-    endTime   DateTime
+    receivedAt  DateTime
 
     // Note: the function to calculate the cacheKey should include the project
     // ID so we don't share cached responses between projects, which could be an
@@ -308,7 +315,7 @@ model LoggedCallModelResponse {
     outputTokens Int?
     finishReason String?
     completionId String?
-    totalCost    Decimal? @db.Decimal(18, 12)
+    cost         Float?
 
     // The LoggedCall that created this LoggedCallModelResponse
     originalLoggedCallId String     @unique @db.Uuid
@@ -322,15 +329,17 @@ model LoggedCallModelResponse {
 }
 
 model LoggedCallTag {
-    id    String  @id @default(uuid()) @db.Uuid
+    id        String  @id @default(uuid()) @db.Uuid
-    name  String
+    name      String
-    value String?
+    value     String?
+    projectId String  @db.Uuid
 
     loggedCallId String     @db.Uuid
     loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
 
-    @@index([name])
+    @@unique([loggedCallId, name])
-    @@index([name, value])
+    @@index([projectId, name])
+    @@index([projectId, name, value])
 }
 
 model ApiKey {
@@ -339,8 +348,8 @@ model ApiKey {
     name   String
     apiKey String @unique
 
-    projectId String   @db.Uuid
+    projectId String  @db.Uuid
-    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -387,16 +396,33 @@ model User {
 
     role UserRole @default(USER)
 
-    accounts          Account[]
+    accounts            Account[]
-    sessions          Session[]
+    sessions            Session[]
-    projectUsers      ProjectUser[]
+    projectUsers        ProjectUser[]
-    projects          Project[]
+    projects            Project[]
-    worldChampEntrant WorldChampEntrant?
+    worldChampEntrant   WorldChampEntrant?
+    sentUserInvitations UserInvitation[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @default(now()) @updatedAt
 }
 
+model UserInvitation {
+    id String @id @default(uuid()) @db.Uuid
+
+    projectId       String          @db.Uuid
+    project         Project         @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    email           String
+    role            ProjectUserRole
+    invitationToken String          @unique
+    senderId        String          @db.Uuid
+    sender          User            @relation(fields: [senderId], references: [id], onDelete: Cascade)
+    createdAt       DateTime        @default(now())
+    updatedAt       DateTime        @updatedAt
+
+    @@unique([projectId, email])
+}
+
 model VerificationToken {
     identifier String
     token      String   @unique
@@ -404,3 +430,33 @@ model VerificationToken {
 
     @@unique([identifier, token])
 }
+
+enum FineTuneStatus {
+    PENDING
+    TRAINING
+    AWAITING_DEPLOYMENT
+    DEPLOYING
+    DEPLOYED
+    ERROR
+}
+
+model FineTune {
+    id String @id @default(uuid()) @db.Uuid
+
+    slug      String @unique
+    baseModel String
+    status    FineTuneStatus @default(PENDING)
+    trainingStartedAt    DateTime?
+    trainingFinishedAt   DateTime?
+    deploymentStartedAt  DateTime?
+    deploymentFinishedAt DateTime?
+
+    datasetId String   @db.Uuid
+    dataset   Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+
+    projectId String  @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}

app/prisma/seed.ts

@@ -2,6 +2,7 @@ import { prisma } from "~/server/db";
 import dedent from "dedent";
 import { generateNewCell } from "~/server/utils/generateNewCell";
 import { promptConstructorVersion } from "~/promptConstructor/version";
+import { env } from "~/env.mjs";
 
 const defaultId = "11111111-1111-1111-1111-111111111111";
 
@@ -9,6 +10,14 @@ await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
+// Mark all users as admins
+await prisma.user.updateMany({
+  where: {},
+  data: {
+    role: "ADMIN",
+  },
+});
+
 // If there's an existing project, just seed into it
 const project =
   (await prisma.project.findFirst({})) ??
@@ -16,6 +25,20 @@ const project =
     data: { id: defaultId },
   }));
 
+if (env.OPENPIPE_API_KEY) {
+  await prisma.apiKey.upsert({
+    where: {
+      apiKey: env.OPENPIPE_API_KEY,
+    },
+    create: {
+      projectId: project.id,
+      name: "Default API Key",
+      apiKey: env.OPENPIPE_API_KEY,
+    },
+    update: {},
+  });
+}
+
 await prisma.experiment.deleteMany({
   where: {
     id: defaultId,

app/prisma/seedDashboard.ts

@@ -13,6 +13,7 @@ const MODEL_RESPONSE_TEMPLATES: {
   inputTokens: number;
   outputTokens: number;
   finishReason: string;
+  tags: { name: string; value: string }[];
 }[] = [
   {
     reqPayload: {
@@ -79,7 +80,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     },
     respStatus: 200,
     respPayload: {
-      id: "chatcmpl-7lNspqePJWVyXwXebupxb1eMozo6Q",
+      id: "chatcmpl-7",
       model: "gpt-3.5-turbo-0613",
       usage: {
         total_tokens: 241,
@@ -107,6 +108,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 236,
     outputTokens: 5,
     finishReason: "stop",
+    tags: [{ name: "prompt_id", value: "define_func" }],
   },
   {
     reqPayload: {
@@ -165,7 +167,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     },
     respStatus: 200,
     respPayload: {
-      id: "chatcmpl-7lNifmc5AncyAvleZRDBhAcLFYBIT",
+      id: "chatcmpl-7",
       model: "gpt-3.5-turbo-0613",
       usage: {
         total_tokens: 227,
@@ -193,6 +195,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 222,
     outputTokens: 5,
     finishReason: "stop",
+    tags: [],
   },
   {
     reqPayload: {
@@ -207,7 +210,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     },
     respStatus: 200,
     respPayload: {
-      id: "chatcmpl-7lNh1TtrsJVgz3Nj70bKkZZk7xPi7",
+      id: "chatcmpl-7",
       model: "gpt-3.5-turbo-0613",
       usage: {
         total_tokens: 21,
@@ -231,6 +234,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 14,
     outputTokens: 7,
     finishReason: "stop",
+    tags: [{ name: "prompt_id", value: "translate_text" }],
   },
   {
     reqPayload: {
@@ -277,7 +281,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     },
     respStatus: 200,
     respPayload: {
-      id: "chatcmpl-7lQS3MktOT8BTgNEytl9dkyssCQqL",
+      id: "chatcmpl-7",
       model: "gpt-4-0613",
       usage: {
         total_tokens: 2910,
@@ -306,6 +310,10 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 2802,
     outputTokens: 108,
     finishReason: "stop",
+    tags: [
+      { name: "prompt_id", value: "chatcmpl-7" },
+      { name: "some_other_tag", value: "some_other_value" },
+    ],
   },
 ];
 
@@ -331,7 +339,7 @@ const loggedCallsToCreate: Prisma.LoggedCallCreateManyInput[] = [];
 const loggedCallModelResponsesToCreate: Prisma.LoggedCallModelResponseCreateManyInput[] = [];
 const loggedCallsToUpdate: Prisma.LoggedCallUpdateArgs[] = [];
 const loggedCallTagsToCreate: Prisma.LoggedCallTagCreateManyInput[] = [];
-for (let i = 0; i < 1437; i++) {
+for (let i = 0; i < 11437; i++) {
   const loggedCallId = uuidv4();
   const loggedCallModelResponseId = uuidv4();
   const template =
@@ -339,17 +347,18 @@ for (let i = 0; i < 1437; i++) {
     MODEL_RESPONSE_TEMPLATES[Math.floor(Math.random() * MODEL_RESPONSE_TEMPLATES.length)]!;
   const model = template.reqPayload.model;
   // choose random time in the last two weeks, with a bias towards the last few days
-  const startTime = new Date(Date.now() - Math.pow(Math.random(), 2) * 1000 * 60 * 60 * 24 * 14);
+  const requestedAt = new Date(Date.now() - Math.pow(Math.random(), 2) * 1000 * 60 * 60 * 24 * 14);
   // choose random delay anywhere from 2 to 10 seconds later for gpt-4, or 1 to 5 seconds for gpt-3.5
   const delay =
     model === "gpt-4" ? 1000 * 2 + Math.random() * 1000 * 8 : 1000 + Math.random() * 1000 * 4;
-  const endTime = new Date(startTime.getTime() + delay);
+  const receivedAt = new Date(requestedAt.getTime() + delay);
   loggedCallsToCreate.push({
     id: loggedCallId,
     cacheHit: false,
-    startTime,
+    requestedAt,
     projectId: project.id,
-    createdAt: startTime,
+    model: template.reqPayload.model,
+    createdAt: requestedAt,
   });
 
   const { promptTokenPrice, completionTokenPrice } =
@@ -365,21 +374,20 @@ for (let i = 0; i < 1437; i++) {
 
   loggedCallModelResponsesToCreate.push({
     id: loggedCallModelResponseId,
-    startTime,
+    requestedAt,
-    endTime,
+    receivedAt,
     originalLoggedCallId: loggedCallId,
     reqPayload: template.reqPayload,
     respPayload: template.respPayload,
-    respStatus: template.respStatus,
+    statusCode: template.respStatus,
-    error: template.error,
+    errorMessage: template.error,
-    createdAt: startTime,
+    createdAt: requestedAt,
     cacheKey: hashRequest(project.id, template.reqPayload as JsonValue),
-    durationMs: endTime.getTime() - startTime.getTime(),
+    durationMs: receivedAt.getTime() - requestedAt.getTime(),
     inputTokens: template.inputTokens,
     outputTokens: template.outputTokens,
     finishReason: template.finishReason,
-    totalCost:
+    cost: template.inputTokens * promptTokenPrice + template.outputTokens * completionTokenPrice,
-      template.inputTokens * promptTokenPrice + template.outputTokens * completionTokenPrice,
   });
   loggedCallsToUpdate.push({
     where: {
@@ -389,11 +397,14 @@ for (let i = 0; i < 1437; i++) {
       modelResponseId: loggedCallModelResponseId,
     },
   });
-  loggedCallTagsToCreate.push({
+  for (const tag of template.tags) {
-    loggedCallId,
+    loggedCallTagsToCreate.push({
-    name: "$model",
+      projectId: project.id,
-    value: template.reqPayload.model,
+      loggedCallId,
-  });
+      name: tag.name,
+      value: tag.value,
+    });
+  }
 }
 
 await prisma.$transaction([

app/scripts/debug-prod.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+
+set -e
+cd "$(dirname "$0")/.."
+apt-get update
+apt-get install -y htop psql

app/scripts/run-prod.sh

@@ -10,6 +10,4 @@ pnpm tsx src/promptConstructor/migrate.ts
 
 echo "Starting the server"
 
-pnpm concurrently --kill-others \
+pnpm start
-  "pnpm start" \
-  "pnpm tsx src/server/tasks/worker.ts"

app/scripts/run-workers-prod.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+echo "Migrating the database"
+pnpm prisma migrate deploy
+
+echo "Starting 4 workers"
+
+pnpm concurrently "pnpm worker" "pnpm worker" "pnpm worker" "pnpm worker"

app/scripts/test-docker.sh

@@ -0,0 +1,13 @@
+#! /bin/bash
+
+set -e
+
+cd "$(dirname "$0")/../.."
+
+echo "Env is"
+echo $ENVIRONMENT
+
+docker build . --file app/Dockerfile --tag "openpipe-prod"
+
+# Run the image
+docker run --env-file app/.env -it --entrypoint "/bin/bash" "openpipe-prod"

app/sentry.server.config.ts

@@ -3,6 +3,7 @@
 // https://docs.sentry.io/platforms/javascript/guides/nextjs/
 
 import * as Sentry from "@sentry/nextjs";
+import { isError } from "lodash-es";
 import { env } from "~/env.mjs";
 
 if (env.NEXT_PUBLIC_SENTRY_DSN) {
@@ -15,4 +16,10 @@ if (env.NEXT_PUBLIC_SENTRY_DSN) {
     // Setting this option to true will print useful information to the console while you're setting up Sentry.
     debug: false,
   });
+} else {
+  // Install local debug exception handler for rejected promises
+  process.on("unhandledRejection", (reason) => {
+    const reasonDetails = isError(reason) ? reason?.stack : reason;
+    console.log("Unhandled Rejection at:", reasonDetails);
+  });
 }

app/src/components/AutoResizeTextArea.tsx

@@ -1,13 +1,13 @@
 import { Textarea, type TextareaProps } from "@chakra-ui/react";
 import ResizeTextarea from "react-textarea-autosize";
-import React, { useLayoutEffect, useState } from "react";
+import React, { useEffect, useState } from "react";
 
 export const AutoResizeTextarea: React.ForwardRefRenderFunction<
   HTMLTextAreaElement,
   TextareaProps & { minRows?: number }
 > = ({ minRows = 1, overflowY = "hidden", ...props }, ref) => {
   const [isRerendered, setIsRerendered] = useState(false);
-  useLayoutEffect(() => setIsRerendered(true), []);
+  useEffect(() => setIsRerendered(true), []);
 
   return (
     <Textarea

app/src/components/BetaModal.tsx

@@ -0,0 +1,65 @@
+import {
+  Button,
+  Modal,
+  ModalBody,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  VStack,
+  Text,
+  HStack,
+  Icon,
+  Link,
+} from "@chakra-ui/react";
+import { BsStars } from "react-icons/bs";
+import { useSession } from "next-auth/react";
+
+export const BetaModal = ({ isOpen, onClose }: { isOpen: boolean; onClose: () => void }) => {
+  const session = useSession();
+
+  const email = session.data?.user.email ?? "";
+
+  return (
+    <Modal
+      isOpen={isOpen}
+      onClose={onClose}
+      closeOnOverlayClick={false}
+      size={{ base: "xl", md: "2xl" }}
+    >
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={BsStars} />
+            <Text>Beta-Only Feature</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalBody maxW="unset">
+          <VStack spacing={8} py={4} alignItems="flex-start">
+            <Text fontSize="md">
+              This feature is currently in beta. To receive early access to beta-only features, join
+              the waitlist. You'll receive an email at <b>{email}</b> when you're approved.
+            </Text>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack spacing={4}>
+            <Button
+              as={Link}
+              textDecoration="none !important"
+              colorScheme="orange"
+              target="_blank"
+              href={`https://ax3nafkw0jp.typeform.com/to/ZNpYqvAc#email=${email}`}
+            >
+              Join Waitlist
+            </Button>
+            <Button colorScheme="blue" onClick={onClose}>
+              Done
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/ChangeModelModal/ChangeModelModal.tsx

@@ -1,3 +1,4 @@
+import { useState, useMemo, useCallback } from "react";
 import {
   Button,
   HStack,
@@ -14,16 +15,18 @@ import {
   VStack,
 } from "@chakra-ui/react";
 import { type PromptVariant } from "@prisma/client";
-import { isObject, isString } from "lodash-es";
+import { isString } from "lodash-es";
-import { useState } from "react";
 import { RiExchangeFundsFill } from "react-icons/ri";
+
 import { type ProviderModel } from "~/modelProviders/types";
 import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import { lookupModel, modelLabel } from "~/utils/utils";
 import CompareFunctions from "../RefinePromptModal/CompareFunctions";
 import { ModelSearch } from "./ModelSearch";
 import { ModelStatsCard } from "./ModelStatsCard";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { useAppStore } from "~/state/store";
 
 export const ChangeModelModal = ({
   variant,
@@ -32,48 +35,43 @@ export const ChangeModelModal = ({
   variant: PromptVariant;
   onClose: () => void;
 }) => {
+  const editorOptionsMap = useAppStore((s) => s.sharedVariantEditor.editorOptionsMap);
+  const originalPromptFn = useMemo(
+    () => editorOptionsMap[variant.uiId]?.getContent() || "",
+    [editorOptionsMap, variant.uiId],
+  );
+
   const originalModel = lookupModel(variant.modelProvider, variant.model);
   const [selectedModel, setSelectedModel] = useState({
     provider: variant.modelProvider,
     model: variant.model,
   } as ProviderModel);
   const [convertedModel, setConvertedModel] = useState<ProviderModel | undefined>();
-  const visibleScenarios = useVisibleScenarioIds();
+  const [modifiedPromptFn, setModifiedPromptFn] = useState<string>();
-
-  const utils = api.useContext();
 
   const experiment = useExperiment();
 
-  const { mutateAsync: getModifiedPromptMutateAsync, data: modifiedPromptFn } =
+  const { mutateAsync: getModifiedPromptMutateAsync } =
     api.promptVariants.getModifiedPromptFn.useMutation();
 
   const [getModifiedPromptFn, modificationInProgress] = useHandledAsyncCallback(async () => {
     if (!experiment) return;
 
-    await getModifiedPromptMutateAsync({
+    const resp = await getModifiedPromptMutateAsync({
       id: variant.id,
+      originalPromptFn,
       newModel: selectedModel,
     });
+    if (maybeReportError(resp)) return;
+    setModifiedPromptFn(resp.payload);
     setConvertedModel(selectedModel);
   }, [getModifiedPromptMutateAsync, onClose, experiment, variant, selectedModel]);
 
-  const replaceVariantMutation = api.promptVariants.replaceVariant.useMutation();
+  const replaceVariant = useCallback(() => {
-
+    if (!modifiedPromptFn) return;
-  const [replaceVariant, replacementInProgress] = useHandledAsyncCallback(async () => {
+    editorOptionsMap[variant.uiId]?.setContent(modifiedPromptFn);
-    if (
-      !variant.experimentId ||
-      !modifiedPromptFn ||
-      (isObject(modifiedPromptFn) && "status" in modifiedPromptFn)
-    )
-      return;
-    await replaceVariantMutation.mutateAsync({
-      id: variant.id,
-      promptConstructor: modifiedPromptFn,
-      streamScenarios: visibleScenarios,
-    });
-    await utils.promptVariants.list.invalidate();
     onClose();
-  }, [replaceVariantMutation, variant, onClose, modifiedPromptFn]);
+  }, [variant.uiId, editorOptionsMap, onClose, modifiedPromptFn]);
 
   const originalLabel = modelLabel(variant.modelProvider, variant.model);
   const selectedLabel = modelLabel(selectedModel.provider, selectedModel.model);
@@ -130,9 +128,9 @@ export const ChangeModelModal = ({
               colorScheme="blue"
               onClick={replaceVariant}
               minW={24}
-              isDisabled={!convertedModel || modificationInProgress || replacementInProgress}
+              isDisabled={!convertedModel || modificationInProgress}
             >
-              {replacementInProgress ? <Spinner boxSize={4} /> : <Text>Accept</Text>}
+              Accept
             </Button>
           </HStack>
         </ModalFooter>

app/src/components/ChangeModelModal/ModelStatsCard.tsx

@@ -87,7 +87,7 @@ export const ModelStatsCard = ({
               label="Price"
               info={
                 <Text>
-                  ${model.pricePerSecond.toFixed(3)}
+                  ${model.pricePerSecond.toFixed(4)}
                   <Text color="gray.500"> / second</Text>
                 </Text>
               }

app/src/components/CopiableCode.tsx

@@ -1,15 +1,16 @@
-import { HStack, Icon, IconButton, Tooltip, Text } from "@chakra-ui/react";
+import { HStack, Icon, IconButton, Tooltip, Text, type StackProps } from "@chakra-ui/react";
 import { useState } from "react";
 import { MdContentCopy } from "react-icons/md";
 import { useHandledAsyncCallback } from "~/utils/hooks";
 
-const CopiableCode = ({ code }: { code: string }) => {
+const CopiableCode = ({ code, ...rest }: { code: string } & StackProps) => {
   const [copied, setCopied] = useState(false);
 
   const [copyToClipboard] = useHandledAsyncCallback(async () => {
     await navigator.clipboard.writeText(code);
     setCopied(true);
   }, [code]);
+
   return (
     <HStack
       backgroundColor="blackAlpha.800"
@@ -18,9 +19,19 @@ const CopiableCode = ({ code }: { code: string }) => {
       padding={3}
       w="full"
       justifyContent="space-between"
+      alignItems="flex-start"
+      {...rest}
     >
-      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5} overflowX="auto">
+      <Text
+        fontFamily="inconsolata"
+        fontWeight="bold"
+        letterSpacing={0.5}
+        overflowX="auto"
+        whiteSpace="pre-wrap"
+      >
         {code}
+        {/* Necessary for trailing newline to actually be displayed */}
+        {code.endsWith("\n") ? "\n" : ""}
       </Text>
       <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
         <IconButton

app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -1,74 +1,41 @@
-import {
+import { Button, Icon, useDisclosure, Text } from "@chakra-ui/react";
-  Button,
-  Icon,
-  AlertDialog,
-  AlertDialogBody,
-  AlertDialogFooter,
-  AlertDialogHeader,
-  AlertDialogContent,
-  AlertDialogOverlay,
-  useDisclosure,
-  Text,
-} from "@chakra-ui/react";
-
 import { useRouter } from "next/router";
-import { useRef } from "react";
 import { BsTrash } from "react-icons/bs";
+
 import { useAppStore } from "~/state/store";
-import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import DeleteExperimentDialog from "../experiments/DeleteExperimentDialog";
 
 export const DeleteButton = () => {
   const experiment = useExperiment();
-  const mutation = api.experiments.delete.useMutation();
-  const utils = api.useContext();
   const router = useRouter();
 
+  const disclosure = useDisclosure();
+
   const closeDrawer = useAppStore((s) => s.closeDrawer);
-
+  const [onDelete] = useHandledAsyncCallback(async () => {
-  const { isOpen, onOpen, onClose } = useDisclosure();
-  const cancelRef = useRef<HTMLButtonElement>(null);
-
-  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id) return;
-    await mutation.mutateAsync({ id: experiment.data.id });
-    await utils.experiments.list.invalidate();
     await router.push({ pathname: "/experiments" });
     closeDrawer();
-
+  }, [router, closeDrawer]);
-    onClose();
-  }, [mutation, experiment.data?.id, router]);
 
   return (
     <>
-      <Button size="sm" variant="ghost" colorScheme="red" fontWeight="normal" onClick={onOpen}>
+      <Button
+        size="sm"
+        variant="ghost"
+        colorScheme="red"
+        fontWeight="normal"
+        onClick={disclosure.onOpen}
+      >
         <Icon as={BsTrash} boxSize={4} />
         <Text ml={2}>Delete Experiment</Text>
       </Button>
 
-      <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+      <DeleteExperimentDialog
-        <AlertDialogOverlay>
+        experimentId={experiment.data?.id}
-          <AlertDialogContent>
+        onDelete={onDelete}
-            <AlertDialogHeader fontSize="lg" fontWeight="bold">
+        disclosure={disclosure}
-              Delete Experiment
+      />
-            </AlertDialogHeader>
-
-            <AlertDialogBody>
-              If you delete this experiment all the associated prompts and scenarios will be deleted
-              as well. Are you sure?
-            </AlertDialogBody>
-
-            <AlertDialogFooter>
-              <Button ref={cancelRef} onClick={onClose}>
-                Cancel
-              </Button>
-              <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
-                Delete
-              </Button>
-            </AlertDialogFooter>
-          </AlertDialogContent>
-        </AlertDialogOverlay>
-      </AlertDialog>
     </>
   );
 };

app/src/components/InfoCircle.tsx

@@ -0,0 +1,14 @@
+import { Tooltip, Icon, VStack } from "@chakra-ui/react";
+import { RiInformationFill } from "react-icons/ri";
+
+const InfoCircle = ({ tooltipText }: { tooltipText: string }) => {
+  return (
+    <Tooltip label={tooltipText} fontSize="sm" shouldWrapChildren maxW={80}>
+      <VStack>
+        <Icon as={RiInformationFill} boxSize={5} color="gray.500" />
+      </VStack>
+    </Tooltip>
+  );
+};
+
+export default InfoCircle;

app/src/components/InputDropdown.tsx

@@ -0,0 +1,91 @@
+import {
+  Input,
+  InputGroup,
+  InputRightElement,
+  Icon,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  VStack,
+  HStack,
+  Button,
+  Text,
+  useDisclosure,
+  type InputGroupProps,
+} from "@chakra-ui/react";
+
+import { FiChevronDown } from "react-icons/fi";
+import { BiCheck } from "react-icons/bi";
+
+type InputDropdownProps<T> = {
+  options: ReadonlyArray<T>;
+  selectedOption: T;
+  onSelect: (option: T) => void;
+  inputGroupProps?: InputGroupProps;
+};
+
+const InputDropdown = <T,>({
+  options,
+  selectedOption,
+  onSelect,
+  inputGroupProps,
+}: InputDropdownProps<T>) => {
+  const popover = useDisclosure();
+
+  return (
+    <Popover placement="bottom-start" {...popover}>
+      <PopoverTrigger>
+        <InputGroup
+          cursor="pointer"
+          w={(selectedOption as string).length * 14 + 180}
+          {...inputGroupProps}
+        >
+          <Input
+            value={selectedOption as string}
+            // eslint-disable-next-line @typescript-eslint/no-empty-function -- controlled input requires onChange
+            onChange={() => {}}
+            cursor="pointer"
+            borderColor={popover.isOpen ? "blue.500" : undefined}
+            _hover={popover.isOpen ? { borderColor: "blue.500" } : undefined}
+            contentEditable={false}
+            // disable focus
+            onFocus={(e) => {
+              e.target.blur();
+            }}
+          />
+          <InputRightElement>
+            <Icon as={FiChevronDown} />
+          </InputRightElement>
+        </InputGroup>
+      </PopoverTrigger>
+      <PopoverContent boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);" minW={0} w="auto">
+        <VStack spacing={0}>
+          {options?.map((option, index) => (
+            <HStack
+              key={index}
+              as={Button}
+              onClick={() => {
+                onSelect(option);
+                popover.onClose();
+              }}
+              w="full"
+              variant="ghost"
+              justifyContent="space-between"
+              fontWeight="semibold"
+              borderRadius={0}
+              colorScheme="blue"
+              color="black"
+              fontSize="sm"
+              borderBottomWidth={1}
+            >
+              <Text mr={16}>{option as string}</Text>
+              {option === selectedOption && <Icon as={BiCheck} color="blue.500" boxSize={5} />}
+            </HStack>
+          ))}
+        </VStack>
+      </PopoverContent>
+    </Popover>
+  );
+};
+
+export default InputDropdown;

app/src/components/OutputsTable/AddVariantButton.tsx

@@ -8,7 +8,7 @@ import {
   useHandledAsyncCallback,
   useVisibleScenarioIds,
 } from "~/utils/hooks";
-import { cellPadding } from "../constants";
+import { cellPadding } from "./constants";
 import { ActionButton } from "./ScenariosHeader";
 
 export default function AddVariantButton() {
@@ -33,25 +33,11 @@ export default function AddVariantButton() {
     <Flex w="100%" justifyContent="flex-end">
       <ActionButton
         onClick={onClick}
-        py={5}
+        py={7}
         leftIcon={<Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />}
       >
         <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
       </ActionButton>
-      {/* <Button
-        alignItems="center"
-        justifyContent="center"
-        fontWeight="normal"
-        bgColor="transparent"
-        _hover={{ bgColor: "gray.100" }}
-        px={cellPadding.x}
-        onClick={onClick}
-        height="unset"
-        minH={headerMinHeight}
-      >
-        <Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />
-        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
-      </Button> */}
     </Flex>
   );
 }

app/src/components/OutputsTable/EditScenarioVars.tsx

@@ -4,7 +4,7 @@ import { useEffect, useState } from "react";
 import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
-import { maybeReportError } from "~/utils/standardResponses";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
 import { FloatingLabelInput } from "./FloatingLabelInput";
 
 export const ScenarioVar = ({

app/src/components/OutputsTable/OutputCell/CellOptions.tsx

@@ -1,7 +1,7 @@
 import { HStack, Icon, IconButton, Spinner, Tooltip, useDisclosure } from "@chakra-ui/react";
 import { BsArrowClockwise, BsInfoCircle } from "react-icons/bs";
 import { useExperimentAccess } from "~/utils/hooks";
-import ExpandedModal from "./PromptModal";
+import PromptModal from "./PromptModal";
 import { type RouterOutputs } from "~/utils/api";
 
 export const CellOptions = ({
@@ -32,7 +32,7 @@ export const CellOptions = ({
               variant="ghost"
             />
           </Tooltip>
-          <ExpandedModal cell={cell} disclosure={modalDisclosure} />
+          <PromptModal cell={cell} disclosure={modalDisclosure} />
         </>
       )}
       {canModify && (

app/src/components/OutputsTable/OutputCell/CellWrapper.tsx

@@ -0,0 +1,29 @@
+import { type StackProps, VStack } from "@chakra-ui/react";
+import { type RouterOutputs } from "~/utils/api";
+import { type Scenario } from "../types";
+import { CellOptions } from "./CellOptions";
+import { OutputStats } from "./OutputStats";
+
+const CellWrapper: React.FC<
+  StackProps & {
+    cell: RouterOutputs["scenarioVariantCells"]["get"] | undefined;
+    hardRefetching: boolean;
+    hardRefetch: () => void;
+    mostRecentResponse:
+      | NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>["modelResponses"][0]
+      | undefined;
+    scenario: Scenario;
+  }
+> = ({ children, cell, hardRefetching, hardRefetch, mostRecentResponse, scenario, ...props }) => (
+  <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
+    {cell && (
+      <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
+    )}
+    <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
+      {children}
+    </VStack>
+    {mostRecentResponse && <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />}
+  </VStack>
+);
+
+export default CellWrapper;

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,17 +1,16 @@
-import { api } from "~/utils/api";
+import { Text } from "@chakra-ui/react";
-import { type PromptVariant, type Scenario } from "../types";
+import stringify from "json-stringify-pretty-compact";
-import { type StackProps, Text, VStack } from "@chakra-ui/react";
+import { Fragment, useEffect, useState, type ReactElement } from "react";
-import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
-import stringify from "json-stringify-pretty-compact";
-import { type ReactElement, useState, useEffect, Fragment, useCallback } from "react";
-import useSocket from "~/utils/useSocket";
-import { OutputStats } from "./OutputStats";
-import { RetryCountdown } from "./RetryCountdown";
 import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import useSocket from "~/utils/useSocket";
+import { type PromptVariant, type Scenario } from "../types";
+import CellWrapper from "./CellWrapper";
 import { ResponseLog } from "./ResponseLog";
-import { CellOptions } from "./TopActions";
+import { RetryCountdown } from "./RetryCountdown";
 
 const WAITING_MESSAGE_INTERVAL = 20000;
 
@@ -33,7 +32,7 @@ export default function OutputCell({
 
   if (!templateHasVariables) disabledReason = "Add a value to the scenario variables to see output";
 
-  const [refetchInterval, setRefetchInterval] = useState(0);
+  const [refetchInterval, setRefetchInterval] = useState<number | false>(false);
   const { data: cell, isLoading: queryLoading } = api.scenarioVariantCells.get.useQuery(
     { scenarioId: scenario.id, variantId: variant.id },
     { refetchInterval },
@@ -44,7 +43,7 @@ export default function OutputCell({
 
   type OutputSchema = Parameters<typeof provider.normalizeOutput>[0];
 
-  const { mutateAsync: hardRefetchMutate } = api.scenarioVariantCells.forceRefetch.useMutation();
+  const { mutateAsync: hardRefetchMutate } = api.scenarioVariantCells.hardRefetch.useMutation();
   const [hardRefetch, hardRefetching] = useHandledAsyncCallback(async () => {
     await hardRefetchMutate({ scenarioId: scenario.id, variantId: variant.id });
     await utils.scenarioVariantCells.get.invalidate({
@@ -64,42 +63,34 @@ export default function OutputCell({
     cell.retrievalStatus === "PENDING" ||
     cell.retrievalStatus === "IN_PROGRESS" ||
     hardRefetching;
-  useEffect(() => setRefetchInterval(awaitingOutput ? 1000 : 0), [awaitingOutput]);
+
+  useEffect(() => setRefetchInterval(awaitingOutput ? 1000 : false), [awaitingOutput]);
 
   // TODO: disconnect from socket if we're not streaming anymore
   const streamedMessage = useSocket<OutputSchema>(cell?.id);
 
   const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
 
-  const CellWrapper = useCallback(
+  const wrapperProps: Parameters<typeof CellWrapper>[0] = {
-    ({ children, ...props }: StackProps) => (
+    cell,
-      <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
+    hardRefetching,
-        {cell && (
+    hardRefetch,
-          <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
+    mostRecentResponse,
-        )}
+    scenario,
-        <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
+  };
-          {children}
-        </VStack>
-        {mostRecentResponse && (
-          <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
-        )}
-      </VStack>
-    ),
-    [hardRefetching, hardRefetch, mostRecentResponse, scenario, cell],
-  );
 
   if (!vars) return null;
 
   if (!cell && !fetchingOutput)
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <Text color="gray.500">Error retrieving output</Text>
       </CellWrapper>
     );
 
   if (cell && cell.errorMessage) {
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <Text color="red.500">{cell.errorMessage}</Text>
       </CellWrapper>
     );
@@ -107,11 +98,16 @@ export default function OutputCell({
 
   if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
 
-  const showLogs = !streamedMessage && !mostRecentResponse?.output;
+  const showLogs = !streamedMessage && !mostRecentResponse?.respPayload;
 
   if (showLogs)
     return (
-      <CellWrapper alignItems="flex-start" fontFamily="inconsolata, monospace" spacing={0}>
+      <CellWrapper
+        {...wrapperProps}
+        alignItems="flex-start"
+        fontFamily="inconsolata, monospace"
+        spacing={0}
+      >
         {cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
         {cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
         {cell?.modelResponses?.map((response) => {
@@ -120,8 +116,13 @@ export default function OutputCell({
             ? response.receivedAt.getTime()
             : Date.now();
           if (response.requestedAt) {
-            numWaitingMessages = Math.floor(
+            numWaitingMessages = Math.min(
-              (relativeWaitingTime - response.requestedAt.getTime()) / WAITING_MESSAGE_INTERVAL,
+              Math.floor(
+                (relativeWaitingTime - response.requestedAt.getTime()) / WAITING_MESSAGE_INTERVAL,
+              ),
+              // Don't try to render more than 15, it'll use too much CPU and
+              // break the page
+              15,
             );
           }
           return (
@@ -146,9 +147,10 @@ export default function OutputCell({
                 <ResponseLog
                   time={response.receivedAt}
                   title="Response received from API"
-                  message={`statusCode: ${response.statusCode ?? ""}\n ${
+                  message={[
-                    response.errorMessage ?? ""
+                    response.statusCode ? `Status: ${response.statusCode}\n` : "",
-                  }`}
+                    response.errorMessage ?? "",
+                  ].join("")}
                 />
               )}
             </Fragment>
@@ -160,15 +162,15 @@ export default function OutputCell({
       </CellWrapper>
     );
 
-  const normalizedOutput = mostRecentResponse?.output
+  const normalizedOutput = mostRecentResponse?.respPayload
-    ? provider.normalizeOutput(mostRecentResponse?.output)
+    ? provider.normalizeOutput(mostRecentResponse?.respPayload)
     : streamedMessage
     ? provider.normalizeOutput(streamedMessage)
     : null;
 
-  if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
+  if (mostRecentResponse?.respPayload && normalizedOutput?.type === "json") {
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <SyntaxHighlighter
           customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
           language="json"
@@ -187,7 +189,7 @@ export default function OutputCell({
   const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
 
   return (
-    <CellWrapper>
+    <CellWrapper {...wrapperProps}>
       <Text whiteSpace="pre-wrap">{contentToDisplay}</Text>
     </CellWrapper>
   );

app/src/components/OutputsTable/OutputCell/PromptModal.tsx

@@ -5,30 +5,103 @@ import {
   ModalContent,
   ModalHeader,
   ModalOverlay,
+  VStack,
+  Text,
+  Box,
   type UseDisclosureReturn,
+  Link,
 } from "@chakra-ui/react";
-import { type RouterOutputs } from "~/utils/api";
+import { api, type RouterOutputs } from "~/utils/api";
 import { JSONTree } from "react-json-tree";
+import CopiableCode from "~/components/CopiableCode";
 
-export default function ExpandedModal(props: {
+const theme = {
+  scheme: "chalk",
+  author: "chris kempson (http://chriskempson.com)",
+  base00: "transparent",
+  base01: "#202020",
+  base02: "#303030",
+  base03: "#505050",
+  base04: "#b0b0b0",
+  base05: "#d0d0d0",
+  base06: "#e0e0e0",
+  base07: "#f5f5f5",
+  base08: "#fb9fb1",
+  base09: "#eda987",
+  base0A: "#ddb26f",
+  base0B: "#acc267",
+  base0C: "#12cfc0",
+  base0D: "#6fc2ef",
+  base0E: "#e1a3ee",
+  base0F: "#deaf8f",
+};
+
+export default function PromptModal(props: {
   cell: NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>;
   disclosure: UseDisclosureReturn;
 }) {
+  const { data } = api.scenarioVariantCells.getTemplatedPromptMessage.useQuery(
+    {
+      cellId: props.cell.id,
+    },
+    {
+      enabled: props.disclosure.isOpen,
+    },
+  );
+
   return (
-    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="2xl">
+    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="xl">
       <ModalOverlay />
       <ModalContent>
-        <ModalHeader>Prompt</ModalHeader>
+        <ModalHeader>Prompt Details</ModalHeader>
         <ModalCloseButton />
         <ModalBody>
-          <JSONTree
+          <VStack py={4} w="">
-            data={props.cell.prompt}
+            <VStack w="full" alignItems="flex-start">
-            invertTheme={true}
+              <Text fontWeight="bold">Full Prompt</Text>
-            theme="chalk"
+              <Box
-            shouldExpandNodeInitially={() => true}
+                w="full"
-            getItemString={() => ""}
+                p={4}
-            hideRoot
+                alignItems="flex-start"
-          />
+                backgroundColor="blackAlpha.800"
+                borderRadius={4}
+              >
+                <JSONTree
+                  data={props.cell.prompt}
+                  theme={theme}
+                  shouldExpandNodeInitially={() => true}
+                  getItemString={() => ""}
+                  hideRoot
+                />
+              </Box>
+            </VStack>
+            {data?.templatedPrompt && (
+              <VStack w="full" mt={4} alignItems="flex-start">
+                <Text fontWeight="bold">Templated prompt message:</Text>
+                <CopiableCode
+                  w="full"
+                  // bgColor="gray.100"
+                  p={4}
+                  borderWidth={1}
+                  whiteSpace="pre-wrap"
+                  code={data.templatedPrompt}
+                />
+              </VStack>
+            )}
+            {data?.learnMoreUrl && (
+              <Link
+                href={data.learnMoreUrl}
+                isExternal
+                color="blue.500"
+                fontWeight="bold"
+                fontSize="sm"
+                mt={4}
+                alignSelf="flex-end"
+              >
+                Learn More
+              </Link>
+            )}
+          </VStack>
         </ModalBody>
       </ModalContent>
     </Modal>

app/src/components/OutputsTable/ScenarioEditor.tsx

@@ -16,7 +16,7 @@ import {
   VStack,
 } from "@chakra-ui/react";
 import { BsArrowsAngleExpand, BsX } from "react-icons/bs";
-import { cellPadding } from "../constants";
+import { cellPadding } from "./constants";
 import { FloatingLabelInput } from "./FloatingLabelInput";
 import { ScenarioEditorModal } from "./ScenarioEditorModal";
 
@@ -111,25 +111,23 @@ export default function ScenarioEditor({
         onDrop={onReorder}
         backgroundColor={isDragTarget ? "gray.100" : "transparent"}
       >
-        {variableLabels.length === 0 ? (
+        {
-          <Box color="gray.500">
-            {vars.data ? "No scenario variables configured" : "Loading..."}
-          </Box>
-        ) : (
           <VStack spacing={4} flex={1} py={2}>
             <HStack justifyContent="space-between" w="100%" align="center" spacing={0}>
               <Text flex={1}>Scenario</Text>
-              <Tooltip label="Expand" hasArrow>
+              {variableLabels.length && (
-                <IconButton
+                <Tooltip label="Expand" hasArrow>
-                  aria-label="Expand"
+                  <IconButton
-                  icon={<Icon as={BsArrowsAngleExpand} boxSize={3} />}
+                    aria-label="Expand"
-                  onClick={() => setScenarioEditorModalOpen(true)}
+                    icon={<Icon as={BsArrowsAngleExpand} boxSize={3} />}
-                  size="xs"
+                    onClick={() => setScenarioEditorModalOpen(true)}
-                  colorScheme="gray"
+                    size="xs"
-                  color="gray.500"
+                    colorScheme="gray"
-                  variant="ghost"
+                    color="gray.500"
-                />
+                    variant="ghost"
-              </Tooltip>
+                  />
+                </Tooltip>
+              )}
               {canModify && props.canHide && (
                 <Tooltip label="Delete" hasArrow>
                   <IconButton
@@ -150,31 +148,38 @@ export default function ScenarioEditor({
                 </Tooltip>
               )}
             </HStack>
-            {variableLabels.map((key) => {
+
-              const value = values[key] ?? "";
+            {variableLabels.length === 0 ? (
-              return (
+              <Box color="gray.500">
-                <FloatingLabelInput
+                {vars.data ? "No scenario variables configured" : "Loading..."}
-                  key={key}
+              </Box>
-                  label={key}
+            ) : (
-                  isDisabled={!canModify}
+              variableLabels.map((key) => {
-                  style={{ width: "100%" }}
+                const value = values[key] ?? "";
-                  maxHeight={32}
+                return (
-                  value={value}
+                  <FloatingLabelInput
-                  onChange={(e) => {
+                    key={key}
-                    setValues((prev) => ({ ...prev, [key]: e.target.value }));
+                    label={key}
-                  }}
+                    isDisabled={!canModify}
-                  onKeyDown={(e) => {
+                    style={{ width: "100%" }}
-                    if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
+                    maxHeight={32}
-                      e.preventDefault();
+                    value={value}
-                      e.currentTarget.blur();
+                    onChange={(e) => {
-                      onSave();
+                      setValues((prev) => ({ ...prev, [key]: e.target.value }));
-                    }
+                    }}
-                  }}
+                    onKeyDown={(e) => {
-                  onMouseEnter={() => setVariableInputHovered(true)}
+                      if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
-                  onMouseLeave={() => setVariableInputHovered(false)}
+                        e.preventDefault();
-                />
+                        e.currentTarget.blur();
-              );
+                        onSave();
-            })}
+                      }
+                    }}
+                    onMouseEnter={() => setVariableInputHovered(true)}
+                    onMouseLeave={() => setVariableInputHovered(false)}
+                  />
+                );
+              })
+            )}
             {hasChanged && (
               <HStack justify="right">
                 <Button
@@ -192,7 +197,7 @@ export default function ScenarioEditor({
               </HStack>
             )}
           </VStack>
-        )}
+        }
       </HStack>
       {scenarioEditorModalOpen && (
         <ScenarioEditorModal

app/src/components/OutputsTable/ScenarioEditorModal.tsx

@@ -65,11 +65,11 @@ export const ScenarioEditorModal = ({
     <Modal
       isOpen
       onClose={onClose}
-      size={{ base: "xl", sm: "2xl", md: "3xl", lg: "5xl", xl: "7xl" }}
+      size={{ base: "xl", sm: "2xl", md: "3xl", lg: "4xl", xl: "5xl" }}
     >
       <ModalOverlay />
       <ModalContent w={1200}>
-        <ModalHeader />
+        <ModalHeader>Edit Scenario</ModalHeader>
         <ModalCloseButton />
         <ModalBody maxW="unset">
           <VStack spacing={8}>

app/src/components/OutputsTable/ScenarioPaginator.tsx

@@ -1,21 +1,16 @@
+import { type StackProps } from "@chakra-ui/react";
+
 import { useScenarios } from "~/utils/hooks";
 import Paginator from "../Paginator";
 
-const ScenarioPaginator = () => {
+const ScenarioPaginator = (props: StackProps) => {
   const { data } = useScenarios();
 
   if (!data) return null;
 
-  const { scenarios, startIndex, lastPage, count } = data;
+  const { count } = data;
 
-  return (
+  return <Paginator count={count} condense {...props} />;
-    <Paginator
-      numItemsLoaded={scenarios.length}
-      startIndex={startIndex}
-      lastPage={lastPage}
-      count={count}
-    />
-  );
 };
 
 export default ScenarioPaginator;

app/src/components/OutputsTable/ScenarioRow.tsx

@@ -10,6 +10,8 @@ const ScenarioRow = (props: {
   variants: PromptVariant[];
   canHide: boolean;
   rowStart: number;
+  isFirst: boolean;
+  isLast: boolean;
 }) => {
   const [isHovered, setIsHovered] = useState(false);
 
@@ -21,10 +23,14 @@ const ScenarioRow = (props: {
         onMouseEnter={() => setIsHovered(true)}
         onMouseLeave={() => setIsHovered(false)}
         sx={isHovered ? highlightStyle : undefined}
-        borderLeftWidth={1}
+        bgColor="white"
-        {...borders}
         rowStart={props.rowStart}
         colStart={1}
+        borderLeftWidth={1}
+        borderTopWidth={props.isFirst ? 1 : 0}
+        borderTopLeftRadius={props.isFirst ? 8 : 0}
+        borderBottomLeftRadius={props.isLast ? 8 : 0}
+        {...borders}
       >
         <ScenarioEditor scenario={props.scenario} hovered={isHovered} canHide={props.canHide} />
       </GridItem>
@@ -34,8 +40,12 @@ const ScenarioRow = (props: {
           onMouseEnter={() => setIsHovered(true)}
           onMouseLeave={() => setIsHovered(false)}
           sx={isHovered ? highlightStyle : undefined}
+          bgColor="white"
           rowStart={props.rowStart}
           colStart={i + 2}
+          borderTopWidth={props.isFirst ? 1 : 0}
+          borderTopRightRadius={props.isFirst && i === props.variants.length - 1 ? 8 : 0}
+          borderBottomRightRadius={props.isLast && i === props.variants.length - 1 ? 8 : 0}
           {...borders}
         >
           <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -11,7 +11,7 @@ import {
   IconButton,
   Spinner,
 } from "@chakra-ui/react";
-import { cellPadding } from "../constants";
+import { cellPadding } from "./constants";
 import {
   useExperiment,
   useExperimentAccess,
@@ -48,7 +48,7 @@ export const ScenariosHeader = () => {
   );
 
   return (
-    <HStack w="100%" pb={cellPadding.y} pt={0} align="center" spacing={0}>
+    <HStack w="100%" py={cellPadding.y} px={cellPadding.x} align="center" spacing={0}>
       <Text fontSize={16} fontWeight="bold">
         Scenarios ({scenarios.data?.count})
       </Text>
@@ -57,11 +57,16 @@ export const ScenariosHeader = () => {
           <MenuButton
             as={IconButton}
             mt={1}
+            ml={2}
             variant="ghost"
             aria-label="Edit Scenarios"
             icon={<Icon as={loading ? Spinner : BsGear} />}
+            maxW={8}
+            minW={8}
+            minH={8}
+            maxH={8}
           />
-          <MenuList fontSize="md" zIndex="dropdown" mt={-3}>
+          <MenuList fontSize="md" zIndex="dropdown" mt={-1}>
             <MenuItem
               icon={<Icon as={BsPlus} boxSize={6} mx="-5px" />}
               onClick={() => onAddScenario(false)}

app/src/components/OutputsTable/VariantEditor.tsx

@@ -10,7 +10,7 @@ import {
 } from "@chakra-ui/react";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { FiMaximize, FiMinimize } from "react-icons/fi";
-import { editorBackground } from "~/state/sharedVariantEditor.slice";
+import { type CreatedEditor, editorBackground } from "~/state/sharedVariantEditor.slice";
 import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 import {
@@ -24,8 +24,10 @@ import { type PromptVariant } from "./types";
 export default function VariantEditor(props: { variant: PromptVariant }) {
   const { canModify } = useExperimentAccess();
   const monaco = useAppStore.use.sharedVariantEditor.monaco();
-  const editorRef = useRef<ReturnType<NonNullable<typeof monaco>["editor"]["create"]> | null>(null);
+  const updateOptionsForEditor = useAppStore.use.sharedVariantEditor.updateOptionsForEditor();
+  const editorRef = useRef<CreatedEditor | null>(null);
   const containerRef = useRef<HTMLDivElement | null>(null);
+  const lastSavedFnRef = useRef(props.variant.promptConstructor);
   const [editorId] = useState(() => `editor_${Math.random().toString(36).substring(7)}`);
   const [isChanged, setIsChanged] = useState(false);
 
@@ -48,22 +50,18 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
   }, [isFullscreen, toggleFullscreen]);
 
   const lastSavedFn = props.variant.promptConstructor;
+  useEffect(() => {
+    // Store in ref so that we can access it dynamically
+    lastSavedFnRef.current = lastSavedFn;
+  }, [lastSavedFn]);
 
   const modifierKey = useModifierKeyLabel();
 
   const checkForChanges = useCallback(() => {
     if (!editorRef.current) return;
     const currentFn = editorRef.current.getValue();
-    setIsChanged(currentFn.length > 0 && currentFn !== lastSavedFn);
+    setIsChanged(currentFn.length > 0 && currentFn !== lastSavedFnRef.current);
-  }, [lastSavedFn]);
+  }, [editorRef]);
-
-  const matchUpdatedSavedFn = useCallback(() => {
-    if (!editorRef.current) return;
-    editorRef.current.setValue(lastSavedFn);
-    setIsChanged(false);
-  }, [lastSavedFn]);
-
-  useEffect(matchUpdatedSavedFn, [matchUpdatedSavedFn, lastSavedFn]);
 
   const replaceVariant = api.promptVariants.replaceVariant.useMutation();
   const utils = api.useContext();
@@ -110,7 +108,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
     setIsChanged(false);
 
     await utils.promptVariants.list.invalidate();
-  }, [checkForChanges]);
+  }, [checkForChanges, replaceVariant.mutateAsync]);
 
   useEffect(() => {
     if (monaco) {
@@ -136,6 +134,11 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
         readOnly: !canModify,
       });
 
+      updateOptionsForEditor(props.variant.uiId, {
+        getContent: () => editorRef.current?.getValue() || "",
+        setContent: (content) => editorRef.current?.setValue(content),
+      });
+
       // Workaround because otherwise the commands only work on whatever
       // editor was loaded on the page last.
       // https://github.com/microsoft/monaco-editor/issues/2947#issuecomment-1422265201
@@ -155,7 +158,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
         });
       });
 
-      editorRef.current.onDidChangeModelContent(checkForChanges);
+      const checkForChangesListener = editorRef.current.onDidChangeModelContent(checkForChanges);
 
       const resizeObserver = new ResizeObserver(() => {
         editorRef.current?.layout();
@@ -164,6 +167,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
 
       return () => {
         resizeObserver.disconnect();
+        checkForChangesListener.dispose();
         editorRef.current?.dispose();
       };
     }
@@ -171,7 +175,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
     // We intentionally skip the onSave and props.savedConfig dependencies here because
     // we don't want to re-render the editor from scratch
     /* eslint-disable-next-line react-hooks/exhaustive-deps */
-  }, [monaco, editorId]);
+  }, [monaco, editorId, updateOptionsForEditor]);
 
   useEffect(() => {
     if (!editorRef.current) return;

app/src/components/OutputsTable/VariantHeader/VariantHeader.tsx

@@ -1,11 +1,11 @@
 import { useState, type DragEvent } from "react";
-import { type PromptVariant } from "../OutputsTable/types";
+import { type PromptVariant } from "../types";
 import { api } from "~/utils/api";
 import { RiDraggable } from "react-icons/ri";
 import { useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
 import { HStack, Icon, Text, GridItem, type GridItemProps } from "@chakra-ui/react"; // Changed here
 import { cellPadding, headerMinHeight } from "../constants";
-import AutoResizeTextArea from "../AutoResizeTextArea";
+import AutoResizeTextArea from "../../AutoResizeTextArea";
 import VariantHeaderMenuButton from "./VariantHeaderMenuButton";
 
 export default function VariantHeader(
@@ -84,6 +84,7 @@ export default function VariantHeader(
     >
       <HStack
         spacing={2}
+        py={2}
         alignItems="flex-start"
         minH={headerMinHeight}
         draggable={!isInputHovered}
@@ -102,7 +103,9 @@ export default function VariantHeader(
           setIsDragTarget(false);
         }}
         onDrop={onReorder}
-        backgroundColor={isDragTarget ? "gray.200" : "gray.100"}
+        backgroundColor={isDragTarget ? "gray.200" : "white"}
+        borderTopLeftRadius={gridItemProps.borderTopLeftRadius}
+        borderTopRightRadius={gridItemProps.borderTopRightRadius}
         h="full"
       >
         <Icon

app/src/components/OutputsTable/VariantHeader/VariantHeaderMenuButton.tsx

@@ -1,6 +1,4 @@
-import { type PromptVariant } from "../OutputsTable/types";
+import { useState } from "react";
-import { api } from "~/utils/api";
-import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
 import {
   Icon,
   Menu,
@@ -14,10 +12,13 @@ import {
 } from "@chakra-ui/react";
 import { BsFillTrashFill, BsGear, BsStars } from "react-icons/bs";
 import { FaRegClone } from "react-icons/fa";
-import { useState } from "react";
-import { RefinePromptModal } from "../RefinePromptModal/RefinePromptModal";
 import { RiExchangeFundsFill } from "react-icons/ri";
-import { ChangeModelModal } from "../ChangeModelModal/ChangeModelModal";
+
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { type PromptVariant } from "../types";
+import { RefinePromptModal } from "../../RefinePromptModal/RefinePromptModal";
+import { ChangeModelModal } from "../../ChangeModelModal/ChangeModelModal";
 
 export default function VariantHeaderMenuButton({
   variant,

app/src/components/OutputsTable/VariantStats.tsx

@@ -1,6 +1,6 @@
 import { HStack, Icon, Text, useToken } from "@chakra-ui/react";
 import { type PromptVariant } from "./types";
-import { cellPadding } from "../constants";
+import { cellPadding } from "./constants";
 import { api } from "~/utils/api";
 import chroma from "chroma-js";
 import { BsCurrencyDollar } from "react-icons/bs";
@@ -21,14 +21,18 @@ export default function VariantStats(props: { variant: PromptVariant }) {
         outputTokens: 0,
         scenarioCount: 0,
         outputCount: 0,
+        awaitingCompletions: false,
         awaitingEvals: false,
       },
       refetchInterval,
     },
   );
 
-  // Poll every two seconds while we are waiting for LLM retrievals to finish
+  // Poll every five seconds while we are waiting for LLM retrievals to finish
-  useEffect(() => setRefetchInterval(data.awaitingEvals ? 5000 : 0), [data.awaitingEvals]);
+  useEffect(
+    () => setRefetchInterval(data.awaitingCompletions || data.awaitingEvals ? 5000 : 0),
+    [data.awaitingCompletions, data.awaitingEvals],
+  );
 
   const [passColor, neutralColor, failColor] = useToken("colors", [
     "green.500",

app/src/components/OutputsTable/index.tsx

@@ -3,13 +3,14 @@ import { api } from "~/utils/api";
 import AddVariantButton from "./AddVariantButton";
 import ScenarioRow from "./ScenarioRow";
 import VariantEditor from "./VariantEditor";
-import VariantHeader from "../VariantHeader/VariantHeader";
+import VariantHeader from "./VariantHeader/VariantHeader";
 import VariantStats from "./VariantStats";
 import { ScenariosHeader } from "./ScenariosHeader";
 import { borders } from "./styles";
 import { useScenarios } from "~/utils/hooks";
 import ScenarioPaginator from "./ScenarioPaginator";
 import { Fragment } from "react";
+import useScrolledPast from "./useHasScrolledPast";
 
 export default function OutputsTable({ experimentId }: { experimentId: string | undefined }) {
   const variants = api.promptVariants.list.useQuery(
@@ -18,6 +19,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
   );
 
   const scenarios = useScenarios();
+  const shouldFlattenHeader = useScrolledPast(50);
 
   if (!variants.data || !scenarios.data) return null;
 
@@ -53,20 +55,30 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
           colStart: i + 2,
           borderLeftWidth: i === 0 ? 1 : 0,
           marginLeft: i === 0 ? "-1px" : 0,
-          backgroundColor: "gray.100",
+          backgroundColor: "white",
         };
+        const isFirst = i === 0;
+        const isLast = i === variants.data.length - 1;
         return (
           <Fragment key={variant.uiId}>
             <VariantHeader
               variant={variant}
               canHide={variants.data.length > 1}
               rowStart={1}
+              borderTopLeftRadius={isFirst && !shouldFlattenHeader ? 8 : 0}
+              borderTopRightRadius={isLast && !shouldFlattenHeader ? 8 : 0}
               {...sharedProps}
             />
             <GridItem rowStart={2} {...sharedProps}>
               <VariantEditor variant={variant} />
             </GridItem>
-            <GridItem rowStart={3} {...sharedProps}>
+            <GridItem
+              rowStart={3}
+              {...sharedProps}
+              borderBottomLeftRadius={isFirst ? 8 : 0}
+              borderBottomRightRadius={isLast ? 8 : 0}
+              boxShadow="5px 5px 15px 1px rgba(0, 0, 0, 0.1);"
+            >
               <VariantStats variant={variant} />
             </GridItem>
           </Fragment>
@@ -77,7 +89,6 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
         colSpan={allCols - 1}
         rowStart={variantHeaderRows + 1}
         colStart={1}
-        {...borders}
         borderRightWidth={0}
       >
         <ScenariosHeader />
@@ -90,6 +101,8 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
           scenario={scenario}
           variants={variants.data}
           canHide={visibleScenariosCount > 1}
+          isFirst={i === 0}
+          isLast={i === visibleScenariosCount - 1}
         />
       ))}
       <GridItem

app/src/components/OutputsTable/useHasScrolledPast.tsx

@@ -0,0 +1,34 @@
+import { useState, useEffect } from "react";
+
+const useScrolledPast = (scrollThreshold: number) => {
+  const [hasScrolledPast, setHasScrolledPast] = useState(true);
+
+  useEffect(() => {
+    const container = document.getElementById("output-container");
+
+    if (!container) {
+      console.warn('Element with id "outputs-container" not found.');
+      return;
+    }
+
+    const checkScroll = () => {
+      const { scrollTop } = container;
+
+      // Check if scrollTop is greater than or equal to scrollThreshold
+      setHasScrolledPast(scrollTop > scrollThreshold);
+    };
+
+    checkScroll();
+
+    container.addEventListener("scroll", checkScroll);
+
+    // Cleanup
+    return () => {
+      container.removeEventListener("scroll", checkScroll);
+    };
+  }, []);
+
+  return hasScrolledPast;
+};
+
+export default useScrolledPast;

app/src/components/Paginator.tsx

@@ -1,77 +1,126 @@
-import { Box, HStack, IconButton } from "@chakra-ui/react";
 import {
-  BsChevronDoubleLeft,
+  HStack,
-  BsChevronDoubleRight,
+  IconButton,
-  BsChevronLeft,
+  Text,
-  BsChevronRight,
+  Select,
-} from "react-icons/bs";
+  type StackProps,
-import { usePage } from "~/utils/hooks";
+  Icon,
+  useBreakpointValue,
+} from "@chakra-ui/react";
+import React, { useCallback } from "react";
+import { FiChevronsLeft, FiChevronsRight, FiChevronLeft, FiChevronRight } from "react-icons/fi";
+import { usePageParams } from "~/utils/hooks";
 
-const Paginator = ({
+const pageSizeOptions = [10, 25, 50, 100];
-  numItemsLoaded,
+
-  startIndex,
+const Paginator = ({ count, ...props }: { count: number; condense?: boolean } & StackProps) => {
-  lastPage,
+  const { page, pageSize, setPageParams } = usePageParams();
-  count,
+
-}: {
+  const lastPage = Math.ceil(count / pageSize);
-  numItemsLoaded: number;
+
-  startIndex: number;
+  const updatePageSize = useCallback(
-  lastPage: number;
+    (newPageSize: number) => {
-  count: number;
+      const newPage = Math.floor(((page - 1) * pageSize) / newPageSize) + 1;
-}) => {
+      setPageParams({ page: newPage, pageSize: newPageSize }, "replace");
-  const [page, setPage] = usePage();
+    },
+    [page, pageSize, setPageParams],
+  );
 
   const nextPage = () => {
     if (page < lastPage) {
-      setPage(page + 1, "replace");
+      setPageParams({ page: page + 1 }, "replace");
     }
   };
 
   const prevPage = () => {
     if (page > 1) {
-      setPage(page - 1, "replace");
+      setPageParams({ page: page - 1 }, "replace");
     }
   };
 
-  const goToLastPage = () => setPage(lastPage, "replace");
+  const goToLastPage = () => setPageParams({ page: lastPage }, "replace");
-  const goToFirstPage = () => setPage(1, "replace");
+  const goToFirstPage = () => setPageParams({ page: 1 }, "replace");
+
+  const isMobile = useBreakpointValue({ base: true, md: false });
+  const condense = isMobile || props.condense;
+
+  if (count === 0) return null;
 
   return (
-    <HStack pt={4}>
+    <HStack
-      <IconButton
+      pt={4}
-        variant="ghost"
+      spacing={8}
-        size="sm"
+      justifyContent={condense ? "flex-start" : "space-between"}
-        onClick={goToFirstPage}
+      alignItems="center"
-        isDisabled={page === 1}
+      w="full"
-        aria-label="Go to first page"
+      {...props}
-        icon={<BsChevronDoubleLeft />}
+    >
-      />
+      {!condense && (
-      <IconButton
+        <>
-        variant="ghost"
+          <HStack>
-        size="sm"
+            <Text>Rows</Text>
-        onClick={prevPage}
+            <Select
-        isDisabled={page === 1}
+              value={pageSize}
-        aria-label="Previous page"
+              onChange={(e) => updatePageSize(parseInt(e.target.value))}
-        icon={<BsChevronLeft />}
+              w={20}
-      />
+              backgroundColor="white"
-      <Box>
+            >
-        {startIndex}-{startIndex + numItemsLoaded - 1} / {count}
+              {pageSizeOptions.map((option) => (
-      </Box>
+                <option key={option} value={option}>
-      <IconButton
+                  {option}
-        variant="ghost"
+                </option>
-        size="sm"
+              ))}
-        onClick={nextPage}
+            </Select>
-        isDisabled={page === lastPage}
+          </HStack>
-        aria-label="Next page"
+          <Text>
-        icon={<BsChevronRight />}
+            Page {page} of {lastPage}
-      />
+          </Text>
-      <IconButton
+        </>
-        variant="ghost"
+      )}
-        size="sm"
+
-        onClick={goToLastPage}
+      <HStack>
-        isDisabled={page === lastPage}
+        <IconButton
-        aria-label="Go to last page"
+          variant="outline"
-        icon={<BsChevronDoubleRight />}
+          size="sm"
-      />
+          onClick={goToFirstPage}
+          isDisabled={page === 1}
+          aria-label="Go to first page"
+          icon={<Icon as={FiChevronsLeft} boxSize={5} strokeWidth={1.5} />}
+          bgColor="white"
+        />
+        <IconButton
+          variant="outline"
+          size="sm"
+          onClick={prevPage}
+          isDisabled={page === 1}
+          aria-label="Previous page"
+          icon={<Icon as={FiChevronLeft} boxSize={5} strokeWidth={1.5} />}
+          bgColor="white"
+        />
+        {condense && (
+          <Text>
+            Page {page} of {lastPage}
+          </Text>
+        )}
+        <IconButton
+          variant="outline"
+          size="sm"
+          onClick={nextPage}
+          isDisabled={page === lastPage}
+          aria-label="Next page"
+          icon={<Icon as={FiChevronRight} boxSize={5} strokeWidth={1.5} />}
+          bgColor="white"
+        />
+        <IconButton
+          variant="outline"
+          size="sm"
+          onClick={goToLastPage}
+          isDisabled={page === lastPage}
+          aria-label="Go to last page"
+          icon={<Icon as={FiChevronsRight} boxSize={5} strokeWidth={1.5} />}
+          bgColor="white"
+        />
+      </HStack>
     </HStack>
   );
 };

app/src/components/RefinePromptModal/RefinePromptModal.tsx

@@ -1,3 +1,4 @@
+import { useState, useMemo, useCallback } from "react";
 import {
   Button,
   Modal,
@@ -9,22 +10,23 @@ import {
   ModalOverlay,
   VStack,
   Text,
-  Spinner,
   HStack,
   Icon,
   SimpleGrid,
 } from "@chakra-ui/react";
 import { BsStars } from "react-icons/bs";
 import { api } from "~/utils/api";
-import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { useHandledAsyncCallback } from "~/utils/hooks";
 import { type PromptVariant } from "@prisma/client";
-import { useState } from "react";
+
 import CompareFunctions from "./CompareFunctions";
 import { CustomInstructionsInput } from "../CustomInstructionsInput";
 import { RefineAction } from "./RefineAction";
-import { isObject, isString } from "lodash-es";
+import { isString } from "lodash-es";
 import { type RefinementAction, type SupportedProvider } from "~/modelProviders/types";
 import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { useAppStore } from "~/state/store";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
 
 export const RefinePromptModal = ({
   variant,
@@ -33,19 +35,23 @@ export const RefinePromptModal = ({
   variant: PromptVariant;
   onClose: () => void;
 }) => {
-  const utils = api.useContext();
+  const editorOptionsMap = useAppStore((s) => s.sharedVariantEditor.editorOptionsMap);
-  const visibleScenarios = useVisibleScenarioIds();
+  const originalPromptFn = useMemo(
+    () => editorOptionsMap[variant.uiId]?.getContent() || "",
+    [editorOptionsMap, variant.uiId],
+  );
 
   const refinementActions =
     frontendModelProviders[variant.modelProvider as SupportedProvider].refinementActions || {};
 
-  const { mutateAsync: getModifiedPromptMutateAsync, data: refinedPromptFn } =
+  const { mutateAsync: getModifiedPromptMutateAsync } =
     api.promptVariants.getModifiedPromptFn.useMutation();
   const [instructions, setInstructions] = useState<string>("");
 
   const [activeRefineActionLabel, setActiveRefineActionLabel] = useState<string | undefined>(
     undefined,
   );
+  const [refinedPromptFn, setRefinedPromptFn] = useState<string>();
 
   const [getModifiedPromptFn, modificationInProgress] = useHandledAsyncCallback(
     async (label?: string) => {
@@ -54,31 +60,22 @@ export const RefinePromptModal = ({
         ? (refinementActions[label] as RefinementAction).instructions
         : instructions;
       setActiveRefineActionLabel(label);
-      await getModifiedPromptMutateAsync({
+      const resp = await getModifiedPromptMutateAsync({
         id: variant.id,
+        originalPromptFn,
         instructions: updatedInstructions,
       });
+      if (maybeReportError(resp)) return;
+      setRefinedPromptFn(resp.payload);
     },
     [getModifiedPromptMutateAsync, onClose, variant, instructions, setActiveRefineActionLabel],
   );
 
-  const replaceVariantMutation = api.promptVariants.replaceVariant.useMutation();
+  const replaceVariant = useCallback(() => {
-
+    if (!refinedPromptFn) return;
-  const [replaceVariant, replacementInProgress] = useHandledAsyncCallback(async () => {
+    editorOptionsMap[variant.uiId]?.setContent(refinedPromptFn);
-    if (
-      !variant.experimentId ||
-      !refinedPromptFn ||
-      (isObject(refinedPromptFn) && "status" in refinedPromptFn)
-    )
-      return;
-    await replaceVariantMutation.mutateAsync({
-      id: variant.id,
-      promptConstructor: refinedPromptFn,
-      streamScenarios: visibleScenarios,
-    });
-    await utils.promptVariants.list.invalidate();
     onClose();
-  }, [replaceVariantMutation, variant, onClose, refinedPromptFn]);
+  }, [variant.uiId, editorOptionsMap, onClose, refinedPromptFn]);
 
   return (
     <Modal
@@ -126,7 +123,7 @@ export const RefinePromptModal = ({
               />
             </VStack>
             <CompareFunctions
-              originalFunction={variant.promptConstructor}
+              originalFunction={originalPromptFn}
               newFunction={isString(refinedPromptFn) ? refinedPromptFn : undefined}
               maxH="40vh"
             />
@@ -139,9 +136,9 @@ export const RefinePromptModal = ({
               colorScheme="blue"
               onClick={replaceVariant}
               minW={24}
-              isDisabled={replacementInProgress || !refinedPromptFn}
+              isDisabled={!refinedPromptFn}
             >
-              {replacementInProgress ? <Spinner boxSize={4} /> : <Text>Accept</Text>}
+              Accept
             </Button>
           </HStack>
         </ModalFooter>

app/src/components/StatsCard.tsx

@@ -1,26 +0,0 @@
-import { VStack, HStack, type StackProps, Text, Divider } from "@chakra-ui/react";
-import Link, { type LinkProps } from "next/link";
-
-const StatsCard = ({
-  title,
-  href,
-  children,
-  ...rest
-}: { title: string; href: string } & StackProps & LinkProps) => {
-  return (
-    <VStack flex={1} borderWidth={1} padding={4} borderRadius={4} borderColor="gray.300" {...rest}>
-      <HStack w="full" justifyContent="space-between">
-        <Text fontSize="md" fontWeight="bold">
-          {title}
-        </Text>
-        <Link href={href}>
-          <Text color="blue">View all</Text>
-        </Link>
-      </HStack>
-      <Divider />
-      {children}
-    </VStack>
-  );
-};
-
-export default StatsCard;

app/src/components/dashboard/LoggedCallTable.tsx

@@ -1,201 +0,0 @@
-import {
-  Box,
-  Card,
-  CardHeader,
-  Heading,
-  Table,
-  Tbody,
-  Td,
-  Th,
-  Thead,
-  Tr,
-  Tooltip,
-  Collapse,
-  HStack,
-  VStack,
-  IconButton,
-  useToast,
-  Icon,
-  Button,
-  ButtonGroup,
-} from "@chakra-ui/react";
-import dayjs from "dayjs";
-import relativeTime from "dayjs/plugin/relativeTime";
-import { ChevronUpIcon, ChevronDownIcon, CopyIcon } from "lucide-react";
-import { useMemo, useState } from "react";
-import { type RouterOutputs, api } from "~/utils/api";
-import SyntaxHighlighter from "react-syntax-highlighter";
-import { atelierCaveLight } from "react-syntax-highlighter/dist/cjs/styles/hljs";
-import stringify from "json-stringify-pretty-compact";
-import Link from "next/link";
-
-dayjs.extend(relativeTime);
-
-type LoggedCall = RouterOutputs["dashboard"]["loggedCalls"][0];
-
-const FormattedJson = ({ json }: { json: any }) => {
-  const jsonString = stringify(json, { maxLength: 40 });
-  const toast = useToast();
-
-  const copyToClipboard = async (text: string) => {
-    try {
-      await navigator.clipboard.writeText(text);
-      toast({
-        title: "Copied to clipboard",
-        status: "success",
-        duration: 2000,
-      });
-    } catch (err) {
-      toast({
-        title: "Failed to copy to clipboard",
-        status: "error",
-        duration: 2000,
-      });
-    }
-  };
-
-  return (
-    <Box position="relative" fontSize="sm" borderRadius="md" overflow="hidden">
-      <SyntaxHighlighter
-        customStyle={{ overflowX: "unset" }}
-        language="json"
-        style={atelierCaveLight}
-        lineProps={{
-          style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
-        }}
-        wrapLines
-      >
-        {jsonString}
-      </SyntaxHighlighter>
-      <IconButton
-        aria-label="Copy"
-        icon={<CopyIcon />}
-        position="absolute"
-        top={1}
-        right={1}
-        size="xs"
-        variant="ghost"
-        onClick={() => void copyToClipboard(jsonString)}
-      />
-    </Box>
-  );
-};
-
-function TableRow({
-  loggedCall,
-  isExpanded,
-  onToggle,
-}: {
-  loggedCall: LoggedCall;
-  isExpanded: boolean;
-  onToggle: () => void;
-}) {
-  const isError = loggedCall.modelResponse?.respStatus !== 200;
-  const timeAgo = dayjs(loggedCall.startTime).fromNow();
-  const fullTime = dayjs(loggedCall.startTime).toString();
-
-  const model = useMemo(
-    () => loggedCall.tags.find((tag) => tag.name.startsWith("$model"))?.value,
-    [loggedCall.tags],
-  );
-
-  return (
-    <>
-      <Tr
-        onClick={onToggle}
-        key={loggedCall.id}
-        _hover={{ bgColor: "gray.100", cursor: "pointer" }}
-        sx={{
-          "> td": { borderBottom: "none" },
-        }}
-      >
-        <Td>
-          <Icon boxSize={6} as={isExpanded ? ChevronUpIcon : ChevronDownIcon} />
-        </Td>
-        <Td>
-          <Tooltip label={fullTime} placement="top">
-            <Box whiteSpace="nowrap" minW="120px">
-              {timeAgo}
-            </Box>
-          </Tooltip>
-        </Td>
-        <Td width="100%">{model}</Td>
-        <Td isNumeric>{((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2)}s</Td>
-        <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
-        <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
-        <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
-          {loggedCall.modelResponse?.respStatus ?? "No response"}
-        </Td>
-      </Tr>
-      <Tr>
-        <Td colSpan={8} p={0}>
-          <Collapse in={isExpanded} unmountOnExit={true}>
-            <VStack p={4} align="stretch">
-              <HStack align="stretch">
-                <VStack flex={1} align="stretch">
-                  <Heading size="sm">Input</Heading>
-                  <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
-                </VStack>
-                <VStack flex={1} align="stretch">
-                  <Heading size="sm">Output</Heading>
-                  <FormattedJson json={loggedCall.modelResponse?.respPayload} />
-                </VStack>
-              </HStack>
-              <ButtonGroup alignSelf="flex-end">
-                <Button as={Link} colorScheme="blue" href={{ pathname: "/experiments" }}>
-                  Experiments
-                </Button>
-              </ButtonGroup>
-            </VStack>
-          </Collapse>
-        </Td>
-      </Tr>
-    </>
-  );
-}
-
-export default function LoggedCallTable() {
-  const [expandedRow, setExpandedRow] = useState<string | null>(null);
-  const loggedCalls = api.dashboard.loggedCalls.useQuery({});
-
-  return (
-    <Card variant="outline" width="100%" overflow="hidden">
-      <CardHeader>
-        <Heading as="h3" size="sm">
-          Logged Calls
-        </Heading>
-      </CardHeader>
-      <Table>
-        <Thead>
-          <Tr>
-            <Th />
-            <Th>Time</Th>
-            <Th>Model</Th>
-            <Th isNumeric>Duration</Th>
-            <Th isNumeric>Input tokens</Th>
-            <Th isNumeric>Output tokens</Th>
-            <Th isNumeric>Status</Th>
-          </Tr>
-        </Thead>
-        <Tbody>
-          {loggedCalls.data?.map((loggedCall) => {
-            return (
-              <TableRow
-                key={loggedCall.id}
-                loggedCall={loggedCall}
-                isExpanded={loggedCall.id === expandedRow}
-                onToggle={() => {
-                  if (loggedCall.id === expandedRow) {
-                    setExpandedRow(null);
-                  } else {
-                    setExpandedRow(loggedCall.id);
-                  }
-                }}
-              />
-            );
-          })}
-        </Tbody>
-      </Table>
-    </Card>
-  );
-}

app/src/components/dashboard/LoggedCallsTable.tsx

@@ -0,0 +1,51 @@
+import { Card, CardHeader, Heading, Table, Tbody, HStack, Button, Text } from "@chakra-ui/react";
+import { useState } from "react";
+import Link from "next/link";
+import { useLoggedCalls } from "~/utils/hooks";
+import { EmptyTableRow, TableHeader, TableRow } from "../requestLogs/TableRow";
+
+export default function LoggedCallsTable() {
+  const { data: loggedCalls } = useLoggedCalls(false);
+
+  const [expandedRow, setExpandedRow] = useState<string | null>(null);
+
+  return (
+    <Card width="100%" overflow="hidden">
+      <CardHeader>
+        <HStack justifyContent="space-between">
+          <Heading as="h3" size="sm">
+            Request Logs
+          </Heading>
+          <Button as={Link} href="/request-logs" variant="ghost" colorScheme="blue">
+            <Text>View All</Text>
+          </Button>
+        </HStack>
+      </CardHeader>
+      <Table>
+        <TableHeader />
+        <Tbody>
+          {loggedCalls?.calls.length ? (
+            loggedCalls?.calls.map((loggedCall) => {
+              return (
+                <TableRow
+                  key={loggedCall.id}
+                  loggedCall={loggedCall}
+                  isExpanded={loggedCall.id === expandedRow}
+                  onToggle={() => {
+                    if (loggedCall.id === expandedRow) {
+                      setExpandedRow(null);
+                    } else {
+                      setExpandedRow(loggedCall.id);
+                    }
+                  }}
+                />
+              );
+            })
+          ) : (
+            <EmptyTableRow filtersApplied={false} />
+          )}
+        </Tbody>
+      </Table>
+    </Card>
+  );
+}

app/src/components/dashboard/UsageGraph.tsx

@@ -0,0 +1,61 @@
+import {
+  ResponsiveContainer,
+  LineChart,
+  Line,
+  XAxis,
+  YAxis,
+  CartesianGrid,
+  Tooltip,
+  Legend,
+} from "recharts";
+import { useMemo } from "react";
+
+import { useSelectedProject } from "~/utils/hooks";
+import dayjs from "~/utils/dayjs";
+import { api } from "~/utils/api";
+
+export default function UsageGraph() {
+  const { data: selectedProject } = useSelectedProject();
+
+  const stats = api.dashboard.stats.useQuery(
+    { projectId: selectedProject?.id ?? "" },
+    { enabled: !!selectedProject },
+  );
+
+  const data = useMemo(() => {
+    return (
+      stats.data?.periods.map(({ period, numQueries, cost }) => ({
+        period,
+        Requests: numQueries,
+        "Total Spent (USD)": parseFloat(cost.toString()),
+      })) || []
+    );
+  }, [stats.data]);
+
+  return (
+    <ResponsiveContainer width="100%" height={400}>
+      <LineChart data={data} margin={{ top: 5, right: 20, left: 10, bottom: 5 }}>
+        <XAxis dataKey="period" tickFormatter={(str: string) => dayjs(str).format("MMM D")} />
+        <YAxis yAxisId="left" dataKey="Requests" orientation="left" stroke="#8884d8" />
+        <YAxis
+          yAxisId="right"
+          dataKey="Total Spent (USD)"
+          orientation="right"
+          unit="$"
+          stroke="#82ca9d"
+        />
+        <Tooltip />
+        <Legend />
+        <CartesianGrid stroke="#f5f5f5" />
+        <Line dataKey="Requests" stroke="#8884d8" yAxisId="left" dot={false} strokeWidth={2} />
+        <Line
+          dataKey="Total Spent (USD)"
+          stroke="#82ca9d"
+          yAxisId="right"
+          dot={false}
+          strokeWidth={2}
+        />
+      </LineChart>
+    </ResponsiveContainer>
+  );
+}

app/src/components/datasets/DatasetCard.tsx

@@ -1,112 +0,0 @@
-import {
-  HStack,
-  Icon,
-  VStack,
-  Text,
-  Divider,
-  Spinner,
-  AspectRatio,
-  SkeletonText,
-} from "@chakra-ui/react";
-import { RiDatabase2Line } from "react-icons/ri";
-import { formatTimePast } from "~/utils/dayjs";
-import Link from "next/link";
-import { useRouter } from "next/router";
-import { BsPlusSquare } from "react-icons/bs";
-import { api } from "~/utils/api";
-import { useHandledAsyncCallback } from "~/utils/hooks";
-import { useAppStore } from "~/state/store";
-
-type DatasetData = {
-  name: string;
-  numEntries: number;
-  id: string;
-  createdAt: Date;
-  updatedAt: Date;
-};
-
-export const DatasetCard = ({ dataset }: { dataset: DatasetData }) => {
-  return (
-    <AspectRatio ratio={1.2} w="full">
-      <VStack
-        as={Link}
-        href={{ pathname: "/data/[id]", query: { id: dataset.id } }}
-        bg="gray.50"
-        _hover={{ bg: "gray.100" }}
-        transition="background 0.2s"
-        cursor="pointer"
-        borderColor="gray.200"
-        borderWidth={1}
-        p={4}
-        justify="space-between"
-      >
-        <HStack w="full" color="gray.700" justify="center">
-          <Icon as={RiDatabase2Line} boxSize={4} />
-          <Text fontWeight="bold">{dataset.name}</Text>
-        </HStack>
-        <HStack h="full" spacing={4} flex={1} align="center">
-          <CountLabel label="Rows" count={dataset.numEntries} />
-        </HStack>
-        <HStack w="full" color="gray.500" fontSize="xs" textAlign="center">
-          <Text flex={1}>Created {formatTimePast(dataset.createdAt)}</Text>
-          <Divider h={4} orientation="vertical" />
-          <Text flex={1}>Updated {formatTimePast(dataset.updatedAt)}</Text>
-        </HStack>
-      </VStack>
-    </AspectRatio>
-  );
-};
-
-const CountLabel = ({ label, count }: { label: string; count: number }) => {
-  return (
-    <VStack alignItems="center" flex={1}>
-      <Text color="gray.500" fontWeight="bold">
-        {label}
-      </Text>
-      <Text fontSize="sm" color="gray.500">
-        {count}
-      </Text>
-    </VStack>
-  );
-};
-
-export const NewDatasetCard = () => {
-  const router = useRouter();
-  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
-  const createMutation = api.datasets.create.useMutation();
-  const [createDataset, isLoading] = useHandledAsyncCallback(async () => {
-    const newDataset = await createMutation.mutateAsync({ projectId: selectedProjectId ?? "" });
-    await router.push({ pathname: "/data/[id]", query: { id: newDataset.id } });
-  }, [createMutation, router, selectedProjectId]);
-
-  return (
-    <AspectRatio ratio={1.2} w="full">
-      <VStack
-        align="center"
-        justify="center"
-        _hover={{ cursor: "pointer", bg: "gray.50" }}
-        transition="background 0.2s"
-        cursor="pointer"
-        borderColor="gray.200"
-        borderWidth={1}
-        p={4}
-        onClick={createDataset}
-      >
-        <Icon as={isLoading ? Spinner : BsPlusSquare} boxSize={8} />
-        <Text display={{ base: "none", md: "block" }} ml={2}>
-          New Dataset
-        </Text>
-      </VStack>
-    </AspectRatio>
-  );
-};
-
-export const DatasetCardSkeleton = () => (
-  <AspectRatio ratio={1.2} w="full">
-    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="gray.50">
-      <SkeletonText noOfLines={1} w="80%" />
-      <SkeletonText noOfLines={2} w="60%" />
-      <SkeletonText noOfLines={1} w="80%" />
-    </VStack>
-  </AspectRatio>
-);

app/src/components/datasets/DatasetEntriesPaginator.tsx

@@ -1,21 +0,0 @@
-import { useDatasetEntries } from "~/utils/hooks";
-import Paginator from "../Paginator";
-
-const DatasetEntriesPaginator = () => {
-  const { data } = useDatasetEntries();
-
-  if (!data) return null;
-
-  const { entries, startIndex, lastPage, count } = data;
-
-  return (
-    <Paginator
-      numItemsLoaded={entries.length}
-      startIndex={startIndex}
-      lastPage={lastPage}
-      count={count}
-    />
-  );
-};
-
-export default DatasetEntriesPaginator;

app/src/components/datasets/DatasetEntriesTable.tsx

@@ -1,31 +0,0 @@
-import { type StackProps, VStack, Table, Th, Tr, Thead, Tbody, Text } from "@chakra-ui/react";
-import { useDatasetEntries } from "~/utils/hooks";
-import TableRow from "./TableRow";
-import DatasetEntriesPaginator from "./DatasetEntriesPaginator";
-
-const DatasetEntriesTable = (props: StackProps) => {
-  const { data } = useDatasetEntries();
-
-  return (
-    <VStack justifyContent="space-between" {...props}>
-      <Table variant="simple" sx={{ "table-layout": "fixed", width: "full" }}>
-        <Thead>
-          <Tr>
-            <Th>Input</Th>
-            <Th>Output</Th>
-          </Tr>
-        </Thead>
-        <Tbody>{data?.entries.map((entry) => <TableRow key={entry.id} entry={entry} />)}</Tbody>
-      </Table>
-      {(!data || data.entries.length) === 0 ? (
-        <Text alignSelf="flex-start" pl={6} color="gray.500">
-          No entries found
-        </Text>
-      ) : (
-        <DatasetEntriesPaginator />
-      )}
-    </VStack>
-  );
-};
-
-export default DatasetEntriesTable;

app/src/components/datasets/DatasetHeaderButtons/DatasetHeaderButtons.tsx

@@ -1,26 +0,0 @@
-import { Button, HStack, useDisclosure } from "@chakra-ui/react";
-import { BiImport } from "react-icons/bi";
-import { BsStars } from "react-icons/bs";
-
-import { GenerateDataModal } from "./GenerateDataModal";
-
-export const DatasetHeaderButtons = () => {
-  const generateModalDisclosure = useDisclosure();
-
-  return (
-    <>
-      <HStack>
-        <Button leftIcon={<BiImport />} colorScheme="blue" variant="ghost">
-          Import Data
-        </Button>
-        <Button leftIcon={<BsStars />} colorScheme="blue" onClick={generateModalDisclosure.onOpen}>
-          Generate Data
-        </Button>
-      </HStack>
-      <GenerateDataModal
-        isOpen={generateModalDisclosure.isOpen}
-        onClose={generateModalDisclosure.onClose}
-      />
-    </>
-  );
-};

app/src/components/datasets/DatasetHeaderButtons/GenerateDataModal.tsx

@@ -1,128 +0,0 @@
-import {
-  Modal,
-  ModalBody,
-  ModalCloseButton,
-  ModalContent,
-  ModalHeader,
-  ModalOverlay,
-  ModalFooter,
-  Text,
-  HStack,
-  VStack,
-  Icon,
-  NumberInput,
-  NumberInputField,
-  NumberInputStepper,
-  NumberIncrementStepper,
-  NumberDecrementStepper,
-  Button,
-} from "@chakra-ui/react";
-import { BsStars } from "react-icons/bs";
-import { useState } from "react";
-import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
-import { api } from "~/utils/api";
-import AutoResizeTextArea from "~/components/AutoResizeTextArea";
-
-export const GenerateDataModal = ({
-  isOpen,
-  onClose,
-}: {
-  isOpen: boolean;
-  onClose: () => void;
-}) => {
-  const utils = api.useContext();
-
-  const datasetId = useDataset().data?.id;
-
-  const [numToGenerate, setNumToGenerate] = useState<number>(20);
-  const [inputDescription, setInputDescription] = useState<string>(
-    "Each input should contain an email body. Half of the emails should contain event details, and the other half should not.",
-  );
-  const [outputDescription, setOutputDescription] = useState<string>(
-    `Each output should contain "true" or "false", where "true" indicates that the email contains event details.`,
-  );
-
-  const generateEntriesMutation = api.datasetEntries.autogenerateEntries.useMutation();
-
-  const [generateEntries, generateEntriesInProgress] = useHandledAsyncCallback(async () => {
-    if (!inputDescription || !outputDescription || !numToGenerate || !datasetId) return;
-    await generateEntriesMutation.mutateAsync({
-      datasetId,
-      inputDescription,
-      outputDescription,
-      numToGenerate,
-    });
-    await utils.datasetEntries.list.invalidate();
-    onClose();
-  }, [
-    generateEntriesMutation,
-    onClose,
-    inputDescription,
-    outputDescription,
-    numToGenerate,
-    datasetId,
-  ]);
-
-  return (
-    <Modal isOpen={isOpen} onClose={onClose} size={{ base: "xl", sm: "2xl", md: "3xl" }}>
-      <ModalOverlay />
-      <ModalContent w={1200}>
-        <ModalHeader>
-          <HStack>
-            <Icon as={BsStars} />
-            <Text>Generate Data</Text>
-          </HStack>
-        </ModalHeader>
-        <ModalCloseButton />
-        <ModalBody maxW="unset">
-          <VStack w="full" spacing={8} padding={8} alignItems="flex-start">
-            <VStack alignItems="flex-start" spacing={2}>
-              <Text fontWeight="bold">Number of Rows:</Text>
-              <NumberInput
-                step={5}
-                defaultValue={15}
-                min={0}
-                max={100}
-                onChange={(valueString) => setNumToGenerate(parseInt(valueString) || 0)}
-                value={numToGenerate}
-                w="24"
-              >
-                <NumberInputField />
-                <NumberInputStepper>
-                  <NumberIncrementStepper />
-                  <NumberDecrementStepper />
-                </NumberInputStepper>
-              </NumberInput>
-            </VStack>
-            <VStack alignItems="flex-start" w="full" spacing={2}>
-              <Text fontWeight="bold">Input Description:</Text>
-              <AutoResizeTextArea
-                value={inputDescription}
-                onChange={(e) => setInputDescription(e.target.value)}
-                placeholder="Each input should contain..."
-              />
-            </VStack>
-            <VStack alignItems="flex-start" w="full" spacing={2}>
-              <Text fontWeight="bold">Output Description (optional):</Text>
-              <AutoResizeTextArea
-                value={outputDescription}
-                onChange={(e) => setOutputDescription(e.target.value)}
-                placeholder="The output should contain..."
-              />
-            </VStack>
-          </VStack>
-        </ModalBody>
-        <ModalFooter>
-          <Button
-            colorScheme="blue"
-            isLoading={generateEntriesInProgress}
-            isDisabled={!numToGenerate || !inputDescription || !outputDescription}
-            onClick={generateEntries}
-          >
-            Generate
-          </Button>
-        </ModalFooter>
-      </ModalContent>
-    </Modal>
-  );
-};

app/src/components/datasets/TableRow.tsx

@@ -1,13 +0,0 @@
-import { Td, Tr } from "@chakra-ui/react";
-import { type DatasetEntry } from "@prisma/client";
-
-const TableRow = ({ entry }: { entry: DatasetEntry }) => {
-  return (
-    <Tr key={entry.id}>
-      <Td>{entry.input}</Td>
-      <Td>{entry.output}</Td>
-    </Tr>
-  );
-};
-
-export default TableRow;

app/src/components/experiments/DeleteExperimentDialog.tsx

@@ -0,0 +1,66 @@
+import { useRef } from "react";
+import {
+  type UseDisclosureReturn,
+  AlertDialog,
+  AlertDialogOverlay,
+  AlertDialogContent,
+  AlertDialogHeader,
+  AlertDialogBody,
+  AlertDialogFooter,
+  Button,
+} from "@chakra-ui/react";
+import { api } from "~/utils/api";
+
+import { useHandledAsyncCallback } from "~/utils/hooks";
+
+const DeleteExperimentDialog = ({
+  experimentId,
+  onDelete,
+  disclosure,
+}: {
+  experimentId?: string;
+  onDelete?: () => void;
+  disclosure: UseDisclosureReturn;
+}) => {
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const mutation = api.experiments.delete.useMutation();
+  const utils = api.useContext();
+
+  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
+    if (!experimentId) return;
+    await mutation.mutateAsync({ id: experimentId });
+    await utils.experiments.list.invalidate();
+    onDelete?.();
+
+    disclosure.onClose();
+  }, [mutation, experimentId, disclosure.onClose]);
+
+  return (
+    <AlertDialog leastDestructiveRef={cancelRef} {...disclosure}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Delete Experiment
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            If you delete this experiment all the associated prompts and scenarios will be deleted
+            as well. Are you sure?
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={disclosure.onClose}>
+              Cancel
+            </Button>
+            <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
+              Delete
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};
+
+export default DeleteExperimentDialog;

app/src/components/experiments/ExperimentCard.tsx

@@ -1,3 +1,4 @@
+import { type MouseEvent, useState } from "react";
 import {
   HStack,
   Icon,
@@ -7,44 +8,60 @@ import {
   Spinner,
   AspectRatio,
   SkeletonText,
+  Card,
+  useDisclosure,
+  Box,
+  Menu,
+  MenuButton,
+  MenuList,
+  MenuItem,
+  IconButton,
+  useToast,
 } from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
-import { formatTimePast } from "~/utils/dayjs";
 import Link from "next/link";
 import { useRouter } from "next/router";
-import { BsPlusSquare } from "react-icons/bs";
+import { BsPlusSquare, BsThreeDotsVertical, BsLink45Deg, BsTrash } from "react-icons/bs";
-import { api } from "~/utils/api";
+
+import { formatTimePast } from "~/utils/dayjs";
+import { type RouterOutputs, api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
 import { useAppStore } from "~/state/store";
+import DeleteExperimentDialog from "./DeleteExperimentDialog";
 
-type ExperimentData = {
+export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["list"][0] }) => {
-  testScenarioCount: number;
+  const [isMenuHovered, setIsMenuHovered] = useState(false);
-  promptVariantCount: number;
-  id: string;
-  label: string;
-  sortIndex: number;
-  createdAt: Date;
-  updatedAt: Date;
-};
 
-export const ExperimentCard = ({ exp }: { exp: ExperimentData }) => {
   return (
-    <AspectRatio ratio={1.2} w="full">
+    <Card
+      w="full"
+      h="full"
+      cursor="pointer"
+      p={4}
+      bg="white"
+      borderRadius={4}
+      _hover={{ bg: isMenuHovered ? undefined : "gray.100" }}
+      transition="background 0.2s"
+      aspectRatio={1.2}
+    >
       <VStack
         as={Link}
-        href={{ pathname: "/experiments/[id]", query: { id: exp.id } }}
+        w="full"
-        bg="gray.50"
+        h="full"
-        _hover={{ bg: "gray.100" }}
+        href={{ pathname: "/experiments/[experimentSlug]", query: { experimentSlug: exp.slug } }}
-        transition="background 0.2s"
-        cursor="pointer"
-        borderColor="gray.200"
-        borderWidth={1}
-        p={4}
         justify="space-between"
       >
-        <HStack w="full" color="gray.700" justify="center">
+        <HStack w="full" justify="space-between" spacing={0}>
-          <Icon as={RiFlaskLine} boxSize={4} />
+          <Box w={6} />
-          <Text fontWeight="bold">{exp.label}</Text>
+          <HStack color="gray.700" justify="center">
+            <Icon as={RiFlaskLine} boxSize={4} />
+            <Text fontWeight="bold">{exp.label}</Text>
+          </HStack>
+          <CardMenu
+            experimentId={exp.id}
+            experimentSlug={exp.slug}
+            setIsMenuHovered={setIsMenuHovered}
+          />
         </HStack>
         <HStack h="full" spacing={4} flex={1} align="center">
           <CountLabel label="Variants" count={exp.promptVariantCount} />
@@ -57,7 +74,76 @@ export const ExperimentCard = ({ exp }: { exp: ExperimentData }) => {
           <Text flex={1}>Updated {formatTimePast(exp.updatedAt)}</Text>
         </HStack>
       </VStack>
-    </AspectRatio>
+    </Card>
+  );
+};
+
+const CardMenu = ({
+  experimentId,
+  experimentSlug,
+  setIsMenuHovered,
+}: {
+  experimentId: string;
+  experimentSlug: string;
+  setIsMenuHovered: (isHovered: boolean) => void;
+}) => {
+  const deleteDisclosure = useDisclosure();
+  const menuDisclosure = useDisclosure();
+  const toast = useToast();
+  const [copyShareLink] = useHandledAsyncCallback(
+    async (e: MouseEvent<HTMLButtonElement>) => {
+      if (typeof window === "undefined") return;
+      e.preventDefault();
+      e.stopPropagation();
+      const shareLink = `${window.location.origin}/experiments/${experimentSlug}`;
+      await navigator.clipboard.writeText(shareLink);
+      toast({
+        title: "Share link copied to clipboard",
+        status: "success",
+        duration: 2000,
+        isClosable: true,
+      });
+      menuDisclosure.onClose();
+    },
+    [toast, menuDisclosure.onClose, experimentSlug],
+  );
+  return (
+    <>
+      <Menu isLazy {...menuDisclosure}>
+        <MenuButton
+          as={IconButton}
+          aria-label="Options"
+          icon={<BsThreeDotsVertical />}
+          variant="ghost"
+          onClick={(e) => {
+            e.preventDefault();
+            e.stopPropagation();
+            menuDisclosure.onOpen();
+          }}
+          onMouseEnter={() => setIsMenuHovered(true)}
+          onMouseLeave={() => setIsMenuHovered(false)}
+          boxSize={6}
+          minW={0}
+        />
+        <MenuList>
+          <MenuItem icon={<Icon as={BsLink45Deg} boxSize={5} />} onClick={copyShareLink}>
+            Copy Link
+          </MenuItem>
+          <MenuItem
+            icon={<Icon as={BsTrash} boxSize={5} />}
+            onClick={(e) => {
+              e.preventDefault();
+              e.stopPropagation();
+              deleteDisclosure.onOpen();
+            }}
+            color="red.500"
+          >
+            Delete
+          </MenuItem>
+        </MenuList>
+      </Menu>
+      <DeleteExperimentDialog experimentId={experimentId} disclosure={deleteDisclosure} />
+    </>
   );
 };
 
@@ -83,36 +169,34 @@ export const NewExperimentCard = () => {
       projectId: selectedProjectId ?? "",
     });
     await router.push({
-      pathname: "/experiments/[id]",
+      pathname: "/experiments/[experimentSlug]",
-      query: { id: newExperiment.id },
+      query: { experimentSlug: newExperiment.slug },
     });
   }, [createMutation, router, selectedProjectId]);
 
   return (
-    <AspectRatio ratio={1.2} w="full">
+    <Card
-      <VStack
+      w="full"
-        align="center"
+      h="full"
-        justify="center"
+      cursor="pointer"
-        _hover={{ cursor: "pointer", bg: "gray.50" }}
+      p={4}
-        transition="background 0.2s"
+      bg="white"
-        cursor="pointer"
+      borderRadius={4}
-        borderColor="gray.200"
+      _hover={{ bg: "gray.100" }}
-        borderWidth={1}
+      transition="background 0.2s"
-        p={4}
+      aspectRatio={1.2}
-        onClick={createExperiment}
+    >
-      >
+      <VStack align="center" justify="center" w="full" h="full" p={4} onClick={createExperiment}>
         <Icon as={isLoading ? Spinner : BsPlusSquare} boxSize={8} />
-        <Text display={{ base: "none", md: "block" }} ml={2}>
+        <Text ml={2}>New Experiment</Text>
-          New Experiment
-        </Text>
       </VStack>
-    </AspectRatio>
+    </Card>
   );
 };
 
 export const ExperimentCardSkeleton = () => (
   <AspectRatio ratio={1.2} w="full">
-    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="gray.50">
+    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="white">
       <SkeletonText noOfLines={1} w="80%" />
       <SkeletonText noOfLines={2} w="60%" />
       <SkeletonText noOfLines={1} w="80%" />

app/src/components/experiments/ExperimentHeaderButtons/useOnForkButtonPressed.tsx

@@ -16,11 +16,14 @@ export const useOnForkButtonPressed = () => {
 
   const [onFork, isForking] = useHandledAsyncCallback(async () => {
     if (!experiment.data?.id || !selectedProjectId) return;
-    const forkedExperimentId = await forkMutation.mutateAsync({
+    const newExperiment = await forkMutation.mutateAsync({
       id: experiment.data.id,
       projectId: selectedProjectId,
     });
-    await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
+    await router.push({
+      pathname: "/experiments/[experimentSlug]",
+      query: { experimentSlug: newExperiment.slug },
+    });
   }, [forkMutation, experiment.data?.id, router]);
 
   const onForkButtonPressed = useCallback(() => {

app/src/components/fineTunes/FineTunesTable.tsx

@@ -0,0 +1,65 @@
+import { Card, Table, Thead, Tr, Th, Tbody, Td, VStack, Icon, Text } from "@chakra-ui/react";
+import { FaTable } from "react-icons/fa";
+import { type FineTuneStatus } from "@prisma/client";
+
+import dayjs from "~/utils/dayjs";
+import { useFineTunes } from "~/utils/hooks";
+
+const FineTunesTable = ({}) => {
+  const { data } = useFineTunes();
+
+  const fineTunes = data?.fineTunes || [];
+
+  return (
+    <Card width="100%" overflowX="auto">
+      {fineTunes.length ? (
+        <Table>
+          <Thead>
+            <Tr>
+              <Th>ID</Th>
+              <Th>Created At</Th>
+              <Th>Base Model</Th>
+              <Th>Dataset Size</Th>
+              <Th>Status</Th>
+            </Tr>
+          </Thead>
+          <Tbody>
+            {fineTunes.map((fineTune) => {
+              return (
+                <Tr key={fineTune.id}>
+                  <Td>{fineTune.slug}</Td>
+                  <Td>{dayjs(fineTune.createdAt).format("MMMM D h:mm A")}</Td>
+                  <Td>{fineTune.baseModel}</Td>
+                  <Td>{fineTune.dataset._count.datasetEntries}</Td>
+                  <Td fontSize="sm" fontWeight="bold">
+                    <Text color={getStatusColor(fineTune.status)}>{fineTune.status}</Text>
+                  </Td>
+                </Tr>
+              );
+            })}
+          </Tbody>
+        </Table>
+      ) : (
+        <VStack py={8}>
+          <Icon as={FaTable} boxSize={16} color="gray.300" />
+          <Text color="gray.400" fontSize="lg" fontWeight="bold">
+            No Fine Tunes Found
+          </Text>
+        </VStack>
+      )}
+    </Card>
+  );
+};
+
+export default FineTunesTable;
+
+const getStatusColor = (status: FineTuneStatus) => {
+  switch (status) {
+    case "DEPLOYED":
+      return "green.500";
+    case "ERROR":
+      return "red.500";
+    default:
+      return "yellow.500";
+  }
+};

app/src/components/nav/AppShell.tsx

@@ -1,4 +1,4 @@
-import { useState, useEffect } from "react";
+import { useState, useEffect, useRef } from "react";
 import {
   Heading,
   VStack,
@@ -9,28 +9,39 @@ import {
   Box,
   Link as ChakraLink,
   Flex,
+  useBreakpointValue,
 } from "@chakra-ui/react";
 import Head from "next/head";
 import Link from "next/link";
+import { useRouter } from "next/router";
 import { BsGearFill, BsGithub, BsPersonCircle } from "react-icons/bs";
 import { IoStatsChartOutline } from "react-icons/io5";
-import { RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
+import { RiHome3Line, RiFlaskLine } from "react-icons/ri";
+import { AiOutlineThunderbolt } from "react-icons/ai";
+import { FaReadme } from "react-icons/fa";
 import { signIn, useSession } from "next-auth/react";
-import UserMenu from "./UserMenu";
+
-import { env } from "~/env.mjs";
 import ProjectMenu from "./ProjectMenu";
 import NavSidebarOption from "./NavSidebarOption";
 import IconLink from "./IconLink";
+import { BetaModal } from "../BetaModal";
+import { useAppStore } from "~/state/store";
 
 const Divider = () => <Box h="1px" bgColor="gray.300" w="full" />;
 
 const NavSidebar = () => {
   const user = useSession().data;
 
+  // Hack to get around initial flash, see https://github.com/chakra-ui/chakra-ui/issues/6452
+  const isMobile = useBreakpointValue({ base: true, md: false, ssr: false });
+  const renderCount = useRef(0);
+  renderCount.current++;
+
+  const displayLogo = isMobile && renderCount.current > 1;
+
   return (
     <VStack
       align="stretch"
-      bgColor="gray.50"
       py={2}
       px={2}
       pb={0}
@@ -40,39 +51,55 @@ const NavSidebar = () => {
       borderRightWidth={1}
       borderColor="gray.300"
     >
-      <HStack
+      {displayLogo && (
-        as={Link}
+        <>
-        href="/"
+          <HStack
-        _hover={{ textDecoration: "none" }}
+            as={Link}
-        spacing={{ base: 1, md: 0 }}
+            href="/"
-        mx={2}
+            _hover={{ textDecoration: "none" }}
-        py={{ base: 1, md: 2 }}
+            spacing={{ base: 1, md: 0 }}
-      >
+            mx={2}
-        <Image src="/logo.svg" alt="" boxSize={6} mr={4} ml={{ base: 0.5, md: 0 }} />
+            py={{ base: 1, md: 2 }}
-        <Heading size="md" fontFamily="inconsolata, monospace">
+          >
-          OpenPipe
+            <Image src="/logo.svg" alt="" boxSize={6} mr={4} ml={{ base: 0.5, md: 0 }} />
-        </Heading>
+            <Heading size="md" fontFamily="inconsolata, monospace">
-      </HStack>
+              OpenPipe
-      <Divider />
+            </Heading>
+          </HStack>
+          <Divider />
+        </>
+      )}
+
       <VStack align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
         {user != null && (
           <>
             <ProjectMenu />
             <Divider />
-            {env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS && (
+
-              <IconLink icon={IoStatsChartOutline} label="Logged Calls" href="/logged-calls" beta />
+            <IconLink icon={RiHome3Line} label="Dashboard" href="/dashboard" />
-            )}
+            <IconLink icon={IoStatsChartOutline} label="Request Logs" href="/request-logs" />
+            <IconLink icon={AiOutlineThunderbolt} label="Fine Tunes" href="/fine-tunes" beta />
             <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
-            {env.NEXT_PUBLIC_SHOW_DATA && (
+            <VStack w="full" alignItems="flex-start" spacing={0} pt={8}>
-              <IconLink icon={RiDatabase2Line} label="Data" href="/data" />
+              <Text
-            )}
+                pl={2}
+                pb={2}
+                fontSize="xs"
+                fontWeight="bold"
+                color="gray.500"
+                display={{ base: "none", md: "flex" }}
+              >
+                CONFIGURATION
+              </Text>
+              <IconLink icon={BsGearFill} label="Project Settings" href="/project/settings" />
+            </VStack>
           </>
         )}
         {user === null && (
           <NavSidebarOption>
             <HStack
               w="full"
-              p={4}
+              p={{ base: 2, md: 4 }}
               as={ChakraLink}
               justifyContent="start"
               onClick={() => {
@@ -87,20 +114,22 @@ const NavSidebar = () => {
           </NavSidebarOption>
         )}
       </VStack>
-      <VStack w="full" alignItems="flex-start" spacing={0}>
+      <HStack
-        <Text
+        w="full"
-          pl={2}
+        px={{ base: 2, md: 4 }}
-          pb={2}
+        py={{ base: 1, md: 2 }}
-          fontSize="xs"
+        as={ChakraLink}
-          fontWeight="bold"
+        justifyContent="start"
-          color="gray.500"
+        href="https://docs.openpipe.ai"
-          display={{ base: "none", md: "flex" }}
+        target="_blank"
-        >
+        color="gray.500"
-          CONFIGURATION
+        spacing={1}
+      >
+        <Icon as={FaReadme} boxSize={4} mr={2} />
+        <Text fontWeight="bold" fontSize="sm">
+          Read the Docs
         </Text>
-        <IconLink icon={BsGearFill} label="Project Settings" href="/project/settings" />
+      </HStack>
-      </VStack>
-      {user && <UserMenu user={user} borderColor={"gray.200"} />}
       <Divider />
       <VStack spacing={0} align="center">
         <ChakraLink
@@ -121,12 +150,15 @@ export default function AppShell({
   children,
   title,
   requireAuth,
+  requireBeta,
 }: {
   children: React.ReactNode;
   title?: string;
   requireAuth?: boolean;
+  requireBeta?: boolean;
 }) {
   const [vh, setVh] = useState("100vh"); // Default height to prevent flicker on initial render
+  const router = useRouter();
 
   useEffect(() => {
     const setHeight = () => {
@@ -154,15 +186,21 @@ export default function AppShell({
     }
   }, [requireAuth, user, authLoading]);
 
+  const flags = useAppStore((s) => s.featureFlags.featureFlags);
+  const flagsLoaded = useAppStore((s) => s.featureFlags.flagsLoaded);
+
   return (
-    <Flex h={vh} w="100vw">
+    <>
-      <Head>
+      <Flex h={vh} w="100vw">
-        <title>{title ? `${title} | OpenPipe` : "OpenPipe"}</title>
+        <Head>
-      </Head>
+          <title>{title ? `${title} | OpenPipe` : "OpenPipe"}</title>
-      <NavSidebar />
+        </Head>
-      <Box h="100%" flex={1} overflowY="auto">
+        <NavSidebar />
-        {children}
+        <Box h="100%" flex={1} overflowY="auto" bgColor="gray.50">
-      </Box>
+          {children}
-    </Flex>
+        </Box>
+      </Flex>
+      <BetaModal isOpen={!!requireBeta && flagsLoaded && !flags.betaAccess} onClose={router.back} />
+    </>
   );
 }

app/src/components/nav/ProjectMenu.tsx

@@ -6,16 +6,19 @@ import {
   PopoverTrigger,
   PopoverContent,
   Flex,
-  IconButton,
   Icon,
   Divider,
   Button,
   useDisclosure,
   Spinner,
+  Link as ChakraLink,
+  Image,
+  Box,
+  Portal,
 } from "@chakra-ui/react";
-import React, { useEffect, useState } from "react";
+import { useEffect } from "react";
 import Link from "next/link";
-import { BsChevronRight, BsGear, BsPlus } from "react-icons/bs";
+import { BsPlus, BsPersonCircle } from "react-icons/bs";
 import { type Project } from "@prisma/client";
 
 import { useAppStore } from "~/state/store";
@@ -23,13 +26,14 @@ import { api } from "~/utils/api";
 import NavSidebarOption from "./NavSidebarOption";
 import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
 import { useRouter } from "next/router";
+import { useSession, signOut } from "next-auth/react";
 
 export default function ProjectMenu() {
   const router = useRouter();
   const utils = api.useContext();
 
   const selectedProjectId = useAppStore((s) => s.selectedProjectId);
-  const setselectedProjectId = useAppStore((s) => s.setselectedProjectId);
+  const setSelectedProjectId = useAppStore((s) => s.setSelectedProjectId);
 
   const { data: projects } = api.projects.list.useQuery();
 
@@ -39,9 +43,9 @@ export default function ProjectMenu() {
       projects[0] &&
       (!selectedProjectId || !projects.find((proj) => proj.id === selectedProjectId))
     ) {
-      setselectedProjectId(projects[0].id);
+      setSelectedProjectId(projects[0].id);
     }
-  }, [selectedProjectId, setselectedProjectId, projects]);
+  }, [selectedProjectId, setSelectedProjectId, projects]);
 
   const { data: selectedProject } = useSelectedProject();
 
@@ -49,28 +53,39 @@ export default function ProjectMenu() {
 
   const createMutation = api.projects.create.useMutation();
   const [createProject, isLoading] = useHandledAsyncCallback(async () => {
-    const newProj = await createMutation.mutateAsync({ name: "New Project" });
+    const newProj = await createMutation.mutateAsync({ name: "Untitled Project" });
     await utils.projects.list.invalidate();
-    setselectedProjectId(newProj.id);
+    setSelectedProjectId(newProj.id);
     await router.push({ pathname: "/project/settings" });
+    popover.onClose();
   }, [createMutation, router]);
 
+  const user = useSession().data;
+
+  const profileImage = user?.user.image ? (
+    <Image src={user.user.image} alt="profile picture" boxSize={6} borderRadius="50%" />
+  ) : (
+    <Icon as={BsPersonCircle} boxSize={6} />
+  );
+
   return (
-    <VStack w="full" alignItems="flex-start" spacing={0}>
+    <VStack
-      <Text
+      w="full"
-        pl={2}
+      alignItems="flex-start"
-        pb={2}
+      spacing={0}
-        fontSize="xs"
+      py={1}
-        fontWeight="bold"
+      zIndex={popover.isOpen ? "dropdown" : undefined}
-        color="gray.500"
+    >
-        display={{ base: "none", md: "flex" }}
+      <Popover
+        placement="bottom"
+        isOpen={popover.isOpen}
+        onOpen={popover.onOpen}
+        onClose={popover.onClose}
+        closeOnBlur
       >
-        PROJECT
-      </Text>
-      <Popover placement="right-end" isOpen={popover.isOpen} onClose={popover.onClose} closeOnBlur>
         <PopoverTrigger>
           <NavSidebarOption>
-            <HStack w="full" onClick={popover.onToggle}>
+            <HStack w="full">
               <Flex
                 p={1}
                 borderRadius={4}
@@ -83,43 +98,79 @@ export default function ProjectMenu() {
               >
                 <Text>{selectedProject?.name[0]?.toUpperCase()}</Text>
               </Flex>
-              <Text fontSize="sm" display={{ base: "none", md: "block" }} py={1} flex={1}>
+              <Text
+                fontSize="sm"
+                display={{ base: "none", md: "block" }}
+                py={1}
+                flex={1}
+                fontWeight="bold"
+              >
                 {selectedProject?.name}
               </Text>
-              <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+              <Box mr={2}>{profileImage}</Box>
             </HStack>
           </NavSidebarOption>
         </PopoverTrigger>
-        <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} w="auto" minW={100} maxW={280}>
+        <Portal>
-          <VStack alignItems="flex-start" spacing={2} py={4} px={2}>
+          <PopoverContent
-            <Text color="gray.500" fontSize="xs" fontWeight="bold" pb={1}>
+            _focusVisible={{ outline: "unset" }}
-              PROJECTS
+            w={220}
-            </Text>
+            ml={{ base: 2, md: 0 }}
-            <Divider />
+            boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
-            <VStack spacing={0} w="full">
+            fontSize="sm"
-              {projects?.map((proj) => (
+          >
-                <ProjectOption
+            <VStack alignItems="flex-start" spacing={1} py={1}>
-                  key={proj.id}
+              <Text px={3} py={2}>
-                  proj={proj}
+                {user?.user.email}
-                  isActive={proj.id === selectedProjectId}
+              </Text>
-                  onClose={popover.onClose}
+              <Divider />
-                />
+              <Text alignSelf="flex-start" fontWeight="bold" px={3} pt={2}>
-              ))}
+                Your Projects
+              </Text>
+              <VStack spacing={0} w="full" px={1}>
+                {projects?.map((proj) => (
+                  <ProjectOption
+                    key={proj.id}
+                    proj={proj}
+                    isActive={proj.id === selectedProjectId}
+                    onClose={popover.onClose}
+                  />
+                ))}
+                <HStack
+                  as={Button}
+                  variant="ghost"
+                  colorScheme="blue"
+                  color="blue.400"
+                  fontSize="sm"
+                  justifyContent="flex-start"
+                  onClick={createProject}
+                  w="full"
+                  borderRadius={4}
+                  spacing={0}
+                >
+                  <Text>Add project</Text>
+                  <Icon as={isLoading ? Spinner : BsPlus} boxSize={4} strokeWidth={0.5} />
+                </HStack>
+              </VStack>
+
+              <Divider />
+              <VStack w="full" px={1}>
+                <ChakraLink
+                  onClick={() => {
+                    signOut().catch(console.error);
+                  }}
+                  _hover={{ bgColor: "gray.200", textDecoration: "none" }}
+                  w="full"
+                  py={2}
+                  px={2}
+                  borderRadius={4}
+                >
+                  <Text>Sign out</Text>
+                </ChakraLink>
+              </VStack>
             </VStack>
-            <HStack
+          </PopoverContent>
-              as={Button}
+        </Portal>
-              variant="ghost"
-              colorScheme="blue"
-              color="blue.400"
-              pr={8}
-              w="full"
-              onClick={createProject}
-            >
-              <Icon as={isLoading ? Spinner : BsPlus} boxSize={6} />
-              <Text>New project</Text>
-            </HStack>
-          </VStack>
-        </PopoverContent>
       </Popover>
     </VStack>
   );
@@ -134,38 +185,26 @@ const ProjectOption = ({
   isActive: boolean;
   onClose: () => void;
 }) => {
-  const setselectedProjectId = useAppStore((s) => s.setselectedProjectId);
+  const setSelectedProjectId = useAppStore((s) => s.setSelectedProjectId);
-  const [gearHovered, setGearHovered] = useState(false);
+
   return (
     <HStack
       as={Link}
       href="/experiments"
       onClick={() => {
-        setselectedProjectId(proj.id);
+        setSelectedProjectId(proj.id);
         onClose();
       }}
       w="full"
       justifyContent="space-between"
-      bgColor={isActive ? "gray.100" : "transparent"}
+      _hover={{ bgColor: "gray.200", textDecoration: "none" }}
-      _hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
+      bgColor={isActive ? "gray.100" : undefined}
-      p={2}
+      py={2}
+      px={4}
       borderRadius={4}
       spacing={4}
     >
       <Text>{proj.name}</Text>
-      <IconButton
-        as={Link}
-        href="/project/settings"
-        aria-label={`Open ${proj.name} settings`}
-        icon={<Icon as={BsGear} boxSize={5} strokeWidth={0.5} color="gray.500" />}
-        variant="ghost"
-        size="xs"
-        p={0}
-        onMouseEnter={() => setGearHovered(true)}
-        onMouseLeave={() => setGearHovered(false)}
-        _hover={{ bgColor: isActive ? "gray.300" : "gray.100", transitionDelay: 0 }}
-        borderRadius={4}
-      />
     </HStack>
   );
 };

app/src/components/nav/UserMenu.tsx

@@ -23,50 +23,48 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
   );
 
   return (
-    <>
+    <Popover placement="right">
-      <Popover placement="right">
+      <PopoverTrigger>
-        <PopoverTrigger>
+        <NavSidebarOption>
-          <NavSidebarOption>
+          <HStack
-            <HStack
+            // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
-              // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
+            py={2}
-              py={2}
+            px={1}
-              px={1}
+            spacing={3}
-              spacing={3}
+            {...rest}
-              {...rest}
+          >
-            >
+            {profileImage}
-              {profileImage}
+            <VStack spacing={0} align="start" flex={1} flexShrink={1}>
-              <VStack spacing={0} align="start" flex={1} flexShrink={1}>
+              <Text fontWeight="bold" fontSize="sm">
-                <Text fontWeight="bold" fontSize="sm">
+                {user.user.name}
-                  {user.user.name}
+              </Text>
-                </Text>
+              <Text color="gray.500" fontSize="xs">
-                <Text color="gray.500" fontSize="xs">
+                {/* {user.user.email} */}
-                  {/* {user.user.email} */}
+              </Text>
-                </Text>
+            </VStack>
-              </VStack>
+            <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
-              <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+          </HStack>
-            </HStack>
+        </NavSidebarOption>
-          </NavSidebarOption>
+      </PopoverTrigger>
-        </PopoverTrigger>
+      <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} minW={48} w="full">
-        <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} minW={48} w="full">
+        <VStack align="stretch" spacing={0}>
-          <VStack align="stretch" spacing={0}>
+          {/* sign out */}
-            {/* sign out */}
+          <HStack
-            <HStack
+            as={Link}
-              as={Link}
+            onClick={() => {
-              onClick={() => {
+              signOut().catch(console.error);
-                signOut().catch(console.error);
+            }}
-              }}
+            px={4}
-              px={4}
+            py={2}
-              py={2}
+            spacing={4}
-              spacing={4}
+            color="gray.500"
-              color="gray.500"
+            fontSize="sm"
-              fontSize="sm"
+          >
-            >
+            <Icon as={BsBoxArrowRight} boxSize={6} />
-              <Icon as={BsBoxArrowRight} boxSize={6} />
+            <Text>Sign out</Text>
-              <Text>Sign out</Text>
+          </HStack>
-            </HStack>
+        </VStack>
-          </VStack>
+      </PopoverContent>
-        </PopoverContent>
+    </Popover>
-      </Popover>
-    </>
   );
 }

app/src/components/projectSettings/InviteMemberModal.tsx

@@ -0,0 +1,128 @@
+import {
+  Button,
+  FormControl,
+  FormLabel,
+  Input,
+  FormHelperText,
+  HStack,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  Spinner,
+  Text,
+  VStack,
+  RadioGroup,
+  Radio,
+} from "@chakra-ui/react";
+import { useState, useEffect } from "react";
+
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { type ProjectUserRole } from "@prisma/client";
+
+export const InviteMemberModal = ({
+  isOpen,
+  onClose,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+}) => {
+  const selectedProject = useSelectedProject().data;
+  const utils = api.useContext();
+
+  const [email, setEmail] = useState("");
+  const [role, setRole] = useState<ProjectUserRole>("MEMBER");
+
+  useEffect(() => {
+    setEmail("");
+    setRole("MEMBER");
+  }, [isOpen]);
+
+  const emailIsValid = !email || !email.match(/.+@.+\..+/);
+
+  const inviteMemberMutation = api.users.inviteToProject.useMutation();
+
+  const [inviteMember, isInviting] = useHandledAsyncCallback(async () => {
+    if (!selectedProject?.id || !role) return;
+    const resp = await inviteMemberMutation.mutateAsync({
+      projectId: selectedProject.id,
+      email,
+      role,
+    });
+    if (maybeReportError(resp)) return;
+    await utils.projects.get.invalidate();
+    onClose();
+  }, [inviteMemberMutation, email, role, selectedProject?.id, onClose]);
+
+  return (
+    <Modal isOpen={isOpen} onClose={onClose}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Text>Invite Member</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          <VStack spacing={8} alignItems="flex-start">
+            <Text>
+              Invite a new member to <b>{selectedProject?.name}</b>.
+            </Text>
+
+            <RadioGroup
+              value={role}
+              onChange={(e) => setRole(e as ProjectUserRole)}
+              colorScheme="orange"
+            >
+              <VStack w="full" alignItems="flex-start">
+                <Radio value="MEMBER">
+                  <Text fontSize="sm">MEMBER</Text>
+                </Radio>
+                <Radio value="ADMIN">
+                  <Text fontSize="sm">ADMIN</Text>
+                </Radio>
+              </VStack>
+            </RadioGroup>
+            <FormControl>
+              <FormLabel>Email</FormLabel>
+              <Input
+                type="email"
+                value={email}
+                onChange={(e) => setEmail(e.target.value)}
+                onKeyDown={(e) => {
+                  if (e.key === "Enter" && (e.metaKey || e.ctrlKey || e.shiftKey)) {
+                    e.preventDefault();
+                    e.currentTarget.blur();
+                    inviteMember();
+                  }
+                }}
+              />
+              <FormHelperText>Enter the email of the person you want to invite.</FormHelperText>
+            </FormControl>
+          </VStack>
+        </ModalBody>
+        <ModalFooter mt={4}>
+          <HStack>
+            <Button colorScheme="gray" onClick={onClose} minW={24}>
+              <Text>Cancel</Text>
+            </Button>
+            <Button
+              colorScheme="orange"
+              onClick={inviteMember}
+              minW={24}
+              isDisabled={emailIsValid || isInviting}
+            >
+              {isInviting ? <Spinner boxSize={4} /> : <Text>Send Invitation</Text>}
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/projectSettings/MemberTable.tsx

@@ -0,0 +1,145 @@
+import { useMemo, useState } from "react";
+import {
+  Table,
+  Thead,
+  Tr,
+  Th,
+  Tbody,
+  Td,
+  IconButton,
+  useDisclosure,
+  Text,
+  Button,
+} from "@chakra-ui/react";
+import { useSession } from "next-auth/react";
+import { BsTrash } from "react-icons/bs";
+import { type User } from "@prisma/client";
+
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+import { InviteMemberModal } from "./InviteMemberModal";
+import { RemoveMemberDialog } from "./RemoveMemberDialog";
+import { api } from "~/utils/api";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+
+const MemberTable = () => {
+  const selectedProject = useSelectedProject().data;
+  const session = useSession().data;
+
+  const utils = api.useContext();
+
+  const [memberToRemove, setMemberToRemove] = useState<User | null>(null);
+  const inviteMemberModal = useDisclosure();
+
+  const cancelInvitationMutation = api.users.cancelProjectInvitation.useMutation();
+
+  const [cancelInvitation, isCancelling] = useHandledAsyncCallback(
+    async (invitationToken: string) => {
+      if (!selectedProject?.id) return;
+      const resp = await cancelInvitationMutation.mutateAsync({
+        invitationToken,
+      });
+      if (maybeReportError(resp)) return;
+      await utils.projects.get.invalidate();
+    },
+    [selectedProject?.id, cancelInvitationMutation],
+  );
+
+  const sortedMembers = useMemo(() => {
+    if (!selectedProject?.projectUsers) return [];
+    return selectedProject.projectUsers.sort((a, b) => {
+      if (a.role === b.role) return a.createdAt < b.createdAt ? -1 : 1;
+      // Take advantage of fact that ADMIN is alphabetically before MEMBER
+      return a.role < b.role ? -1 : 1;
+    });
+  }, [selectedProject?.projectUsers]);
+
+  return (
+    <>
+      <Table fontSize={{ base: "sm", md: "md" }}>
+        <Thead
+          sx={{
+            th: {
+              base: { px: 0 },
+              md: { px: 6 },
+            },
+          }}
+        >
+          <Tr>
+            <Th>Name</Th>
+            <Th display={{ base: "none", md: "table-cell" }}>Email</Th>
+            <Th>Role</Th>
+            {selectedProject?.role === "ADMIN" && <Th />}
+          </Tr>
+        </Thead>
+        <Tbody
+          sx={{
+            td: {
+              base: { px: 0 },
+              md: { px: 6 },
+            },
+          }}
+        >
+          {selectedProject &&
+            sortedMembers.map((member) => {
+              return (
+                <Tr key={member.id}>
+                  <Td>
+                    <Text fontWeight="bold">{member.user.name}</Text>
+                  </Td>
+                  <Td display={{ base: "none", md: "table-cell" }} h="full">
+                    {member.user.email}
+                  </Td>
+                  <Td fontSize={{ base: "xs", md: "sm" }}>{member.role}</Td>
+                  {selectedProject.role === "ADMIN" && (
+                    <Td textAlign="end">
+                      {member.user.id !== session?.user?.id &&
+                        member.user.id !== selectedProject.personalProjectUserId && (
+                          <IconButton
+                            aria-label="Remove member"
+                            colorScheme="red"
+                            icon={<BsTrash />}
+                            onClick={() => setMemberToRemove(member.user)}
+                          />
+                        )}
+                    </Td>
+                  )}
+                </Tr>
+              );
+            })}
+          {selectedProject?.projectUserInvitations?.map((invitation) => {
+            return (
+              <Tr key={invitation.id}>
+                <Td>
+                  <Text as="i">Invitation pending</Text>
+                </Td>
+                <Td>{invitation.email}</Td>
+                <Td fontSize="sm">{invitation.role}</Td>
+                {selectedProject.role === "ADMIN" && (
+                  <Td textAlign="end">
+                    <Button
+                      size="sm"
+                      colorScheme="red"
+                      variant="ghost"
+                      onClick={() => cancelInvitation(invitation.invitationToken)}
+                      isLoading={isCancelling}
+                    >
+                      Cancel
+                    </Button>
+                  </Td>
+                )}
+              </Tr>
+            );
+          })}
+        </Tbody>
+      </Table>
+      <InviteMemberModal isOpen={inviteMemberModal.isOpen} onClose={inviteMemberModal.onClose} />
+      <RemoveMemberDialog
+        member={memberToRemove}
+        isOpen={!!memberToRemove}
+        onClose={() => setMemberToRemove(null)}
+      />
+    </>
+  );
+};
+
+export default MemberTable;

app/src/components/projectSettings/RemoveMemberDialog.tsx

@@ -0,0 +1,71 @@
+import {
+  Button,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+  Text,
+  VStack,
+  Spinner,
+} from "@chakra-ui/react";
+import { type User } from "@prisma/client";
+
+import { useRouter } from "next/router";
+import { useRef } from "react";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+
+export const RemoveMemberDialog = ({
+  isOpen,
+  onClose,
+  member,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+  member: User | null;
+}) => {
+  const selectedProject = useSelectedProject();
+  const removeUserMutation = api.users.removeUserFromProject.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onRemoveConfirm, isRemoving] = useHandledAsyncCallback(async () => {
+    if (!selectedProject.data?.id || !member?.id) return;
+    await removeUserMutation.mutateAsync({ projectId: selectedProject.data.id, userId: member.id });
+    await utils.projects.get.invalidate();
+    onClose();
+  }, [removeUserMutation, selectedProject, router]);
+
+  return (
+    <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Remove Member
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            <VStack spacing={4} alignItems="flex-start">
+              <Text>
+                Are you sure you want to remove <b>{member?.name}</b> from the project?
+              </Text>
+            </VStack>
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={onClose}>
+              Cancel
+            </Button>
+            <Button colorScheme="red" onClick={onRemoveConfirm} ml={3} w={20}>
+              {isRemoving ? <Spinner /> : "Remove"}
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};

app/src/components/requestLogs/ActionButton.tsx

@@ -0,0 +1,51 @@
+import { useState } from "react";
+
+import { Button, HStack, type ButtonProps, Icon, Text } from "@chakra-ui/react";
+import { type IconType } from "react-icons";
+import { useAppStore } from "~/state/store";
+import { BetaModal } from "../BetaModal";
+
+const ActionButton = ({
+  icon,
+  label,
+  requireBeta = false,
+  onClick,
+  ...buttonProps
+}: {
+  icon: IconType;
+  label: string;
+  requireBeta?: boolean;
+  onClick?: () => void;
+} & ButtonProps) => {
+  const flags = useAppStore((s) => s.featureFlags.featureFlags);
+  const flagsLoaded = useAppStore((s) => s.featureFlags.flagsLoaded);
+
+  const [betaModalOpen, setBetaModalOpen] = useState(false);
+
+  const isBetaBlocked = requireBeta && flagsLoaded && !flags.betaAccess;
+  return (
+    <>
+      <Button
+        colorScheme="blue"
+        color="black"
+        bgColor="white"
+        borderColor="gray.300"
+        borderRadius={4}
+        variant="outline"
+        size="sm"
+        fontSize="sm"
+        fontWeight="normal"
+        onClick={isBetaBlocked ? () => setBetaModalOpen(true) : onClick}
+        {...buttonProps}
+      >
+        <HStack spacing={1}>
+          {icon && <Icon as={icon} color={requireBeta ? "orange.400" : undefined} />}
+          <Text display={{ base: "none", md: "flex" }}>{label}</Text>
+        </HStack>
+      </Button>
+      <BetaModal isOpen={betaModalOpen} onClose={() => setBetaModalOpen(false)} />
+    </>
+  );
+};
+
+export default ActionButton;

app/src/components/requestLogs/ColumnVisiblityDropdown.tsx

@@ -0,0 +1,117 @@
+import {
+  Icon,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  VStack,
+  HStack,
+  Button,
+  Text,
+  useDisclosure,
+  Box,
+} from "@chakra-ui/react";
+import { BiCheck } from "react-icons/bi";
+import { BsToggles } from "react-icons/bs";
+import { useMemo } from "react";
+
+import { useIsClientRehydrated, useTagNames } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
+import { StaticColumnKeys } from "~/state/columnVisiblitySlice";
+import ActionButton from "./ActionButton";
+
+const ColumnVisiblityDropdown = () => {
+  const tagNames = useTagNames().data;
+
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const toggleColumnVisibility = useAppStore((s) => s.columnVisibility.toggleColumnVisibility);
+  const totalColumns = Object.keys(StaticColumnKeys).length + (tagNames?.length ?? 0);
+
+  const popover = useDisclosure();
+
+  const columnVisiblityOptions = useMemo(() => {
+    const options: { label: string; key: string }[] = [
+      {
+        label: "Sent At",
+        key: StaticColumnKeys.SENT_AT,
+      },
+      {
+        label: "Model",
+        key: StaticColumnKeys.MODEL,
+      },
+      {
+        label: "Duration",
+        key: StaticColumnKeys.DURATION,
+      },
+      {
+        label: "Input Tokens",
+        key: StaticColumnKeys.INPUT_TOKENS,
+      },
+      {
+        label: "Output Tokens",
+        key: StaticColumnKeys.OUTPUT_TOKENS,
+      },
+      {
+        label: "Status Code",
+        key: StaticColumnKeys.STATUS_CODE,
+      },
+    ];
+    for (const tagName of tagNames ?? []) {
+      options.push({
+        label: tagName,
+        key: tagName,
+      });
+    }
+    return options;
+  }, [tagNames]);
+
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
+  return (
+    <Popover
+      placement="bottom-start"
+      isOpen={popover.isOpen}
+      onOpen={popover.onOpen}
+      onClose={popover.onClose}
+    >
+      <PopoverTrigger>
+        <Box>
+          <ActionButton
+            label={`Columns (${visibleColumns.size}/${totalColumns})`}
+            icon={BsToggles}
+          />
+        </Box>
+      </PopoverTrigger>
+      <PopoverContent boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);" minW={0} w="auto">
+        <VStack spacing={0} maxH={400} overflowY="auto">
+          {columnVisiblityOptions?.map((option, index) => (
+            <HStack
+              key={index}
+              as={Button}
+              onClick={() => toggleColumnVisibility(option.key)}
+              w="full"
+              minH={10}
+              variant="ghost"
+              justifyContent="space-between"
+              fontWeight="semibold"
+              borderRadius={0}
+              colorScheme="blue"
+              color="black"
+              fontSize="sm"
+              borderBottomWidth={1}
+            >
+              <Text mr={16}>{option.label}</Text>
+              <Box w={5}>
+                {visibleColumns.has(option.key) && (
+                  <Icon as={BiCheck} color="blue.500" boxSize={5} />
+                )}
+              </Box>
+            </HStack>
+          ))}
+        </VStack>
+      </PopoverContent>
+    </Popover>
+  );
+};
+
+export default ColumnVisiblityDropdown;

app/src/components/requestLogs/ExportButton.tsx

@@ -0,0 +1,211 @@
+import { useState, useEffect } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  Checkbox,
+  NumberInput,
+  NumberInputField,
+  NumberInputStepper,
+  NumberIncrementStepper,
+  NumberDecrementStepper,
+  Collapse,
+  Flex,
+  useDisclosure,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import { BiExport } from "react-icons/bi";
+
+import { useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "./ActionButton";
+import InputDropdown from "../InputDropdown";
+import { FiChevronUp, FiChevronDown } from "react-icons/fi";
+import InfoCircle from "../InfoCircle";
+
+const SUPPORTED_EXPORT_FORMATS = ["alpaca-finetune", "openai-fine-tune", "unformatted"];
+
+const ExportButton = () => {
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Export"
+        icon={BiExport}
+        isDisabled={selectedLogIds.size === 0}
+        requireBeta
+      />
+      <ExportLogsModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default ExportButton;
+
+const ExportLogsModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const clearSelectedLogIds = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
+
+  const [selectedExportFormat, setSelectedExportFormat] = useState(SUPPORTED_EXPORT_FORMATS[0]);
+  const [testingSplit, setTestingSplit] = useState(10);
+  const [removeDuplicates, setRemoveDuplicates] = useState(true);
+  const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      setSelectedExportFormat(SUPPORTED_EXPORT_FORMATS[0]);
+      setTestingSplit(10);
+      setRemoveDuplicates(true);
+    }
+  }, [disclosure.isOpen]);
+
+  const exportLogsMutation = api.loggedCalls.export.useMutation();
+
+  const [exportLogs, exportInProgress] = useHandledAsyncCallback(async () => {
+    if (!selectedProjectId || !selectedLogIds.size || !testingSplit || !selectedExportFormat)
+      return;
+    const response = await exportLogsMutation.mutateAsync({
+      projectId: selectedProjectId,
+      selectedLogIds: Array.from(selectedLogIds),
+      testingSplit,
+      selectedExportFormat,
+      removeDuplicates,
+    });
+
+    const dataUrl = `data:application/pdf;base64,${response}`;
+    const blob = await fetch(dataUrl).then((res) => res.blob());
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement("a");
+
+    a.href = url;
+    a.download = `data.zip`;
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+
+    disclosure.onClose();
+    clearSelectedLogIds();
+  }, [
+    exportLogsMutation,
+    selectedProjectId,
+    selectedLogIds,
+    testingSplit,
+    selectedExportFormat,
+    removeDuplicates,
+  ]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={BiExport} />
+            <Text>Export Logs</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              We'll export the <b>{selectedLogIds.size}</b> logs you have selected in the format of
+              your choice.
+            </Text>
+            <VStack alignItems="flex-start" spacing={4}>
+              <Flex
+                flexDir={{ base: "column", md: "row" }}
+                alignItems={{ base: "flex-start", md: "center" }}
+              >
+                <HStack w={48} alignItems="center" spacing={1}>
+                  <Text fontWeight="bold">Format:</Text>
+                  <InfoCircle tooltipText="Format logs for for fine tuning or export them without formatting." />
+                </HStack>
+                <InputDropdown
+                  options={SUPPORTED_EXPORT_FORMATS}
+                  selectedOption={selectedExportFormat}
+                  onSelect={(option) => setSelectedExportFormat(option)}
+                  inputGroupProps={{ w: 48 }}
+                />
+              </Flex>
+              <Flex
+                flexDir={{ base: "column", md: "row" }}
+                alignItems={{ base: "flex-start", md: "center" }}
+              >
+                <HStack w={48} alignItems="center" spacing={1}>
+                  <Text fontWeight="bold">Testing Split:</Text>
+                  <InfoCircle tooltipText="The percent of your logs that will be reserved for testing and saved in another file. Logs are split randomly." />
+                </HStack>
+                <HStack>
+                  <NumberInput
+                    defaultValue={10}
+                    onChange={(_, num) => setTestingSplit(num)}
+                    min={1}
+                    max={100}
+                    w={48}
+                  >
+                    <NumberInputField />
+                    <NumberInputStepper>
+                      <NumberIncrementStepper />
+                      <NumberDecrementStepper />
+                    </NumberInputStepper>
+                  </NumberInput>
+                </HStack>
+              </Flex>
+            </VStack>
+            <VStack alignItems="flex-start" spacing={0}>
+              <Button
+                variant="unstyled"
+                color="blue.600"
+                onClick={() => setShowAdvancedOptions(!showAdvancedOptions)}
+              >
+                <HStack>
+                  <Text>Advanced Options</Text>
+                  <Icon as={showAdvancedOptions ? FiChevronUp : FiChevronDown} />
+                </HStack>
+              </Button>
+              <Collapse in={showAdvancedOptions} unmountOnExit={true}>
+                <VStack align="stretch" pt={4}>
+                  <HStack>
+                    <Checkbox
+                      colorScheme="blue"
+                      isChecked={removeDuplicates}
+                      onChange={(e) => setRemoveDuplicates(e.target.checked)}
+                    >
+                      <Text>Remove duplicates</Text>
+                    </Checkbox>
+                    <InfoCircle tooltipText="To avoid overfitting and speed up training, automatically deduplicate logs with matching input and output." />
+                  </HStack>
+                </VStack>
+              </Collapse>
+            </VStack>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button colorScheme="blue" onClick={exportLogs} isLoading={exportInProgress} minW={24}>
+              Export
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/requestLogs/FineTuneButton.tsx

@@ -0,0 +1,162 @@
+import { useState, useEffect } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  useDisclosure,
+  type UseDisclosureReturn,
+  Input,
+} from "@chakra-ui/react";
+import { AiTwotoneThunderbolt } from "react-icons/ai";
+import humanId from "human-id";
+import { useRouter } from "next/router";
+
+import { useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "./ActionButton";
+import InputDropdown from "../InputDropdown";
+import { FiChevronDown } from "react-icons/fi";
+
+const SUPPORTED_BASE_MODELS = ["llama2-7b", "llama2-13b", "llama2-70b", "gpt-3.5-turbo"];
+
+const FineTuneButton = () => {
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Fine Tune"
+        icon={AiTwotoneThunderbolt}
+        isDisabled={selectedLogIds.size === 0}
+        requireBeta
+      />
+      <FineTuneModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default FineTuneButton;
+
+const FineTuneModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const clearSelectedLogIds = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
+
+  const [selectedBaseModel, setSelectedBaseModel] = useState(SUPPORTED_BASE_MODELS[0]);
+  const [modelSlug, setModelSlug] = useState(humanId({ separator: "-", capitalize: false }));
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      setSelectedBaseModel(SUPPORTED_BASE_MODELS[0]);
+      setModelSlug(humanId({ separator: "-", capitalize: false }));
+    }
+  }, [disclosure.isOpen]);
+
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const createFineTuneMutation = api.fineTunes.create.useMutation();
+
+  const [createFineTune, creationInProgress] = useHandledAsyncCallback(async () => {
+    if (!selectedProjectId || !modelSlug || !selectedBaseModel || !selectedLogIds.size) return;
+    await createFineTuneMutation.mutateAsync({
+      projectId: selectedProjectId,
+      slug: modelSlug,
+      baseModel: selectedBaseModel,
+      selectedLogIds: Array.from(selectedLogIds),
+    });
+
+    await utils.fineTunes.list.invalidate();
+    await router.push({ pathname: "/fine-tunes" });
+    clearSelectedLogIds();
+    disclosure.onClose();
+  }, [createFineTuneMutation, selectedProjectId, selectedLogIds, modelSlug, selectedBaseModel]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={AiTwotoneThunderbolt} />
+            <Text>Fine Tune</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              We'll train on the <b>{selectedLogIds.size}</b> logs you've selected.
+            </Text>
+            <VStack>
+              <HStack spacing={2} w="full">
+                <Text fontWeight="bold" w={36}>
+                  Model ID:
+                </Text>
+                <Input
+                  value={modelSlug}
+                  onChange={(e) => setModelSlug(e.target.value)}
+                  w={48}
+                  placeholder="unique-id"
+                  onKeyDown={(e) => {
+                    // If the user types anything other than a-z, A-Z, or 0-9, replace it with -
+                    if (!/[a-zA-Z0-9]/.test(e.key)) {
+                      e.preventDefault();
+                      setModelSlug((s) => s && `${s}-`);
+                    }
+                  }}
+                />
+              </HStack>
+              <HStack spacing={2}>
+                <Text fontWeight="bold" w={36}>
+                  Base model:
+                </Text>
+                <InputDropdown
+                  options={SUPPORTED_BASE_MODELS}
+                  selectedOption={selectedBaseModel}
+                  onSelect={(option) => setSelectedBaseModel(option)}
+                  inputGroupProps={{ w: 48 }}
+                />
+              </HStack>
+            </VStack>
+            <Button variant="unstyled" color="blue.600">
+              <HStack>
+                <Text>Advanced Options</Text>
+                <Icon as={FiChevronDown} />
+              </HStack>
+            </Button>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="blue"
+              onClick={createFineTune}
+              isLoading={creationInProgress}
+              minW={24}
+              isDisabled={!modelSlug}
+            >
+              Start Training
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/requestLogs/FormattedJson.tsx

@@ -0,0 +1,55 @@
+import { Box, IconButton, useToast } from "@chakra-ui/react";
+import { CopyIcon } from "lucide-react";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import { atelierCaveLight } from "react-syntax-highlighter/dist/cjs/styles/hljs";
+import stringify from "json-stringify-pretty-compact";
+
+const FormattedJson = ({ json }: { json: any }) => {
+  const jsonString = stringify(json, { maxLength: 40 });
+  const toast = useToast();
+
+  const copyToClipboard = async (text: string) => {
+    try {
+      await navigator.clipboard.writeText(text);
+      toast({
+        title: "Copied to clipboard",
+        status: "success",
+        duration: 2000,
+      });
+    } catch (err) {
+      toast({
+        title: "Failed to copy to clipboard",
+        status: "error",
+        duration: 2000,
+      });
+    }
+  };
+
+  return (
+    <Box position="relative" fontSize="sm" borderRadius="md" overflow="hidden">
+      <SyntaxHighlighter
+        customStyle={{ overflowX: "unset" }}
+        language="json"
+        style={atelierCaveLight}
+        lineProps={{
+          style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
+        }}
+        wrapLines
+      >
+        {jsonString}
+      </SyntaxHighlighter>
+      <IconButton
+        aria-label="Copy"
+        icon={<CopyIcon />}
+        position="absolute"
+        top={1}
+        right={1}
+        size="xs"
+        variant="ghost"
+        onClick={() => void copyToClipboard(jsonString)}
+      />
+    </Box>
+  );
+};
+
+export { FormattedJson };

app/src/components/requestLogs/LogFilters/AddFilterButton.tsx

@@ -0,0 +1,30 @@
+import { Button, HStack, Icon, Text } from "@chakra-ui/react";
+import { BsPlus } from "react-icons/bs";
+import { comparators, defaultFilterableFields } from "~/state/logFiltersSlice";
+import { useAppStore } from "~/state/store";
+
+const AddFilterButton = () => {
+  const addFilter = useAppStore((s) => s.logFilters.addFilter);
+
+  return (
+    <HStack
+      as={Button}
+      variant="ghost"
+      onClick={() =>
+        addFilter({
+          id: Date.now().toString(),
+          field: defaultFilterableFields[0],
+          comparator: comparators[0],
+          value: "",
+        })
+      }
+      spacing={0}
+      fontSize="sm"
+    >
+      <Icon as={BsPlus} boxSize={5} />
+      <Text>Add Filter</Text>
+    </HStack>
+  );
+};
+
+export default AddFilterButton;

app/src/components/requestLogs/LogFilters/LogFilter.tsx

@@ -0,0 +1,44 @@
+import { useCallback, useState } from "react";
+import { HStack, IconButton, Input } from "@chakra-ui/react";
+import { BsTrash } from "react-icons/bs";
+
+import { type LogFilter } from "~/state/logFiltersSlice";
+import { useAppStore } from "~/state/store";
+import { debounce } from "lodash-es";
+import SelectFieldDropdown from "./SelectFieldDropdown";
+import SelectComparatorDropdown from "./SelectComparatorDropdown";
+
+const LogFilter = ({ filter }: { filter: LogFilter }) => {
+  const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
+  const deleteFilter = useAppStore((s) => s.logFilters.deleteFilter);
+
+  const [editedValue, setEditedValue] = useState(filter.value);
+
+  const debouncedUpdateFilter = useCallback(
+    debounce((filter: LogFilter) => updateFilter(filter), 500, {
+      leading: true,
+    }),
+    [updateFilter],
+  );
+
+  return (
+    <HStack>
+      <SelectFieldDropdown filter={filter} />
+      <SelectComparatorDropdown filter={filter} />
+      <Input
+        value={editedValue}
+        onChange={(e) => {
+          setEditedValue(e.target.value);
+          debouncedUpdateFilter({ ...filter, value: e.target.value });
+        }}
+      />
+      <IconButton
+        aria-label="Delete Filter"
+        icon={<BsTrash />}
+        onClick={() => deleteFilter(filter.id)}
+      />
+    </HStack>
+  );
+};
+
+export default LogFilter;

app/src/components/requestLogs/LogFilters/LogFilters.tsx

@@ -0,0 +1,30 @@
+import { VStack, Text } from "@chakra-ui/react";
+
+import AddFilterButton from "./AddFilterButton";
+import { useAppStore } from "~/state/store";
+import LogFilter from "./LogFilter";
+
+const LogFilters = () => {
+  const filters = useAppStore((s) => s.logFilters.filters);
+  return (
+    <VStack
+      bgColor="white"
+      borderRadius={8}
+      borderWidth={1}
+      w="full"
+      alignItems="flex-start"
+      p={4}
+      spacing={4}
+    >
+      <Text fontWeight="bold" color="gray.500">
+        Filters
+      </Text>
+      {filters.map((filter) => (
+        <LogFilter key={filter.id} filter={filter} />
+      ))}
+      <AddFilterButton />
+    </VStack>
+  );
+};
+
+export default LogFilters;

app/src/components/requestLogs/LogFilters/SelectComparatorDropdown.tsx

@@ -0,0 +1,19 @@
+import { comparators, type LogFilter } from "~/state/logFiltersSlice";
+import { useAppStore } from "~/state/store";
+import InputDropdown from "~/components/InputDropdown";
+
+const SelectComparatorDropdown = ({ filter }: { filter: LogFilter }) => {
+  const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
+
+  const { comparator } = filter;
+
+  return (
+    <InputDropdown
+      options={comparators}
+      selectedOption={comparator}
+      onSelect={(option) => updateFilter({ ...filter, comparator: option })}
+    />
+  );
+};
+
+export default SelectComparatorDropdown;

app/src/components/requestLogs/LogFilters/SelectFieldDropdown.tsx

@@ -0,0 +1,22 @@
+import { defaultFilterableFields, type LogFilter } from "~/state/logFiltersSlice";
+import { useAppStore } from "~/state/store";
+import { useTagNames } from "~/utils/hooks";
+import InputDropdown from "~/components/InputDropdown";
+
+const SelectFieldDropdown = ({ filter }: { filter: LogFilter }) => {
+  const tagNames = useTagNames().data;
+
+  const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
+
+  const { field } = filter;
+
+  return (
+    <InputDropdown
+      options={[...defaultFilterableFields, ...(tagNames || [])]}
+      selectedOption={field}
+      onSelect={(option) => updateFilter({ ...filter, field: option })}
+    />
+  );
+};
+
+export default SelectFieldDropdown;

app/src/components/requestLogs/LoggedCallsPaginator.tsx

@@ -0,0 +1,16 @@
+import { type StackProps } from "@chakra-ui/react";
+
+import { useLoggedCalls } from "~/utils/hooks";
+import Paginator from "../Paginator";
+
+const LoggedCallsPaginator = (props: StackProps) => {
+  const { data } = useLoggedCalls();
+
+  if (!data) return null;
+
+  const { count } = data;
+
+  return <Paginator count={count} {...props} />;
+};
+
+export default LoggedCallsPaginator;

app/src/components/requestLogs/LoggedCallsTable.tsx

@@ -0,0 +1,40 @@
+import { Card, Table, Tbody } from "@chakra-ui/react";
+import { useState } from "react";
+import { useLoggedCalls } from "~/utils/hooks";
+import { TableHeader, TableRow, EmptyTableRow } from "./TableRow";
+
+export default function LoggedCallsTable() {
+  const [expandedRow, setExpandedRow] = useState<string | null>(null);
+  const loggedCalls = useLoggedCalls().data;
+
+  return (
+    <Card width="100%" overflowX="auto">
+      <Table>
+        <TableHeader showOptions />
+        <Tbody>
+          {loggedCalls?.calls.length ? (
+            loggedCalls?.calls?.map((loggedCall) => {
+              return (
+                <TableRow
+                  key={loggedCall.id}
+                  loggedCall={loggedCall}
+                  isExpanded={loggedCall.id === expandedRow}
+                  onToggle={() => {
+                    if (loggedCall.id === expandedRow) {
+                      setExpandedRow(null);
+                    } else {
+                      setExpandedRow(loggedCall.id);
+                    }
+                  }}
+                  showOptions
+                />
+              );
+            })
+          ) : (
+            <EmptyTableRow />
+          )}
+        </Tbody>
+      </Table>
+    </Card>
+  );
+}

app/src/components/requestLogs/TableRow.tsx

@@ -0,0 +1,239 @@
+import {
+  Box,
+  Heading,
+  Td,
+  Tr,
+  Thead,
+  Th,
+  Tooltip,
+  Collapse,
+  HStack,
+  VStack,
+  Button,
+  ButtonGroup,
+  Text,
+  Checkbox,
+  Link as ChakraLink,
+} from "@chakra-ui/react";
+import Link from "next/link";
+
+import dayjs from "~/utils/dayjs";
+import { type RouterOutputs } from "~/utils/api";
+import { FormattedJson } from "./FormattedJson";
+import { useAppStore } from "~/state/store";
+import { useIsClientRehydrated, useLoggedCalls, useTagNames } from "~/utils/hooks";
+import { useMemo } from "react";
+import { StaticColumnKeys } from "~/state/columnVisiblitySlice";
+
+type LoggedCall = RouterOutputs["loggedCalls"]["list"]["calls"][0];
+
+export const TableHeader = ({ showOptions }: { showOptions?: boolean }) => {
+  const matchingLogIds = useLoggedCalls().data?.matchingLogIds;
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const addAll = useAppStore((s) => s.selectedLogs.addSelectedLogIds);
+  const clearAll = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
+  const allSelected = useMemo(() => {
+    if (!matchingLogIds || !matchingLogIds.length) return false;
+    return matchingLogIds.every((id) => selectedLogIds.has(id));
+  }, [selectedLogIds, matchingLogIds]);
+  const tagNames = useTagNames().data;
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
+  return (
+    <Thead>
+      <Tr>
+        {showOptions && (
+          <Th pr={0}>
+            <HStack minW={16}>
+              <Checkbox
+                isChecked={allSelected}
+                onChange={() => {
+                  allSelected ? clearAll() : addAll(matchingLogIds || []);
+                }}
+              />
+              <Text>
+                ({selectedLogIds.size ? `${selectedLogIds.size}/` : ""}
+                {matchingLogIds?.length || 0})
+              </Text>
+            </HStack>
+          </Th>
+        )}
+        {visibleColumns.has(StaticColumnKeys.SENT_AT) && <Th>Sent At</Th>}
+        {visibleColumns.has(StaticColumnKeys.MODEL) && <Th>Model</Th>}
+        {tagNames
+          ?.filter((tagName) => visibleColumns.has(tagName))
+          .map((tagName) => (
+            <Th key={tagName} textTransform={"none"}>
+              {tagName}
+            </Th>
+          ))}
+        {visibleColumns.has(StaticColumnKeys.DURATION) && <Th isNumeric>Duration</Th>}
+        {visibleColumns.has(StaticColumnKeys.INPUT_TOKENS) && <Th isNumeric>Input tokens</Th>}
+        {visibleColumns.has(StaticColumnKeys.OUTPUT_TOKENS) && <Th isNumeric>Output tokens</Th>}
+        {visibleColumns.has(StaticColumnKeys.STATUS_CODE) && <Th isNumeric>Status</Th>}
+      </Tr>
+    </Thead>
+  );
+};
+
+export const TableRow = ({
+  loggedCall,
+  isExpanded,
+  onToggle,
+  showOptions,
+}: {
+  loggedCall: LoggedCall;
+  isExpanded: boolean;
+  onToggle: () => void;
+  showOptions?: boolean;
+}) => {
+  const isError = loggedCall.modelResponse?.statusCode !== 200;
+  const requestedAt = dayjs(loggedCall.requestedAt).format("MMMM D h:mm A");
+  const fullTime = dayjs(loggedCall.requestedAt).toString();
+
+  const isChecked = useAppStore((s) => s.selectedLogs.selectedLogIds.has(loggedCall.id));
+  const toggleChecked = useAppStore((s) => s.selectedLogs.toggleSelectedLogId);
+
+  const tagNames = useTagNames().data;
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+
+  const visibleTagNames = useMemo(() => {
+    return tagNames?.filter((tagName) => visibleColumns.has(tagName)) ?? [];
+  }, [tagNames, visibleColumns]);
+
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
+  return (
+    <>
+      <Tr
+        onClick={onToggle}
+        key={loggedCall.id}
+        _hover={{ bgColor: "gray.50", cursor: "pointer" }}
+        sx={{
+          "> td": { borderBottom: "none" },
+        }}
+        fontSize="sm"
+      >
+        {showOptions && (
+          <Td>
+            <Checkbox isChecked={isChecked} onChange={() => toggleChecked(loggedCall.id)} />
+          </Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.SENT_AT) && (
+          <Td>
+            <Tooltip label={fullTime} placement="top">
+              <Box whiteSpace="nowrap" minW="120px">
+                {requestedAt}
+              </Box>
+            </Tooltip>
+          </Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.MODEL) && (
+          <Td>
+            <HStack justifyContent="flex-start">
+              <Text
+                colorScheme="purple"
+                color="purple.500"
+                borderColor="purple.500"
+                px={1}
+                borderRadius={4}
+                borderWidth={1}
+                fontSize="xs"
+                whiteSpace="nowrap"
+              >
+                {loggedCall.model}
+              </Text>
+            </HStack>
+          </Td>
+        )}
+        {visibleTagNames.map((tagName) => (
+          <Td key={tagName}>{loggedCall.tags[tagName]}</Td>
+        ))}
+        {visibleColumns.has(StaticColumnKeys.DURATION) && (
+          <Td isNumeric>
+            {loggedCall.cacheHit ? (
+              <Text color="gray.500">Cached</Text>
+            ) : (
+              ((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2) + "s"
+            )}
+          </Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.INPUT_TOKENS) && (
+          <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.OUTPUT_TOKENS) && (
+          <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.STATUS_CODE) && (
+          <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
+            {loggedCall.modelResponse?.statusCode ?? "No response"}
+          </Td>
+        )}
+      </Tr>
+      <Tr>
+        <Td colSpan={visibleColumns.size + 1} w="full" p={0}>
+          <Collapse in={isExpanded} unmountOnExit={true}>
+            <VStack p={4} align="stretch">
+              <HStack align="stretch">
+                <VStack flex={1} align="stretch">
+                  <Heading size="sm">Input</Heading>
+                  <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
+                </VStack>
+                <VStack flex={1} align="stretch">
+                  <Heading size="sm">Output</Heading>
+                  <FormattedJson json={loggedCall.modelResponse?.respPayload} />
+                </VStack>
+              </HStack>
+              <ButtonGroup alignSelf="flex-end">
+                <Button as={Link} colorScheme="blue" href={{ pathname: "/experiments" }}>
+                  Experiments
+                </Button>
+              </ButtonGroup>
+            </VStack>
+          </Collapse>
+        </Td>
+      </Tr>
+    </>
+  );
+};
+
+export const EmptyTableRow = ({ filtersApplied = true }: { filtersApplied?: boolean }) => {
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const filters = useAppStore((state) => state.logFilters.filters);
+  const { isLoading } = useLoggedCalls();
+
+  if (isLoading) return null;
+
+  if (filters.length && filtersApplied) {
+    return (
+      <Tr>
+        <Td w="full" colSpan={visibleColumns.size + 1}>
+          <Text color="gray.500" textAlign="center" w="full" p={4}>
+            No matching request logs found. Try removing some filters.
+          </Text>
+        </Td>
+      </Tr>
+    );
+  }
+
+  return (
+    <Tr>
+      <Td w="full" colSpan={visibleColumns.size + 1}>
+        <Text color="gray.500" textAlign="center" w="full" p={4}>
+          This project has no request logs. Learn how to add request logs to your project in our{" "}
+          <ChakraLink
+            href="https://docs.openpipe.ai/getting-started/quick-start"
+            target="_blank"
+            color="blue.600"
+          >
+            Quick Start
+          </ChakraLink>{" "}
+          guide.
+        </Text>
+      </Td>
+    </Tr>
+  );
+};

app/src/env.mjs

@@ -21,6 +21,19 @@ export const env = createEnv({
     ANTHROPIC_API_KEY: z.string().default("placeholder"),
     SENTRY_AUTH_TOKEN: z.string().optional(),
     OPENPIPE_API_KEY: z.string().optional(),
+    SENDER_EMAIL: z.string().default("placeholder"),
+    SMTP_HOST: z.string().default("placeholder"),
+    SMTP_PORT: z.string().default("placeholder"),
+    SMTP_LOGIN: z.string().default("placeholder"),
+    SMTP_PASSWORD: z.string().default("placeholder"),
+    WORKER_CONCURRENCY: z
+      .string()
+      .default("10")
+      .transform((val) => parseInt(val)),
+    WORKER_MAX_POOL_SIZE: z
+      .string()
+      .default("10")
+      .transform((val) => parseInt(val)),
   },
 
   /**
@@ -33,8 +46,6 @@ export const env = createEnv({
     NEXT_PUBLIC_SOCKET_URL: z.string().url().default("http://localhost:3318"),
     NEXT_PUBLIC_HOST: z.string().url().default("http://localhost:3000"),
     NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
-    NEXT_PUBLIC_SHOW_DATA: z.string().optional(),
-    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: z.string().optional(),
   },
 
   /**
@@ -49,7 +60,6 @@ export const env = createEnv({
     NEXT_PUBLIC_POSTHOG_KEY: process.env.NEXT_PUBLIC_POSTHOG_KEY,
     NEXT_PUBLIC_SOCKET_URL: process.env.NEXT_PUBLIC_SOCKET_URL,
     NEXT_PUBLIC_HOST: process.env.NEXT_PUBLIC_HOST,
-    NEXT_PUBLIC_SHOW_DATA: process.env.NEXT_PUBLIC_SHOW_DATA,
     GITHUB_CLIENT_ID: process.env.GITHUB_CLIENT_ID,
     GITHUB_CLIENT_SECRET: process.env.GITHUB_CLIENT_SECRET,
     REPLICATE_API_TOKEN: process.env.REPLICATE_API_TOKEN,
@@ -57,7 +67,13 @@ export const env = createEnv({
     NEXT_PUBLIC_SENTRY_DSN: process.env.NEXT_PUBLIC_SENTRY_DSN,
     SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
     OPENPIPE_API_KEY: process.env.OPENPIPE_API_KEY,
-    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: process.env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS,
+    SENDER_EMAIL: process.env.SENDER_EMAIL,
+    SMTP_HOST: process.env.SMTP_HOST,
+    SMTP_PORT: process.env.SMTP_PORT,
+    SMTP_LOGIN: process.env.SMTP_LOGIN,
+    SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+    WORKER_CONCURRENCY: process.env.WORKER_CONCURRENCY,
+    WORKER_MAX_POOL_SIZE: process.env.WORKER_MAX_POOL_SIZE,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/modelProviders/frontendModelProviders.ts

@@ -1,6 +1,7 @@
 import openaiChatCompletionFrontend from "./openai-ChatCompletion/frontend";
 import replicateLlama2Frontend from "./replicate-llama2/frontend";
 import anthropicFrontend from "./anthropic-completion/frontend";
+import openpipeFrontend from "./openpipe-chat/frontend";
 import { type SupportedProvider, type FrontendModelProvider } from "./types";
 
 // Keep attributes here that need to be accessible from the frontend. We can't
@@ -10,6 +11,7 @@ const frontendModelProviders: Record<SupportedProvider, FrontendModelProvider<an
   "openai/ChatCompletion": openaiChatCompletionFrontend,
   "replicate/llama2": replicateLlama2Frontend,
   "anthropic/completion": anthropicFrontend,
+  "openpipe/Chat": openpipeFrontend,
 };
 
 export default frontendModelProviders;

app/src/modelProviders/modelProviders.ts

@@ -1,12 +1,14 @@
 import openaiChatCompletion from "./openai-ChatCompletion";
 import replicateLlama2 from "./replicate-llama2";
 import anthropicCompletion from "./anthropic-completion";
+import openpipeChatCompletion from "./openpipe-chat";
 import { type SupportedProvider, type ModelProvider } from "./types";
 
 const modelProviders: Record<SupportedProvider, ModelProvider<any, any, any>> = {
   "openai/ChatCompletion": openaiChatCompletion,
   "replicate/llama2": replicateLlama2,
   "anthropic/completion": anthropicCompletion,
+  "openpipe/Chat": openpipeChatCompletion,
 };
 
 export default modelProviders;

app/src/modelProviders/openai-ChatCompletion/getCompletion.ts

@@ -1,56 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unsafe-call */
-import {
+import { isArray, isString } from "lodash-es";
-  type ChatCompletionChunk,
-  type ChatCompletion,
-  type CompletionCreateParams,
-} from "openai/resources/chat";
-import { type CompletionResponse } from "../types";
-import { isArray, isString, omit } from "lodash-es";
-import { openai } from "~/server/utils/openai";
 import { APIError } from "openai";
-import frontendModelProvider from "./frontend";
+import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
-import modelProvider, { type SupportedModel } from ".";
+import mergeChunks from "openpipe/openai/mergeChunks";
-
+import { openai } from "~/server/utils/openai";
-const mergeStreamedChunks = (
+import { type CompletionResponse } from "../types";
-  base: ChatCompletion | null,
-  chunk: ChatCompletionChunk,
-): ChatCompletion => {
-  if (base === null) {
-    return mergeStreamedChunks({ ...chunk, choices: [] }, chunk);
-  }
-
-  const choices = [...base.choices];
-  for (const choice of chunk.choices) {
-    const baseChoice = choices.find((c) => c.index === choice.index);
-    if (baseChoice) {
-      baseChoice.finish_reason = choice.finish_reason ?? baseChoice.finish_reason;
-      baseChoice.message = baseChoice.message ?? { role: "assistant" };
-
-      if (choice.delta?.content)
-        baseChoice.message.content =
-          ((baseChoice.message.content as string) ?? "") + (choice.delta.content ?? "");
-      if (choice.delta?.function_call) {
-        const fnCall = baseChoice.message.function_call ?? {};
-        fnCall.name =
-          ((fnCall.name as string) ?? "") + ((choice.delta.function_call.name as string) ?? "");
-        fnCall.arguments =
-          ((fnCall.arguments as string) ?? "") +
-          ((choice.delta.function_call.arguments as string) ?? "");
-      }
-    } else {
-      // @ts-expect-error the types are correctly telling us that finish_reason
-      // could be null, but don't want to fix it right now.
-      choices.push({ ...omit(choice, "delta"), message: { role: "assistant", ...choice.delta } });
-    }
-  }
-
-  const merged: ChatCompletion = {
-    ...base,
-    choices,
-  };
-
-  return merged;
-};
 
 export async function getCompletion(
   input: CompletionCreateParams,
@@ -61,19 +15,25 @@ export async function getCompletion(
 
   try {
     if (onStream) {
-      console.log("got started");
       const resp = await openai.chat.completions.create(
-        { ...input, stream: true },
+        {
+          ...input,
+          stream: true,
+          openpipe: {
+            tags: {
+              prompt_id: "getCompletion",
+              stream: "true",
+            },
+          },
+        },
         {
           maxRetries: 0,
         },
       );
       for await (const part of resp) {
-        console.log("got part", part);
+        finalCompletion = mergeChunks(finalCompletion, part);
-        finalCompletion = mergeStreamedChunks(finalCompletion, part);
         onStream(finalCompletion);
       }
-      console.log("got final", finalCompletion);
       if (!finalCompletion) {
         return {
           type: "error",
@@ -83,7 +43,16 @@ export async function getCompletion(
       }
     } else {
       const resp = await openai.chat.completions.create(
-        { ...input, stream: false },
+        {
+          ...input,
+          stream: false,
+          openpipe: {
+            tags: {
+              prompt_id: "getCompletion",
+              stream: "false",
+            },
+          },
+        },
         {
           maxRetries: 0,
         },

app/src/modelProviders/openai-ChatCompletion/refinementActions.ts

@@ -12,7 +12,6 @@ export const refinementActions: Record<string, RefinementAction> = {
   
       definePrompt("openai/ChatCompletion", {
           model: "gpt-4",
-          stream: true,
           messages: [
               {
               role: "system",
@@ -29,7 +28,6 @@ export const refinementActions: Record<string, RefinementAction> = {
   
       definePrompt("openai/ChatCompletion", {
           model: "gpt-4",
-          stream: true,
           messages: [
               {
               role: "system",
@@ -120,13 +118,12 @@ export const refinementActions: Record<string, RefinementAction> = {
   "Convert to function call": {
     icon: TfiThought,
     description: "Use function calls to get output from the model in a more structured way.",
-    instructions: `OpenAI functions are a specialized way for an LLM to return output.
+    instructions: `OpenAI functions are a specialized way for an LLM to return its final output.
   
-      This is what a prompt looks like before adding a function:
+      Example 1 before:
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-4",
-        stream: true,
         messages: [
           {
             role: "system",
@@ -139,11 +136,10 @@ export const refinementActions: Record<string, RefinementAction> = {
         ],
       });
   
-      This is what one looks like after adding a function:
+      Example 1 after:
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-4",
-        stream: true,
         messages: [
           {
             role: "system",
@@ -156,7 +152,7 @@ export const refinementActions: Record<string, RefinementAction> = {
         ],
         functions: [
           {
-            name: "extract_sentiment",
+            name: "log_extracted_sentiment",
             parameters: {
               type: "object", // parameters must always be an object with a properties key
               properties: { // properties key is required
@@ -169,13 +165,13 @@ export const refinementActions: Record<string, RefinementAction> = {
           },
         ],
         function_call: {
-          name: "extract_sentiment",
+          name: "log_extracted_sentiment",
         },
       });
   
-      Here's another example of adding a function:
+      =========
 
-      Before:
+      Example 2 before:
   
       definePrompt("openai/ChatCompletion", {
           model: "gpt-3.5-turbo",
@@ -197,7 +193,7 @@ export const refinementActions: Record<string, RefinementAction> = {
           temperature: 0,
       });
   
-      After:
+      Example 2 after:
   
       definePrompt("openai/ChatCompletion", {
           model: "gpt-3.5-turbo",
@@ -215,7 +211,7 @@ export const refinementActions: Record<string, RefinementAction> = {
           temperature: 0,
           functions: [
             {
-              name: "score_post",
+              name: "log_post_score",
               parameters: {
                 type: "object",
                 properties: {
@@ -227,17 +223,16 @@ export const refinementActions: Record<string, RefinementAction> = {
             },
           ],
           function_call: {
-            name: "score_post",
+            name: "log_post_score",
           },
         });
   
-      Another example
+      =========
   
-      Before:
+      Example 3 before:
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-3.5-turbo",
-        stream: true,
         messages: [
           {
             role: "system",
@@ -246,7 +241,7 @@ export const refinementActions: Record<string, RefinementAction> = {
         ],
       });
   
-      After:
+      Example 3 after:
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-3.5-turbo",
@@ -258,22 +253,25 @@ export const refinementActions: Record<string, RefinementAction> = {
         ],
         functions: [
           {
-            name: "write_in_language",
+            name: "log_translated_text",
             parameters: {
               type: "object",
               properties: {
-                text: {
+                translated_text: {
                   type: "string",
+                  description: "The text, written in the language specified in the prompt",
                 },
               },
             },
           },
         ],
         function_call: {
-          name: "write_in_language",
+          name: "log_translated_text",
         },
       });
 
+      =========
+  
       Add an OpenAI function that takes one or more nested parameters that match the expected output from this prompt.`,
   },
 };

app/src/modelProviders/openpipe-chat/frontend.ts

@@ -0,0 +1,98 @@
+import { type OpenpipeChatOutput, type SupportedModel } from ".";
+import { type FrontendModelProvider } from "../types";
+import { refinementActions } from "./refinementActions";
+import {
+  templateOpenOrcaPrompt,
+  templateAlpacaInstructPrompt,
+  //   templateSystemUserAssistantPrompt,
+  templateInstructionInputResponsePrompt,
+  templateAiroborosPrompt,
+  templateGryphePrompt,
+  templateVicunaPrompt,
+} from "./templatePrompt";
+
+const frontendModelProvider: FrontendModelProvider<SupportedModel, OpenpipeChatOutput> = {
+  name: "OpenAI ChatCompletion",
+
+  models: {
+    "Open-Orca/OpenOrcaxOpenChat-Preview2-13B": {
+      name: "OpenOrcaxOpenChat-Preview2-13B",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+      templatePrompt: templateOpenOrcaPrompt,
+    },
+    "Open-Orca/OpenOrca-Platypus2-13B": {
+      name: "OpenOrca-Platypus2-13B",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/Open-Orca/OpenOrca-Platypus2-13B",
+      templatePrompt: templateAlpacaInstructPrompt,
+      defaultStopTokens: ["</s>"],
+    },
+    // "stabilityai/StableBeluga-13B": {
+    //   name: "StableBeluga-13B",
+    //   contextWindow: 4096,
+    //   pricePerSecond: 0.0003,
+    //   speed: "medium",
+    //   provider: "openpipe/Chat",
+    //   learnMoreUrl: "https://huggingface.co/stabilityai/StableBeluga-13B",
+    //   templatePrompt: templateSystemUserAssistantPrompt,
+    // },
+    "NousResearch/Nous-Hermes-Llama2-13b": {
+      name: "Nous-Hermes-Llama2-13b",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/NousResearch/Nous-Hermes-Llama2-13b",
+      templatePrompt: templateInstructionInputResponsePrompt,
+    },
+    "jondurbin/airoboros-l2-13b-gpt4-2.0": {
+      name: "airoboros-l2-13b-gpt4-2.0",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/jondurbin/airoboros-l2-13b-gpt4-2.0",
+      templatePrompt: templateAiroborosPrompt,
+    },
+    "lmsys/vicuna-13b-v1.5": {
+      name: "vicuna-13b-v1.5",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/lmsys/vicuna-13b-v1.5",
+      templatePrompt: templateVicunaPrompt,
+    },
+    "Gryphe/MythoMax-L2-13b": {
+      name: "MythoMax-L2-13b",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/Gryphe/MythoMax-L2-13b",
+      templatePrompt: templateGryphePrompt,
+    },
+    "NousResearch/Nous-Hermes-llama-2-7b": {
+      name: "Nous-Hermes-llama-2-7b",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/NousResearch/Nous-Hermes-llama-2-7b",
+      templatePrompt: templateInstructionInputResponsePrompt,
+    },
+  },
+
+  refinementActions,
+
+  normalizeOutput: (output) => ({ type: "text", value: output }),
+};
+
+export default frontendModelProvider;