Don't define CellWrapper inline

This way we don't re-render the entire cell every time a variable changes. Better performance and handles modals correctly.

OutputCell is still a pretty messy component, which we'll have to address at some point, but the complexity is still manageable for now.

.dockerignore

@@ -0,0 +1,5 @@
+**/node_modules/
+.git
+**/.venv/
+**/.env*
+**/.next/

.github/workflows/app-ci.yaml

@@ -6,6 +6,10 @@ on:
   push:
     branches: [main]
 
+defaults:
+  run:
+    working-directory: app
+
 jobs:
   run-checks:
     runs-on: ubuntu-latest

.gitignore

@@ -1,45 +1,5 @@
-# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
-# dependencies
-/node_modules
-/.pnp
-.pnp.js
-
-# testing
-/coverage
-
-# database
-/prisma/db.sqlite
-/prisma/db.sqlite-journal
-
-# next.js
-/.next/
-/out/
-next-env.d.ts
-
-# production
-/build
-
-# misc
-.DS_Store
-*.pem
-
-# debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-.pnpm-debug.log*
-
-# local env files
-# do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env
-.env*.local
-
-# vercel
-.vercel
-
-# typescript
+.venv/
+*.pyc
+node_modules/
 *.tsbuildinfo
-
-# Sentry Auth Token
-.sentryclirc

.prettierignore

@@ -1,2 +1,2 @@
 *.schema.json
-pnpm-lock.yaml
+app/pnpm-lock.yaml

Dockerfile

@@ -1,45 +0,0 @@
-# Adapted from https://create.t3.gg/en/deployment/docker#3-create-dockerfile
-
-FROM node:20.1.0-bullseye as base
-RUN yarn global add pnpm
-
-# DEPS
-FROM base as deps
-
-WORKDIR /app
-
-COPY prisma ./
-
-COPY package.json pnpm-lock.yaml ./
-
-RUN pnpm install --frozen-lockfile
-
-# BUILDER
-FROM base as builder
-
-# Include all NEXT_PUBLIC_* env vars here
-ARG NEXT_PUBLIC_POSTHOG_KEY
-ARG NEXT_PUBLIC_SOCKET_URL
-ARG NEXT_PUBLIC_HOST
-ARG NEXT_PUBLIC_SENTRY_DSN
-ARG SENTRY_AUTH_TOKEN
-
-WORKDIR /app
-COPY --from=deps /app/node_modules ./node_modules
-COPY . .
-RUN SKIP_ENV_VALIDATION=1 pnpm build
-
-# RUNNER
-FROM base as runner
-WORKDIR /app
-
-ENV NODE_ENV production
-ENV NEXT_TELEMETRY_DISABLED 1
-
-COPY --from=builder /app/ ./
-
-EXPOSE 3000
-ENV PORT 3000
-
-# Run the "run-prod.sh" script
-CMD /app/run-prod.sh

README.md

@@ -65,7 +65,14 @@ OpenPipe includes a tool to generate new test scenarios based on your existing p
 4. Clone this repository: `git clone https://github.com/openpipe/openpipe`
 5. Install the dependencies: `cd openpipe && pnpm install`
 6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`.
-7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma db push` to create the database.
+7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma migrate dev` to create the database.
 8. Create a [GitHub OAuth App](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app) and update the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` values. (Note: a PR to make auth optional when running locally would be a great contribution!)
 9. Start the app: `pnpm dev`.
 10. Navigate to [http://localhost:3000](http://localhost:3000)
+
+## Testing Locally
+
+1. Copy your `.env` file to `.env.test`.
+2. Update the `DATABASE_URL` to have a different database name than your development one
+3. Run `DATABASE_URL=[your new datatase url] pnpm prisma migrate dev --skip-seed --skip-generate`
+4. Run `pnpm test`

app/.env.example

@@ -31,3 +31,12 @@ NEXT_PUBLIC_HOST="http://localhost:3000"
 # Next Auth Github Provider
 GITHUB_CLIENT_ID="your_client_id"
 GITHUB_CLIENT_SECRET="your_secret"
+
+OPENPIPE_BASE_URL="http://localhost:3000/api/v1"
+OPENPIPE_API_KEY="your_key"
+
+SENDER_EMAIL="placeholder"
+SMTP_HOST="placeholder"
+SMTP_PORT="placeholder"
+SMTP_LOGIN="placeholder"
+SMTP_PASSWORD="placeholder"

app/.eslintrc.cjs

@@ -6,7 +6,7 @@ const config = {
   overrides: [
     {
       extends: ["plugin:@typescript-eslint/recommended-requiring-type-checking"],
-      files: ["*.ts", "*.tsx"],
+      files: ["*.mts", "*.ts", "*.tsx"],
       parserOptions: {
         project: path.join(__dirname, "tsconfig.json"),
       },

app/.gitignore

@@ -0,0 +1,49 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# database
+/prisma/db.sqlite
+/prisma/db.sqlite-journal
+
+# next.js
+/.next/
+/out/
+next-env.d.ts
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# local env files
+# do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
+.env
+.env*.local
+.env.test
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+
+# Sentry Auth Token
+.sentryclirc
+
+# custom openai intialization
+src/server/utils/openaiCustomConfig.json

app/@types/nextjs-routes.d.ts

@@ -14,13 +14,18 @@ declare module "nextjs-routes" {
     | StaticRoute<"/account/signin">
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
     | StaticRoute<"/api/experiments/og-image">
-    | StaticRoute<"/api/sentry-example-api">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
+    | DynamicRoute<"/api/v1/[...trpc]", { "trpc": string[] }>
+    | StaticRoute<"/api/v1/openapi">
+    | StaticRoute<"/dashboard">
     | DynamicRoute<"/data/[id]", { "id": string }>
     | StaticRoute<"/data">
     | DynamicRoute<"/experiments/[id]", { "id": string }>
     | StaticRoute<"/experiments">
     | StaticRoute<"/">
+    | DynamicRoute<"/invitations/[invitationToken]", { "invitationToken": string }>
+    | StaticRoute<"/project/settings">
+    | StaticRoute<"/request-logs">
     | StaticRoute<"/sentry-example-page">
     | StaticRoute<"/world-champs">
     | StaticRoute<"/world-champs/signup">;

app/Dockerfile

@@ -0,0 +1,48 @@
+# Adapted from https://create.t3.gg/en/deployment/docker#3-create-dockerfile
+
+FROM node:20.1.0-bullseye as base
+RUN yarn global add pnpm
+
+# DEPS
+FROM base as deps
+
+WORKDIR /code
+
+COPY app/prisma app/package.json ./app/
+COPY client-libs/typescript/package.json ./client-libs/typescript/
+COPY pnpm-lock.yaml pnpm-workspace.yaml ./
+
+RUN cd app && pnpm install --frozen-lockfile
+
+# BUILDER
+FROM base as builder
+
+# Include all NEXT_PUBLIC_* env vars here
+ARG NEXT_PUBLIC_POSTHOG_KEY
+ARG NEXT_PUBLIC_SOCKET_URL
+ARG NEXT_PUBLIC_HOST
+ARG NEXT_PUBLIC_SENTRY_DSN
+ARG SENTRY_AUTH_TOKEN
+ARG NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS
+
+WORKDIR /code
+COPY --from=deps /code/node_modules ./node_modules
+COPY --from=deps /code/app/node_modules ./app/node_modules
+COPY --from=deps /code/client-libs/typescript/node_modules ./client-libs/typescript/node_modules
+COPY . .
+RUN cd app && SKIP_ENV_VALIDATION=1 pnpm build
+
+# RUNNER
+FROM base as runner
+WORKDIR /code/app
+
+ENV NODE_ENV production
+ENV NEXT_TELEMETRY_DISABLED 1
+
+COPY --from=builder /code/ /code/
+
+EXPOSE 3000
+ENV PORT 3000
+
+# Run the "run-prod.sh" script
+CMD /code/app/run-prod.sh

app/next.config.mjs

@@ -36,6 +36,8 @@ let config = {
     });
     return config;
   },
+
+  transpilePackages: ["openpipe"],
 };
 
 config = nextRoutes()(config);

app/openapitools.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json",
+  "spaces": 2,
+  "generator-cli": {
+    "version": "6.6.0"
+  }
+}

app/package.json

@@ -1,5 +1,6 @@
 {
-  "name": "openpipe",
+  "name": "openpipe-app",
+  "private": true,
   "type": "module",
   "version": "0.1.0",
   "license": "Apache-2.0",
@@ -9,22 +10,21 @@
   },
   "scripts": {
     "build": "next build",
-    "dev:next": "next dev",
+    "dev:next": "TZ=UTC next dev",
     "dev:wss": "pnpm tsx --watch src/wss-server.ts",
     "dev:worker": "NODE_ENV='development' pnpm tsx --watch src/server/tasks/worker.ts",
     "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss' 'pnpm dev:worker'",
     "postinstall": "prisma generate",
     "lint": "next lint",
-    "start": "next start",
-    "codegen": "tsx src/codegen/export-openai-types.ts",
+    "start": "TZ=UTC next start",
+    "codegen:clients": "tsx src/server/scripts/client-codegen.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
-    "test": "pnpm vitest --no-threads"
+    "test": "pnpm vitest"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.5.8",
     "@apidevtools/json-schema-ref-parser": "^10.1.0",
-    "@babel/preset-typescript": "^7.22.5",
     "@babel/standalone": "^7.22.9",
     "@chakra-ui/anatomy": "^2.2.0",
     "@chakra-ui/next-js": "^2.1.4",
@@ -37,6 +37,7 @@
     "@monaco-editor/loader": "^1.3.3",
     "@next-auth/prisma-adapter": "^1.0.5",
     "@prisma/client": "^4.14.0",
+    "@sendinblue/client": "^3.3.1",
     "@sentry/nextjs": "^7.61.0",
     "@t3-oss/env-nextjs": "^0.3.1",
     "@tabler/icons-react": "^2.22.0",
@@ -50,6 +51,7 @@
     "chroma-js": "^2.4.2",
     "concurrently": "^8.2.0",
     "cors": "^2.8.5",
+    "crypto-random-string": "^5.0.0",
     "dayjs": "^1.11.8",
     "dedent": "^1.0.1",
     "dotenv": "^16.3.1",
@@ -62,12 +64,19 @@
     "json-schema-to-typescript": "^13.0.2",
     "json-stringify-pretty-compact": "^4.0.0",
     "jsonschema": "^1.4.1",
+    "kysely": "^0.26.1",
     "lodash-es": "^4.17.21",
+    "lucide-react": "^0.265.0",
+    "marked": "^7.0.3",
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
     "next-query-params": "^4.2.3",
+    "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
+    "nodemailer": "^6.9.4",
     "openai": "4.0.0-beta.7",
+    "openpipe": "workspace:*",
+    "pg": "^8.11.2",
     "pluralize": "^8.0.0",
     "posthog-js": "^1.75.3",
     "posthog-node": "^3.1.1",
@@ -83,10 +92,12 @@
     "react-syntax-highlighter": "^15.5.0",
     "react-textarea-autosize": "^8.5.0",
     "recast": "^0.23.3",
+    "recharts": "^2.7.2",
     "replicate": "^0.12.3",
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
     "superjson": "1.12.2",
+    "trpc-openapi": "^1.2.0",
     "tsx": "^3.12.7",
     "type-fest": "^4.0.0",
     "use-query-params": "^2.2.1",
@@ -106,6 +117,8 @@
     "@types/json-schema": "^7.0.12",
     "@types/lodash-es": "^4.17.8",
     "@types/node": "^18.16.0",
+    "@types/nodemailer": "^6.4.9",
+    "@types/pg": "^8.10.2",
     "@types/pluralize": "^0.0.30",
     "@types/prismjs": "^1.26.0",
     "@types/react": "^18.2.6",
@@ -120,6 +133,7 @@
     "eslint-plugin-unused-imports": "^2.0.0",
     "monaco-editor": "^0.40.0",
     "openapi-typescript": "^6.3.4",
+    "openapi-typescript-codegen": "^0.25.0",
     "prisma": "^4.14.0",
     "raw-loader": "^4.0.2",
     "typescript": "^5.0.4",

app/prisma/migrations/20230806024615_add_logged_calls_and_api_keys/migration.sql

@@ -0,0 +1,90 @@
+-- CreateTable
+CREATE TABLE "LoggedCall" (
+    "id" UUID NOT NULL,
+    "startTime" TIMESTAMP(3) NOT NULL,
+    "cacheHit" BOOLEAN NOT NULL,
+    "modelResponseId" UUID NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCall_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LoggedCallModelResponse" (
+    "id" UUID NOT NULL,
+    "reqPayload" JSONB NOT NULL,
+    "respStatus" INTEGER,
+    "respPayload" JSONB,
+    "error" TEXT,
+    "startTime" TIMESTAMP(3) NOT NULL,
+    "endTime" TIMESTAMP(3) NOT NULL,
+    "cacheKey" TEXT,
+    "durationMs" INTEGER,
+    "inputTokens" INTEGER,
+    "outputTokens" INTEGER,
+    "finishReason" TEXT,
+    "completionId" TEXT,
+    "totalCost" DECIMAL(18,12),
+    "originalLoggedCallId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCallModelResponse_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LoggedCallTag" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "value" TEXT,
+    "loggedCallId" UUID NOT NULL,
+
+    CONSTRAINT "LoggedCallTag_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "ApiKey" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "apiKey" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "ApiKey_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "LoggedCall_startTime_idx" ON "LoggedCall"("startTime");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LoggedCallModelResponse_originalLoggedCallId_key" ON "LoggedCallModelResponse"("originalLoggedCallId");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallModelResponse_cacheKey_idx" ON "LoggedCallModelResponse"("cacheKey");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_name_idx" ON "LoggedCallTag"("name");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_name_value_idx" ON "LoggedCallTag"("name", "value");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ApiKey_apiKey_key" ON "ApiKey"("apiKey");
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_modelResponseId_fkey" FOREIGN KEY ("modelResponseId") REFERENCES "LoggedCallModelResponse"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCallModelResponse" ADD CONSTRAINT "LoggedCallModelResponse_originalLoggedCallId_fkey" FOREIGN KEY ("originalLoggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCallTag" ADD CONSTRAINT "LoggedCallTag_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/migrations/20230807044936_add_name_to_organization/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Organization" ADD COLUMN     "name" TEXT NOT NULL DEFAULT 'Project 1';

app/prisma/migrations/20230808034313_make_model_response_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LoggedCall" ALTER COLUMN "modelResponseId" DROP NOT NULL;

app/prisma/migrations/20230809222649_rename_organization_to_project/migration.sql

@@ -0,0 +1,37 @@
+-- Rename Enum
+ALTER TYPE "OrganizationUserRole" RENAME TO "ProjectUserRole";
+
+-- Drop and recreate foreign keys
+ALTER TABLE "ApiKey" DROP CONSTRAINT "ApiKey_organizationId_fkey";
+ALTER TABLE "Dataset" DROP CONSTRAINT "Dataset_organizationId_fkey";
+ALTER TABLE "Experiment" DROP CONSTRAINT "Experiment_organizationId_fkey";
+ALTER TABLE "LoggedCall" DROP CONSTRAINT "LoggedCall_organizationId_fkey";
+ALTER TABLE "OrganizationUser" DROP CONSTRAINT "OrganizationUser_organizationId_fkey";
+ALTER TABLE "OrganizationUser" DROP CONSTRAINT "OrganizationUser_userId_fkey";
+
+-- Rename columns
+ALTER TABLE "ApiKey" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Dataset" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Experiment" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "LoggedCall" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "OrganizationUser" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Organization" RENAME COLUMN "personalOrgUserId" TO "personalProjectUserId";
+
+-- Rename table
+ALTER TABLE "Organization" RENAME TO "Project";
+ALTER TABLE "OrganizationUser" RENAME TO "ProjectUser";
+
+-- Recreate foreign keys
+ALTER TABLE "Experiment" ADD CONSTRAINT "Experiment_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "Dataset" ADD CONSTRAINT "Dataset_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ProjectUser" ADD CONSTRAINT "ProjectUser_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ProjectUser" ADD CONSTRAINT "ProjectUser_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- Rename indexes
+ALTER TABLE "Project" RENAME CONSTRAINT "Organization_pkey" TO "Project_pkey";
+ALTER TABLE "ProjectUser" RENAME CONSTRAINT "OrganizationUser_pkey" TO "ProjectUser_pkey";
+ALTER TABLE "Project" RENAME CONSTRAINT "Organization_personalOrgUserId_fkey" TO "Project_personalProjectUserId_fkey";
+ALTER INDEX "Organization_personalOrgUserId_key" RENAME TO "Project_personalProjectUserId_key";
+ALTER INDEX "OrganizationUser_organizationId_userId_key" RENAME TO "ProjectUser_projectId_userId_key";

app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

app/prisma/migrations/20230811200114_rename_model_response_fields/migration.sql

@@ -0,0 +1,66 @@
+/*
+  Warnings:
+
+  - You are about to rename the column `completionTokens` to `outputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `promptTokens` to `inputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `startTime` on the `LoggedCall` table to `requestedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `startTime` on the `LoggedCallModelResponse` table to `requestedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `endTime` on the `LoggedCallModelResponse` table to `receivedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `error` on the `LoggedCallModelResponse` table to `errorMessage`. Ensure compatibility with application logic.
+  - You are about to rename the column `respStatus` on the `LoggedCallModelResponse` table to `statusCode`. Ensure compatibility with application logic.
+  - You are about to rename the column `totalCost` on the `LoggedCallModelResponse` table to `cost`. Ensure compatibility with application logic.
+  - You are about to rename the column `inputHash` on the `ModelResponse` table to `cacheKey`. Ensure compatibility with application logic.
+  - You are about to rename the column `output` on the `ModelResponse` table to `respPayload`. Ensure compatibility with application logic.
+
+*/
+-- DropIndex
+DROP INDEX "LoggedCall_startTime_idx";
+
+-- DropIndex
+DROP INDEX "ModelResponse_inputHash_idx";
+
+-- Rename completionTokens to outputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "completionTokens" TO "outputTokens";
+
+-- Rename promptTokens to inputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "promptTokens" TO "inputTokens";
+
+-- AlterTable
+ALTER TABLE "LoggedCall"
+RENAME COLUMN "startTime" TO "requestedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "startTime" TO "requestedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "endTime" TO "receivedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "error" TO "errorMessage";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "respStatus" TO "statusCode";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "totalCost" TO "cost";
+
+-- AlterTable
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "inputHash" TO "cacheKey";
+
+-- AlterTable
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "output" TO "respPayload";
+
+-- CreateIndex
+CREATE INDEX "LoggedCall_requestedAt_idx" ON "LoggedCall"("requestedAt");
+
+-- CreateIndex
+CREATE INDEX "ModelResponse_cacheKey_idx" ON "ModelResponse"("cacheKey");

app/prisma/migrations/20230811231429_add_model_to_logged_call/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LoggedCall" ADD COLUMN     "model" TEXT;

app/prisma/migrations/20230814233722_index_tags_by_project/migration.sql

@@ -0,0 +1,22 @@
+-- DropIndex
+DROP INDEX "LoggedCallTag_name_idx";
+DROP INDEX "LoggedCallTag_name_value_idx";
+
+-- AlterTable: Add projectId column without NOT NULL constraint for now
+ALTER TABLE "LoggedCallTag" ADD COLUMN "projectId" UUID;
+
+-- Set the default value
+UPDATE "LoggedCallTag" lct
+SET "projectId" = lc."projectId"
+FROM "LoggedCall" lc
+WHERE lct."loggedCallId" = lc.id;
+
+-- Now set the NOT NULL constraint
+ALTER TABLE "LoggedCallTag" ALTER COLUMN "projectId" SET NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_projectId_name_idx" ON "LoggedCallTag"("projectId", "name");
+CREATE INDEX "LoggedCallTag_projectId_name_value_idx" ON "LoggedCallTag"("projectId", "name", "value");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LoggedCallTag_loggedCallId_name_key" ON "LoggedCallTag"("loggedCallId", "name");

app/prisma/migrations/20230815234811_add_user_invitations/migration.sql

@@ -0,0 +1,25 @@
+-- CreateTable
+CREATE TABLE "UserInvitation" (
+    "id" UUID NOT NULL,
+    "projectId" UUID NOT NULL,
+    "email" TEXT NOT NULL,
+    "role" "ProjectUserRole" NOT NULL,
+    "invitationToken" TEXT NOT NULL,
+    "senderId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "UserInvitation_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "UserInvitation_invitationToken_key" ON "UserInvitation"("invitationToken");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "UserInvitation_projectId_email_key" ON "UserInvitation"("projectId", "email");
+
+-- AddForeignKey
+ALTER TABLE "UserInvitation" ADD CONSTRAINT "UserInvitation_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "UserInvitation" ADD CONSTRAINT "UserInvitation_senderId_fkey" FOREIGN KEY ("senderId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/schema.prisma

@@ -16,8 +16,8 @@ model Experiment {
 
     sortIndex Int @default(0)
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -112,17 +112,17 @@ model ScenarioVariantCell {
 model ModelResponse {
     id String @id @default(uuid()) @db.Uuid
 
-    inputHash        String
-    requestedAt      DateTime?
-    receivedAt       DateTime?
-    output           Json?
-    cost             Float?
-    promptTokens     Int?
-    completionTokens Int?
-    statusCode       Int?
-    errorMessage     String?
-    retryTime        DateTime?
-    outdated         Boolean   @default(false)
+    cacheKey     String
+    requestedAt  DateTime?
+    receivedAt   DateTime?
+    respPayload  Json?
+    cost         Float?
+    inputTokens  Int?
+    outputTokens Int?
+    statusCode   Int?
+    errorMessage String?
+    retryTime    DateTime?
+    outdated     Boolean   @default(false)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -131,7 +131,7 @@ model ModelResponse {
     scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
     outputEvaluations     OutputEvaluation[]
 
-    @@index([inputHash])
+    @@index([cacheKey])
 }
 
 enum EvalType {
@@ -180,8 +180,8 @@ model Dataset {
     name           String
     datasetEntries DatasetEntry[]
 
-    organizationId String       @db.Uuid
-    organization   Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String  @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -200,31 +200,36 @@ model DatasetEntry {
     updatedAt DateTime @updatedAt
 }
 
-model Organization {
-    id                String  @id @default(uuid()) @db.Uuid
-    personalOrgUserId String? @unique @db.Uuid
-    PersonalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+model Project {
+    id   String @id @default(uuid()) @db.Uuid
+    name String @default("Project 1")
 
-    createdAt         DateTime           @default(now())
-    updatedAt         DateTime           @updatedAt
-    organizationUsers OrganizationUser[]
-    experiments       Experiment[]
-    datasets          Dataset[]
+    personalProjectUserId String? @unique @db.Uuid
+    personalProjectUser   User?   @relation(fields: [personalProjectUserId], references: [id], onDelete: Cascade)
+
+    createdAt               DateTime      @default(now())
+    updatedAt               DateTime      @updatedAt
+    projectUsers            ProjectUser[]
+    projectUserInvitations  UserInvitation[]
+    experiments             Experiment[]
+    datasets                Dataset[]
+    loggedCalls             LoggedCall[]
+    apiKeys                 ApiKey[]
 }
 
-enum OrganizationUserRole {
+enum ProjectUserRole {
     ADMIN
     MEMBER
     VIEWER
 }
 
-model OrganizationUser {
+model ProjectUser {
     id String @id @default(uuid()) @db.Uuid
 
-    role OrganizationUserRole
+    role ProjectUserRole
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     userId String @db.Uuid
     user   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
@@ -232,7 +237,7 @@ model OrganizationUser {
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    @@unique([organizationId, userId])
+    @@unique([projectId, userId])
 }
 
 model WorldChampEntrant {
@@ -249,6 +254,102 @@ model WorldChampEntrant {
     @@unique([userId])
 }
 
+model LoggedCall {
+    id String @id @default(uuid()) @db.Uuid
+
+    requestedAt DateTime
+
+    // True if this call was served from the cache, false otherwise
+    cacheHit Boolean
+
+    // A LoggedCall is always associated with a LoggedCallModelResponse. If this
+    // is a cache miss, we create a new LoggedCallModelResponse.
+    // If it's a cache hit, it's a pre-existing LoggedCallModelResponse.    
+    modelResponseId String?                  @db.Uuid
+    modelResponse   LoggedCallModelResponse? @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
+
+    // The responses created by this LoggedCall. Will be empty if this LoggedCall was a cache hit.
+    createdResponses LoggedCallModelResponse[] @relation(name: "ModelResponseOriginalCall")
+
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    model String?
+    tags  LoggedCallTag[]
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+
+    @@index([requestedAt])
+}
+
+model LoggedCallModelResponse {
+    id String @id @default(uuid()) @db.Uuid
+
+    reqPayload Json
+
+    // The HTTP status returned by the model provider
+    statusCode  Int?
+    respPayload Json?
+
+    // Should be null if the request was successful, and some string if the request failed.
+    errorMessage String?
+
+    requestedAt DateTime
+    receivedAt  DateTime
+
+    // Note: the function to calculate the cacheKey should include the project
+    // ID so we don't share cached responses between projects, which could be an
+    // attack vector. Also, we should only set the cacheKey on the model if the
+    // request was successful.
+    cacheKey String?
+
+    // Derived fields
+    durationMs   Int?
+    inputTokens  Int?
+    outputTokens Int?
+    finishReason String?
+    completionId String?
+    cost         Decimal? @db.Decimal(18, 12)
+
+    // The LoggedCall that created this LoggedCallModelResponse
+    originalLoggedCallId String     @unique @db.Uuid
+    originalLoggedCall   LoggedCall @relation(name: "ModelResponseOriginalCall", fields: [originalLoggedCallId], references: [id], onDelete: Cascade)
+
+    createdAt   DateTime     @default(now())
+    updatedAt   DateTime     @updatedAt
+    loggedCalls LoggedCall[]
+
+    @@index([cacheKey])
+}
+
+model LoggedCallTag {
+    id    String  @id @default(uuid()) @db.Uuid
+    name  String
+    value String?
+    projectId String   @db.Uuid
+
+    loggedCallId String     @db.Uuid
+    loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
+
+    @@unique([loggedCallId, name])
+    @@index([projectId, name])
+    @@index([projectId, name, value])
+}
+
+model ApiKey {
+    id String @id @default(uuid()) @db.Uuid
+
+    name   String
+    apiKey String @unique
+
+    projectId String  @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
 model Account {
     id                       String  @id @default(uuid()) @db.Uuid
     userId                   String  @db.Uuid
@@ -290,16 +391,33 @@ model User {
 
     role UserRole @default(USER)
 
-    accounts          Account[]
-    sessions          Session[]
-    organizationUsers OrganizationUser[]
-    organizations     Organization[]
-    worldChampEntrant WorldChampEntrant?
+    accounts                Account[]
+    sessions                Session[]
+    projectUsers            ProjectUser[]
+    projects                Project[]
+    worldChampEntrant       WorldChampEntrant?
+    sentUserInvitations     UserInvitation[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @default(now()) @updatedAt
 }
 
+model UserInvitation {
+    id String @id @default(uuid()) @db.Uuid
+
+    projectId String @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    email String
+    role ProjectUserRole
+    invitationToken String @unique
+    senderId String @db.Uuid
+    sender User @relation(fields: [senderId], references: [id], onDelete: Cascade)
+
+    @@unique([projectId, email])
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
 model VerificationToken {
     identifier String
     token      String   @unique

app/prisma/seed.ts

@@ -2,20 +2,31 @@ import { prisma } from "~/server/db";
 import dedent from "dedent";
 import { generateNewCell } from "~/server/utils/generateNewCell";
 import { promptConstructorVersion } from "~/promptConstructor/version";
+import { env } from "~/env.mjs";
 
 const defaultId = "11111111-1111-1111-1111-111111111111";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
+if (env.OPENPIPE_API_KEY) {
+  await prisma.apiKey.create({
+    data: {
+      projectId: project.id,
+      name: "Default API Key",
+      apiKey: env.OPENPIPE_API_KEY,
+    },
+  });
+}
+
 await prisma.experiment.deleteMany({
   where: {
     id: defaultId,
@@ -26,7 +37,7 @@ await prisma.experiment.create({
   data: {
     id: defaultId,
     label: "Country Capitals Example",
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 

app/prisma/seedAgiEval.ts

@@ -7,14 +7,14 @@ import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111112";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
@@ -47,7 +47,7 @@ for (const dataset of datasets) {
   const oldExperiment = await prisma.experiment.findFirst({
     where: {
       label: experimentName,
-      organizationId: org.id,
+      projectId: project.id,
     },
   });
   if (oldExperiment) {
@@ -60,7 +60,7 @@ for (const dataset of datasets) {
     data: {
       id: oldExperiment?.id ?? undefined,
       label: experimentName,
-      organizationId: org.id,
+      projectId: project.id,
     },
   });
 

app/prisma/seedTwitterSentiment.ts

@@ -6,14 +6,14 @@ import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111112";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
@@ -27,7 +27,7 @@ const experimentName = `Twitter Sentiment Analysis`;
 const oldExperiment = await prisma.experiment.findFirst({
   where: {
     label: experimentName,
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 if (oldExperiment) {
@@ -40,7 +40,7 @@ const experiment = await prisma.experiment.create({
   data: {
     id: oldExperiment?.id ?? undefined,
     label: experimentName,
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 

app/public/favicons/safari-pinned-tab.svg

@@ -9,10 +9,9 @@ Created by potrace 1.14, written by Peter Selinger 2001-2017
 </metadata>
 <g transform="translate(0.000000,550.000000) scale(0.100000,-0.100000)"
 fill="#000000" stroke="none">
-<path d="M813 5478 c-18 -13 -37 -36 -43 -52 -6 -19 -10 -236 -10 -603 0 -638
--1 -626 65 -657 25 -12 67 -16 179 -16 l146 0 0 -2032 0 -2032 23 -33 c12 -18
-35 -37 51 -43 19 -7 539 -10 1528 -10 1663 0 1549 -5 1582 65 14 30 16 235 16
-2059 l0 2026 156 0 156 0 39 39 39 39 0 587 c0 651 1 638 -65 669 -30 14 -223
-16 -1932 16 l-1898 0 -32 -22z"/>
+<path d="M785 5474 l-25 -27 0 -622 0 -622 25 -27 24 -26 171 0 170 0 0 -2050
+0 -2051 25 -25 24 -24 1557 2 1556 3 19 24 c19 23 19 70 19 2072 l0 2049 169
+0 c165 0 169 1 195 25 l26 24 0 626 0 626 -26 24 -27 25 -1939 0 -1939 0 -24
+-26z"/>
 </g>
 </svg>

app/public/logo.svg

@@ -0,0 +1,28 @@
+<svg width="398" height="550" viewBox="0 0 398 550" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M39 125H359V542C359 546.418 355.418 550 351 550H47C42.5817 550 39 546.418 39 542V125Z" fill="black"/>
+<path d="M0 8C0 3.58172 3.58172 0 8 0H390C394.418 0 398 3.58172 398 8V127C398 131.418 394.418 135 390 135H7.99999C3.58171 135 0 131.418 0 127V8Z" fill="black"/>
+<path d="M50 135H348V535C348 537.209 346.209 539 344 539H54C51.7909 539 50 537.209 50 535V135Z" fill="#FF5733"/>
+<path d="M11 14.0001C11 11.791 12.7909 10.0001 15 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H15C12.7909 124 11 122.209 11 120V14.0001Z" fill="#FF5733"/>
+<path d="M11 14.0001C11 11.791 12.7909 10.0001 15 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H15C12.7909 124 11 122.209 11 120V14.0001Z" fill="url(#paint0_linear_102_49)"/>
+<path d="M50 134H348V535C348 537.209 346.209 539 344 539H54C51.7909 539 50 537.209 50 535V134Z" fill="url(#paint1_linear_102_49)"/>
+<path d="M108 142H156V535H108V142Z" fill="white"/>
+<path d="M300 135H348V535C348 537.209 346.209 539 344 539H300V135Z" fill="white" fill-opacity="0.25"/>
+<path d="M96 142H108V535H96V142Z" fill="white" fill-opacity="0.5"/>
+<path d="M84 10.0001H133V120H84V10.0001Z" fill="white"/>
+<path d="M339 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H339V10.0001Z" fill="white" fill-opacity="0.25"/>
+<path d="M71.9995 10.0001H83.9995V120H71.9995V10.0001Z" fill="white" fill-opacity="0.5"/>
+<path d="M108 534.529H156V539.019H108V534.529Z" fill="#AAAAAA"/>
+<path opacity="0.5" d="M95.9927 534.529H107.982V539.019H95.9927V534.529Z" fill="#AAAAAA"/>
+<path d="M84.0029 119.887H133.007V124.027H84.0029V119.887Z" fill="#AAAAAA"/>
+<path opacity="0.5" d="M71.9883 119.887H83.978V124.027H71.9883V119.887Z" fill="#AAAAAA"/>
+<defs>
+<linearGradient id="paint0_linear_102_49" x1="335" y1="67.0001" x2="137" y2="67.0001" gradientUnits="userSpaceOnUse">
+<stop stop-color="#D62600"/>
+<stop offset="1" stop-color="#FF5733" stop-opacity="0"/>
+</linearGradient>
+<linearGradient id="paint1_linear_102_49" x1="306.106" y1="336.5" x2="149.597" y2="336.5" gradientUnits="userSpaceOnUse">
+<stop stop-color="#D62600"/>
+<stop offset="1" stop-color="#FF5733" stop-opacity="0"/>
+</linearGradient>
+</defs>
+</svg>

app/src/components/AutoResizeTextArea.tsx

@@ -1,13 +1,13 @@
 import { Textarea, type TextareaProps } from "@chakra-ui/react";
 import ResizeTextarea from "react-textarea-autosize";
-import React, { useLayoutEffect, useState } from "react";
+import React, { useEffect, useState } from "react";
 
 export const AutoResizeTextarea: React.ForwardRefRenderFunction<
   HTMLTextAreaElement,
   TextareaProps & { minRows?: number }
 > = ({ minRows = 1, overflowY = "hidden", ...props }, ref) => {
   const [isRerendered, setIsRerendered] = useState(false);
-  useLayoutEffect(() => setIsRerendered(true), []);
+  useEffect(() => setIsRerendered(true), []);
 
   return (
     <Textarea

app/src/components/ChangeModelModal/ModelStatsCard.tsx

@@ -87,7 +87,7 @@ export const ModelStatsCard = ({
               label="Price"
               info={
                 <Text>
-                  ${model.pricePerSecond.toFixed(3)}
+                  ${model.pricePerSecond.toFixed(4)}
                   <Text color="gray.500"> / second</Text>
                 </Text>
               }

app/src/components/CopiableCode.tsx

@@ -0,0 +1,51 @@
+import { HStack, Icon, IconButton, Tooltip, Text, type StackProps } from "@chakra-ui/react";
+import { useState } from "react";
+import { MdContentCopy } from "react-icons/md";
+import { useHandledAsyncCallback } from "~/utils/hooks";
+
+const CopiableCode = ({ code, ...rest }: { code: string } & StackProps) => {
+  const [copied, setCopied] = useState(false);
+
+  const [copyToClipboard] = useHandledAsyncCallback(async () => {
+    await navigator.clipboard.writeText(code);
+    setCopied(true);
+  }, [code]);
+
+  return (
+    <HStack
+      backgroundColor="blackAlpha.800"
+      color="white"
+      borderRadius={4}
+      padding={3}
+      w="full"
+      justifyContent="space-between"
+      alignItems="flex-start"
+      {...rest}
+    >
+      <Text
+        fontFamily="inconsolata"
+        fontWeight="bold"
+        letterSpacing={0.5}
+        overflowX="auto"
+        whiteSpace="pre-wrap"
+      >
+        {code}
+        {/* Necessary for trailing newline to actually be displayed */}
+        {code.endsWith("\n") ? "\n" : ""}
+      </Text>
+      <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
+        <IconButton
+          aria-label="Copy"
+          icon={<Icon as={MdContentCopy} boxSize={5} />}
+          size="xs"
+          colorScheme="white"
+          variant="ghost"
+          onClick={copyToClipboard}
+          onMouseLeave={() => setCopied(false)}
+        />
+      </Tooltip>
+    </HStack>
+  );
+};
+
+export default CopiableCode;

app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -14,6 +14,7 @@ import {
 import { useRouter } from "next/router";
 import { useRef } from "react";
 import { BsTrash } from "react-icons/bs";
+import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 
@@ -23,6 +24,8 @@ export const DeleteButton = () => {
   const utils = api.useContext();
   const router = useRouter();
 
+  const closeDrawer = useAppStore((s) => s.closeDrawer);
+
   const { isOpen, onOpen, onClose } = useDisclosure();
   const cancelRef = useRef<HTMLButtonElement>(null);
 
@@ -31,6 +34,8 @@ export const DeleteButton = () => {
     await mutation.mutateAsync({ id: experiment.data.id });
     await utils.experiments.list.invalidate();
     await router.push({ pathname: "/experiments" });
+    closeDrawer();
+
     onClose();
   }, [mutation, experiment.data?.id, router]);
 

app/src/components/Favicon.tsx

@@ -8,8 +8,8 @@ export default function Favicon() {
       <link rel="icon" type="image/png" sizes="16x16" href="/favicons/favicon-16x16.png" />
       <link rel="manifest" href="/favicons/site.webmanifest" />
       <link rel="shortcut icon" href="/favicons/favicon.ico" />
+      <link rel="mask-icon" href="/favicons/safari-pinned-tab.svg" color="#5bbad5" />
       <meta name="msapplication-TileColor" content="#da532c" />
-      <meta name="msapplication-config" content="/favicons/browserconfig.xml" />
       <meta name="theme-color" content="#ffffff" />
     </Head>
   );

app/src/components/InputDropdown.tsx

@@ -0,0 +1,80 @@
+import {
+  Input,
+  InputGroup,
+  InputRightElement,
+  Icon,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  VStack,
+  HStack,
+  Button,
+  Text,
+  useDisclosure,
+} from "@chakra-ui/react";
+
+import { FiChevronDown } from "react-icons/fi";
+import { BiCheck } from "react-icons/bi";
+
+type InputDropdownProps<T> = {
+  options: ReadonlyArray<T>;
+  selectedOption: T;
+  onSelect: (option: T) => void;
+};
+
+const InputDropdown = <T,>({ options, selectedOption, onSelect }: InputDropdownProps<T>) => {
+  const popover = useDisclosure();
+
+  return (
+    <Popover placement="bottom-start" {...popover}>
+      <PopoverTrigger>
+        <InputGroup cursor="pointer" w={(selectedOption as string).length * 14 + 180}>
+          <Input
+            value={selectedOption as string}
+            // eslint-disable-next-line @typescript-eslint/no-empty-function -- controlled input requires onChange
+            onChange={() => {}}
+            cursor="pointer"
+            borderColor={popover.isOpen ? "blue.500" : undefined}
+            _hover={popover.isOpen ? { borderColor: "blue.500" } : undefined}
+            contentEditable={false}
+            // disable focus
+            onFocus={(e) => {
+              e.target.blur();
+            }}
+          />
+          <InputRightElement>
+            <Icon as={FiChevronDown} />
+          </InputRightElement>
+        </InputGroup>
+      </PopoverTrigger>
+      <PopoverContent boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);" minW={0} w="auto">
+        <VStack spacing={0}>
+          {options?.map((option, index) => (
+            <HStack
+              key={index}
+              as={Button}
+              onClick={() => {
+                onSelect(option);
+                popover.onClose();
+              }}
+              w="full"
+              variant="ghost"
+              justifyContent="space-between"
+              fontWeight="semibold"
+              borderRadius={0}
+              colorScheme="blue"
+              color="black"
+              fontSize="sm"
+              borderBottomWidth={1}
+            >
+              <Text mr={16}>{option as string}</Text>
+              {option === selectedOption && <Icon as={BiCheck} color="blue.500" boxSize={5} />}
+            </HStack>
+          ))}
+        </VStack>
+      </PopoverContent>
+    </Popover>
+  );
+};
+
+export default InputDropdown;

app/src/components/OutputsTable/AddVariantButton.tsx

@@ -33,25 +33,11 @@ export default function AddVariantButton() {
     <Flex w="100%" justifyContent="flex-end">
       <ActionButton
         onClick={onClick}
-        py={5}
+        py={7}
         leftIcon={<Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />}
       >
         <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
       </ActionButton>
-      {/* <Button
-        alignItems="center"
-        justifyContent="center"
-        fontWeight="normal"
-        bgColor="transparent"
-        _hover={{ bgColor: "gray.100" }}
-        px={cellPadding.x}
-        onClick={onClick}
-        height="unset"
-        minH={headerMinHeight}
-      >
-        <Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />
-        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
-      </Button> */}
     </Flex>
   );
 }

app/src/components/OutputsTable/EditEvaluations.tsx

@@ -12,6 +12,7 @@ import {
   Select,
   FormHelperText,
   Code,
+  IconButton,
 } from "@chakra-ui/react";
 import { type Evaluation, EvalType } from "@prisma/client";
 import { useCallback, useState } from "react";
@@ -183,46 +184,37 @@ export default function EditEvaluations() {
                 <Text flex={1}>
                   {evaluation.evalType}: &quot;{evaluation.value}&quot;
                 </Text>
-                <Button
+
+                <IconButton
+                  aria-label="Edit"
                   variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                   minW="unset"
+                  color="gray.400"
                   onClick={() => setEditingId(evaluation.id)}
-                  _hover={{
-                    color: "gray.800",
-                    cursor: "pointer",
-                  }}
-                >
-                  <Icon as={BsPencil} boxSize={4} />
-                </Button>
-                <Button
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
+                  icon={<Icon as={BsPencil} />}
+                />
+                <IconButton
+                  aria-label="Delete"
                   variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                   minW="unset"
+                  color="gray.400"
                   onClick={() => onDelete(evaluation.id)}
-                  _hover={{
-                    color: "gray.800",
-                    cursor: "pointer",
-                  }}
-                >
-                  <Icon as={BsX} boxSize={6} />
-                </Button>
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
+                  icon={<Icon as={BsX} boxSize={6} />}
+                />
               </HStack>
             ),
           )}
           {editingId == null && (
             <Button
               onClick={() => setEditingId("new")}
-              alignSelf="flex-start"
+              alignSelf="end"
               size="sm"
               mt={4}
               colorScheme="blue"
             >
-              Add Evaluation
+              New Evaluation
             </Button>
           )}
           {editingId == "new" && (

app/src/components/OutputsTable/EditScenarioVars.tsx

@@ -0,0 +1,185 @@
+import { Text, Button, HStack, Heading, Icon, IconButton, Stack, VStack } from "@chakra-ui/react";
+import { type TemplateVariable } from "@prisma/client";
+import { useEffect, useState } from "react";
+import { BsPencil, BsX } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+
+export const ScenarioVar = ({
+  variable,
+  isEditing,
+  setIsEditing,
+}: {
+  variable: Pick<TemplateVariable, "id" | "label">;
+  isEditing: boolean;
+  setIsEditing: (isEditing: boolean) => void;
+}) => {
+  const utils = api.useContext();
+
+  const [label, setLabel] = useState(variable.label);
+
+  useEffect(() => {
+    setLabel(variable.label);
+  }, [variable.label]);
+
+  const renameVarMutation = api.scenarioVars.rename.useMutation();
+  const [onRename] = useHandledAsyncCallback(async () => {
+    const resp = await renameVarMutation.mutateAsync({ id: variable.id, label });
+    if (maybeReportError(resp)) return;
+
+    setIsEditing(false);
+    await utils.scenarioVars.list.invalidate();
+    await utils.scenarios.list.invalidate();
+  }, [label, variable.id]);
+
+  const deleteMutation = api.scenarioVars.delete.useMutation();
+  const [onDeleteVar] = useHandledAsyncCallback(async () => {
+    await deleteMutation.mutateAsync({ id: variable.id });
+    await utils.scenarioVars.list.invalidate();
+  }, [variable.id]);
+
+  if (isEditing) {
+    return (
+      <HStack w="full">
+        <FloatingLabelInput
+          flex={1}
+          label="Renamed Variable"
+          value={label}
+          onChange={(e) => setLabel(e.target.value)}
+          onKeyDown={(e) => {
+            if (e.key === "Enter") {
+              e.preventDefault();
+              onRename();
+            }
+            // If the user types a space, replace it with an underscore
+            if (e.key === " ") {
+              e.preventDefault();
+              setLabel((label) => label && `${label}_`);
+            }
+          }}
+        />
+        <Button size="sm" onClick={() => setIsEditing(false)}>
+          Cancel
+        </Button>
+        <Button size="sm" colorScheme="blue" onClick={onRename}>
+          Save
+        </Button>
+      </HStack>
+    );
+  } else {
+    return (
+      <HStack w="full" borderTopWidth={1} borderColor="gray.200">
+        <Text flex={1}>{variable.label}</Text>
+        <IconButton
+          aria-label="Edit"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={() => setIsEditing(true)}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsPencil} />}
+        />
+        <IconButton
+          aria-label="Delete"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={onDeleteVar}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsX} boxSize={6} />}
+        />
+      </HStack>
+    );
+  }
+};
+
+export default function EditScenarioVars() {
+  const experiment = useExperiment();
+  const vars = useScenarioVars();
+
+  const [currentlyEditingId, setCurrentlyEditingId] = useState<string | null>(null);
+
+  const [newVariable, setNewVariable] = useState<string>("");
+  const newVarIsValid = newVariable?.length ?? 0 > 0;
+
+  const utils = api.useContext();
+  const addVarMutation = api.scenarioVars.create.useMutation();
+  const [onAddVar] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    if (!newVariable) return;
+    const resp = await addVarMutation.mutateAsync({
+      experimentId: experiment.data.id,
+      label: newVariable,
+    });
+    if (maybeReportError(resp)) return;
+
+    await utils.scenarioVars.list.invalidate();
+    setNewVariable("");
+  }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
+
+  return (
+    <Stack>
+      <Heading size="sm">Scenario Variables</Heading>
+      <VStack spacing={4}>
+        <Text fontSize="sm">
+          Scenario variables can be used in your prompt variants as well as evaluations.
+        </Text>
+        <VStack spacing={0} w="full">
+          {vars.data?.map((variable) => (
+            <ScenarioVar
+              variable={variable}
+              key={variable.id}
+              isEditing={currentlyEditingId === variable.id}
+              setIsEditing={(isEditing) => {
+                if (isEditing) {
+                  setCurrentlyEditingId(variable.id);
+                } else {
+                  setCurrentlyEditingId(null);
+                }
+              }}
+            />
+          ))}
+        </VStack>
+        {currentlyEditingId !== "new" && (
+          <Button
+            colorScheme="blue"
+            size="sm"
+            onClick={() => setCurrentlyEditingId("new")}
+            alignSelf="end"
+          >
+            New Variable
+          </Button>
+        )}
+        {currentlyEditingId === "new" && (
+          <HStack w="full">
+            <FloatingLabelInput
+              flex={1}
+              label="New Variable"
+              value={newVariable}
+              onChange={(e) => setNewVariable(e.target.value)}
+              onKeyDown={(e) => {
+                if (e.key === "Enter") {
+                  e.preventDefault();
+                  onAddVar();
+                }
+                // If the user types a space, replace it with an underscore
+                if (e.key === " ") {
+                  e.preventDefault();
+                  setNewVariable((v) => v && `${v}_`);
+                }
+              }}
+            />
+            <Button size="sm" onClick={() => setCurrentlyEditingId(null)}>
+              Cancel
+            </Button>
+            <Button size="sm" colorScheme="blue" onClick={onAddVar}>
+              Save
+            </Button>
+          </HStack>
+        )}
+      </VStack>
+    </Stack>
+  );
+}

app/src/components/OutputsTable/OutputCell/CellOptions.tsx

@@ -1,7 +1,7 @@
 import { HStack, Icon, IconButton, Spinner, Tooltip, useDisclosure } from "@chakra-ui/react";
 import { BsArrowClockwise, BsInfoCircle } from "react-icons/bs";
 import { useExperimentAccess } from "~/utils/hooks";
-import ExpandedModal from "./PromptModal";
+import PromptModal from "./PromptModal";
 import { type RouterOutputs } from "~/utils/api";
 
 export const CellOptions = ({
@@ -18,13 +18,13 @@ export const CellOptions = ({
   const modalDisclosure = useDisclosure();
 
   return (
-    <HStack justifyContent="flex-end" w="full">
+    <HStack justifyContent="flex-end" w="full" spacing={1}>
       {cell && (
         <>
           <Tooltip label="See Prompt">
             <IconButton
               aria-label="See Prompt"
-              icon={<Icon as={BsInfoCircle} boxSize={4} />}
+              icon={<Icon as={BsInfoCircle} boxSize={3.5} />}
               onClick={modalDisclosure.onOpen}
               size="xs"
               colorScheme="gray"
@@ -32,7 +32,7 @@ export const CellOptions = ({
               variant="ghost"
             />
           </Tooltip>
-          <ExpandedModal cell={cell} disclosure={modalDisclosure} />
+          <PromptModal cell={cell} disclosure={modalDisclosure} />
         </>
       )}
       {canModify && (

app/src/components/OutputsTable/OutputCell/CellWrapper.tsx

@@ -0,0 +1,29 @@
+import { type StackProps, VStack } from "@chakra-ui/react";
+import { type RouterOutputs } from "~/utils/api";
+import { type Scenario } from "../types";
+import { CellOptions } from "./CellOptions";
+import { OutputStats } from "./OutputStats";
+
+const CellWrapper: React.FC<
+  StackProps & {
+    cell: RouterOutputs["scenarioVariantCells"]["get"] | undefined;
+    hardRefetching: boolean;
+    hardRefetch: () => void;
+    mostRecentResponse:
+      | NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>["modelResponses"][0]
+      | undefined;
+    scenario: Scenario;
+  }
+> = ({ children, cell, hardRefetching, hardRefetch, mostRecentResponse, scenario, ...props }) => (
+  <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
+    {cell && (
+      <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
+    )}
+    <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
+      {children}
+    </VStack>
+    {mostRecentResponse && <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />}
+  </VStack>
+);
+
+export default CellWrapper;

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,17 +1,16 @@
-import { api } from "~/utils/api";
-import { type PromptVariant, type Scenario } from "../types";
-import { type StackProps, Text, VStack } from "@chakra-ui/react";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { Text } from "@chakra-ui/react";
+import stringify from "json-stringify-pretty-compact";
+import { Fragment, useEffect, useState, type ReactElement } from "react";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
-import stringify from "json-stringify-pretty-compact";
-import { type ReactElement, useState, useEffect, Fragment, useCallback } from "react";
-import useSocket from "~/utils/useSocket";
-import { OutputStats } from "./OutputStats";
-import { RetryCountdown } from "./RetryCountdown";
 import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import useSocket from "~/utils/useSocket";
+import { type PromptVariant, type Scenario } from "../types";
+import CellWrapper from "./CellWrapper";
 import { ResponseLog } from "./ResponseLog";
-import { CellOptions } from "./TopActions";
+import { RetryCountdown } from "./RetryCountdown";
 
 const WAITING_MESSAGE_INTERVAL = 20000;
 
@@ -23,10 +22,7 @@ export default function OutputCell({
   variant: PromptVariant;
 }): ReactElement | null {
   const utils = api.useContext();
-  const experiment = useExperiment();
-  const vars = api.templateVars.list.useQuery({
-    experimentId: experiment.data?.id ?? "",
-  }).data;
+  const vars = useScenarioVars().data;
 
   const scenarioVariables = scenario.variableValues as Record<string, string>;
   const templateHasVariables =
@@ -36,7 +32,7 @@ export default function OutputCell({
 
   if (!templateHasVariables) disabledReason = "Add a value to the scenario variables to see output";
 
-  const [refetchInterval, setRefetchInterval] = useState(0);
+  const [refetchInterval, setRefetchInterval] = useState<number | false>(false);
   const { data: cell, isLoading: queryLoading } = api.scenarioVariantCells.get.useQuery(
     { scenarioId: scenario.id, variantId: variant.id },
     { refetchInterval },
@@ -67,42 +63,34 @@ export default function OutputCell({
     cell.retrievalStatus === "PENDING" ||
     cell.retrievalStatus === "IN_PROGRESS" ||
     hardRefetching;
-  useEffect(() => setRefetchInterval(awaitingOutput ? 1000 : 0), [awaitingOutput]);
+
+  useEffect(() => setRefetchInterval(awaitingOutput ? 1000 : false), [awaitingOutput]);
 
   // TODO: disconnect from socket if we're not streaming anymore
   const streamedMessage = useSocket<OutputSchema>(cell?.id);
 
   const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
 
-  const CellWrapper = useCallback(
-    ({ children, ...props }: StackProps) => (
-      <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
-        {cell && (
-          <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
-        )}
-        <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
-          {children}
-        </VStack>
-        {mostRecentResponse && (
-          <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
-        )}
-      </VStack>
-    ),
-    [hardRefetching, hardRefetch, mostRecentResponse, scenario, cell],
-  );
+  const wrapperProps: Parameters<typeof CellWrapper>[0] = {
+    cell,
+    hardRefetching,
+    hardRefetch,
+    mostRecentResponse,
+    scenario,
+  };
 
   if (!vars) return null;
 
   if (!cell && !fetchingOutput)
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <Text color="gray.500">Error retrieving output</Text>
       </CellWrapper>
     );
 
   if (cell && cell.errorMessage) {
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <Text color="red.500">{cell.errorMessage}</Text>
       </CellWrapper>
     );
@@ -110,11 +98,16 @@ export default function OutputCell({
 
   if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
 
-  const showLogs = !streamedMessage && !mostRecentResponse?.output;
+  const showLogs = !streamedMessage && !mostRecentResponse?.respPayload;
 
   if (showLogs)
     return (
-      <CellWrapper alignItems="flex-start" fontFamily="inconsolata, monospace" spacing={0}>
+      <CellWrapper
+        {...wrapperProps}
+        alignItems="flex-start"
+        fontFamily="inconsolata, monospace"
+        spacing={0}
+      >
         {cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
         {cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
         {cell?.modelResponses?.map((response) => {
@@ -123,8 +116,13 @@ export default function OutputCell({
             ? response.receivedAt.getTime()
             : Date.now();
           if (response.requestedAt) {
-            numWaitingMessages = Math.floor(
-              (relativeWaitingTime - response.requestedAt.getTime()) / WAITING_MESSAGE_INTERVAL,
+            numWaitingMessages = Math.min(
+              Math.floor(
+                (relativeWaitingTime - response.requestedAt.getTime()) / WAITING_MESSAGE_INTERVAL,
+              ),
+              // Don't try to render more than 15, it'll use too much CPU and
+              // break the page
+              15,
             );
           }
           return (
@@ -163,15 +161,15 @@ export default function OutputCell({
       </CellWrapper>
     );
 
-  const normalizedOutput = mostRecentResponse?.output
-    ? provider.normalizeOutput(mostRecentResponse?.output)
+  const normalizedOutput = mostRecentResponse?.respPayload
+    ? provider.normalizeOutput(mostRecentResponse?.respPayload)
     : streamedMessage
     ? provider.normalizeOutput(streamedMessage)
     : null;
 
-  if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
+  if (mostRecentResponse?.respPayload && normalizedOutput?.type === "json") {
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <SyntaxHighlighter
           customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
           language="json"
@@ -190,8 +188,8 @@ export default function OutputCell({
   const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
 
   return (
-    <CellWrapper>
-      <Text>{contentToDisplay}</Text>
+    <CellWrapper {...wrapperProps}>
+      <Text whiteSpace="pre-wrap">{contentToDisplay}</Text>
     </CellWrapper>
   );
 }

app/src/components/OutputsTable/OutputCell/OutputStats.tsx

@@ -19,8 +19,8 @@ export const OutputStats = ({
       ? modelResponse.receivedAt.getTime() - modelResponse.requestedAt.getTime()
       : 0;
 
-  const promptTokens = modelResponse.promptTokens;
-  const completionTokens = modelResponse.completionTokens;
+  const inputTokens = modelResponse.inputTokens;
+  const outputTokens = modelResponse.outputTokens;
 
   return (
     <HStack
@@ -55,8 +55,8 @@ export const OutputStats = ({
       </HStack>
       {modelResponse.cost && (
         <CostTooltip
-          promptTokens={promptTokens}
-          completionTokens={completionTokens}
+          inputTokens={inputTokens}
+          outputTokens={outputTokens}
           cost={modelResponse.cost}
         >
           <HStack spacing={0}>

app/src/components/OutputsTable/OutputCell/PromptModal.tsx

@@ -0,0 +1,109 @@
+import {
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalHeader,
+  ModalOverlay,
+  VStack,
+  Text,
+  Box,
+  type UseDisclosureReturn,
+  Link,
+} from "@chakra-ui/react";
+import { api, type RouterOutputs } from "~/utils/api";
+import { JSONTree } from "react-json-tree";
+import CopiableCode from "~/components/CopiableCode";
+
+const theme = {
+  scheme: "chalk",
+  author: "chris kempson (http://chriskempson.com)",
+  base00: "transparent",
+  base01: "#202020",
+  base02: "#303030",
+  base03: "#505050",
+  base04: "#b0b0b0",
+  base05: "#d0d0d0",
+  base06: "#e0e0e0",
+  base07: "#f5f5f5",
+  base08: "#fb9fb1",
+  base09: "#eda987",
+  base0A: "#ddb26f",
+  base0B: "#acc267",
+  base0C: "#12cfc0",
+  base0D: "#6fc2ef",
+  base0E: "#e1a3ee",
+  base0F: "#deaf8f",
+};
+
+export default function PromptModal(props: {
+  cell: NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>;
+  disclosure: UseDisclosureReturn;
+}) {
+  const { data } = api.scenarioVariantCells.getTemplatedPromptMessage.useQuery(
+    {
+      cellId: props.cell.id,
+    },
+    {
+      enabled: props.disclosure.isOpen,
+    },
+  );
+
+  return (
+    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="xl">
+      <ModalOverlay />
+      <ModalContent>
+        <ModalHeader>Prompt Details</ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          <VStack py={4} w="">
+            <VStack w="full" alignItems="flex-start">
+              <Text fontWeight="bold">Full Prompt</Text>
+              <Box
+                w="full"
+                p={4}
+                alignItems="flex-start"
+                backgroundColor="blackAlpha.800"
+                borderRadius={4}
+              >
+                <JSONTree
+                  data={props.cell.prompt}
+                  theme={theme}
+                  shouldExpandNodeInitially={() => true}
+                  getItemString={() => ""}
+                  hideRoot
+                />
+              </Box>
+            </VStack>
+            {data?.templatedPrompt && (
+              <VStack w="full" mt={4} alignItems="flex-start">
+                <Text fontWeight="bold">Templated prompt message:</Text>
+                <CopiableCode
+                  w="full"
+                  // bgColor="gray.100"
+                  p={4}
+                  borderWidth={1}
+                  whiteSpace="pre-wrap"
+                  code={data.templatedPrompt}
+                />
+              </VStack>
+            )}
+            {data?.learnMoreUrl && (
+              <Link
+                href={data.learnMoreUrl}
+                isExternal
+                color="blue.500"
+                fontWeight="bold"
+                fontSize="sm"
+                mt={4}
+                alignSelf="flex-end"
+              >
+                Learn More
+              </Link>
+            )}
+          </VStack>
+        </ModalBody>
+      </ModalContent>
+    </Modal>
+  );
+}

app/src/components/OutputsTable/ScenarioEditor.tsx

@@ -1,7 +1,7 @@
 import { isEqual } from "lodash-es";
 import { useEffect, useState, type DragEvent } from "react";
 import { api } from "~/utils/api";
-import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperimentAccess, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
 import { type Scenario } from "./types";
 
 import {
@@ -41,8 +41,7 @@ export default function ScenarioEditor({
     if (savedValues) setValues(savedValues);
   }, [savedValues]);
 
-  const experiment = useExperiment();
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = useScenarioVars();
 
   const variableLabels = vars.data?.map((v) => v.label) ?? [];
 

app/src/components/OutputsTable/ScenarioEditorModal.tsx

@@ -58,7 +58,7 @@ export const ScenarioEditorModal = ({
     await utils.scenarios.list.invalidate();
   }, [mutation, values]);
 
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = api.scenarioVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
   const variableLabels = vars.data?.map((v) => v.label) ?? [];
 
   return (

app/src/components/OutputsTable/ScenarioPaginator.tsx

@@ -0,0 +1,16 @@
+import { type StackProps } from "@chakra-ui/react";
+
+import { useScenarios } from "~/utils/hooks";
+import Paginator from "../Paginator";
+
+const ScenarioPaginator = (props: StackProps) => {
+  const { data } = useScenarios();
+
+  if (!data) return null;
+
+  const { count } = data;
+
+  return <Paginator count={count} condense {...props} />;
+};
+
+export default ScenarioPaginator;

app/src/components/OutputsTable/ScenarioRow.tsx

@@ -10,6 +10,8 @@ const ScenarioRow = (props: {
   variants: PromptVariant[];
   canHide: boolean;
   rowStart: number;
+  isFirst: boolean;
+  isLast: boolean;
 }) => {
   const [isHovered, setIsHovered] = useState(false);
 
@@ -21,10 +23,14 @@ const ScenarioRow = (props: {
         onMouseEnter={() => setIsHovered(true)}
         onMouseLeave={() => setIsHovered(false)}
         sx={isHovered ? highlightStyle : undefined}
-        borderLeftWidth={1}
-        {...borders}
+        bgColor="white"
         rowStart={props.rowStart}
         colStart={1}
+        borderLeftWidth={1}
+        borderTopWidth={props.isFirst ? 1 : 0}
+        borderTopLeftRadius={props.isFirst ? 8 : 0}
+        borderBottomLeftRadius={props.isLast ? 8 : 0}
+        {...borders}
       >
         <ScenarioEditor scenario={props.scenario} hovered={isHovered} canHide={props.canHide} />
       </GridItem>
@@ -34,8 +40,12 @@ const ScenarioRow = (props: {
           onMouseEnter={() => setIsHovered(true)}
           onMouseLeave={() => setIsHovered(false)}
           sx={isHovered ? highlightStyle : undefined}
+          bgColor="white"
           rowStart={props.rowStart}
           colStart={i + 2}
+          borderTopWidth={props.isFirst ? 1 : 0}
+          borderTopRightRadius={props.isFirst && i === props.variants.length - 1 ? 8 : 0}
+          borderBottomRightRadius={props.isLast && i === props.variants.length - 1 ? 8 : 0}
           {...borders}
         >
           <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -48,7 +48,7 @@ export const ScenariosHeader = () => {
   );
 
   return (
-    <HStack w="100%" pb={cellPadding.y} pt={0} align="center" spacing={0}>
+    <HStack w="100%" py={cellPadding.y} px={cellPadding.x} align="center" spacing={0}>
       <Text fontSize={16} fontWeight="bold">
         Scenarios ({scenarios.data?.count})
       </Text>
@@ -57,11 +57,16 @@ export const ScenariosHeader = () => {
           <MenuButton
             as={IconButton}
             mt={1}
+            ml={2}
             variant="ghost"
             aria-label="Edit Scenarios"
             icon={<Icon as={loading ? Spinner : BsGear} />}
+            maxW={8}
+            minW={8}
+            minH={8}
+            maxH={8}
           />
-          <MenuList fontSize="md" zIndex="dropdown" mt={-3}>
+          <MenuList fontSize="md" zIndex="dropdown" mt={-1}>
             <MenuItem
               icon={<Icon as={BsPlus} boxSize={6} mx="-5px" />}
               onClick={() => onAddScenario(false)}
@@ -72,7 +77,7 @@ export const ScenariosHeader = () => {
               Autogenerate Scenario
             </MenuItem>
             <MenuItem icon={<BsPencil />} onClick={openDrawer}>
-              Edit Vars
+              Add or Remove Variables
             </MenuItem>
           </MenuList>
         </Menu>

app/src/components/OutputsTable/VariantStats.tsx

@@ -17,18 +17,22 @@ export default function VariantStats(props: { variant: PromptVariant }) {
       initialData: {
         evalResults: [],
         overallCost: 0,
-        promptTokens: 0,
-        completionTokens: 0,
+        inputTokens: 0,
+        outputTokens: 0,
         scenarioCount: 0,
         outputCount: 0,
+        awaitingCompletions: false,
         awaitingEvals: false,
       },
       refetchInterval,
     },
   );
 
-  // Poll every two seconds while we are waiting for LLM retrievals to finish
-  useEffect(() => setRefetchInterval(data.awaitingEvals ? 5000 : 0), [data.awaitingEvals]);
+  // Poll every five seconds while we are waiting for LLM retrievals to finish
+  useEffect(
+    () => setRefetchInterval(data.awaitingCompletions || data.awaitingEvals ? 5000 : 0),
+    [data.awaitingCompletions, data.awaitingEvals],
+  );
 
   const [passColor, neutralColor, failColor] = useToken("colors", [
     "green.500",
@@ -68,8 +72,8 @@ export default function VariantStats(props: { variant: PromptVariant }) {
       </HStack>
       {data.overallCost && (
         <CostTooltip
-          promptTokens={data.promptTokens}
-          completionTokens={data.completionTokens}
+          inputTokens={data.inputTokens}
+          outputTokens={data.outputTokens}
           cost={data.overallCost}
         >
           <HStack spacing={0} align="center" color="gray.500">