Number experiments based only on current org

Previously we were naming each new experiment based on the highest existing sort index globally, which doesn't make sense. Better to just use the local one.

.env.example

@@ -17,4 +17,17 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/openpipe?schema=publ
 # https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key
 OPENAI_API_KEY=""
 
+# Replicate API token. Create a token here: https://replicate.com/account/api-tokens
+REPLICATE_API_TOKEN=""
+
 NEXT_PUBLIC_SOCKET_URL="http://localhost:3318"
+
+# Next Auth
+NEXTAUTH_SECRET="your_secret"
+NEXTAUTH_URL="http://localhost:3000"
+
+NEXT_PUBLIC_HOST="http://localhost:3000"
+
+# Next Auth Github Provider
+GITHUB_CLIENT_ID="your_client_id"
+GITHUB_CLIENT_SECRET="your_secret"

.eslintrc.cjs

@@ -37,6 +37,7 @@ const config = {
       "warn",
       { vars: "all", varsIgnorePattern: "^_", args: "after-used", argsIgnorePattern: "^_" },
     ],
+    "react/no-unescaped-entities": "off",
   },
 };
 

.github/workflows/ci.yaml

@@ -0,0 +1,53 @@
+name: CI checks
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+jobs:
+  run-checks:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: "20"
+
+      - uses: pnpm/action-setup@v2
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          version: 8.6.1
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install Dependencies
+        run: pnpm install
+
+      - name: Check types
+        run: pnpm tsc
+
+      - name: Lint
+        run: SKIP_ENV_VALIDATION=1 pnpm lint
+
+      - name: Check prettier
+        run: pnpm prettier . --check

.gitignore

@@ -40,3 +40,6 @@ yarn-error.log*
 
 # typescript
 *.tsbuildinfo
+
+# Sentry Auth Token
+.sentryclirc

.prettierignore

@@ -1,2 +1,2 @@
-src/codegen/openai.schema.json
+*.schema.json
 pnpm-lock.yaml

.tool-versions

@@ -0,0 +1 @@
+nodejs 20.2.0

.vscode/settings.json

@@ -1,6 +1,3 @@
 {
-  "eslint.format.enable": true,
+  "eslint.format.enable": true
-  "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
-  }
 }

@types/nextjs-routes.d.ts

@@ -11,11 +11,19 @@ declare module "nextjs-routes" {
   } from "next";
 
   export type Route =
+    | StaticRoute<"/account/signin">
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
+    | StaticRoute<"/api/experiments/og-image">
+    | StaticRoute<"/api/sentry-example-api">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
+    | DynamicRoute<"/data/[id]", { "id": string }>
+    | StaticRoute<"/data">
     | DynamicRoute<"/experiments/[id]", { "id": string }>
     | StaticRoute<"/experiments">
-    | StaticRoute<"/">;
+    | StaticRoute<"/">
+    | StaticRoute<"/sentry-example-page">
+    | StaticRoute<"/world-champs">
+    | StaticRoute<"/world-champs/signup">;
 
   interface StaticRoute<Pathname> {
     pathname: Pathname;

Dockerfile

@@ -19,8 +19,10 @@ FROM base as builder
 
 # Include all NEXT_PUBLIC_* env vars here
 ARG NEXT_PUBLIC_POSTHOG_KEY
-ARG NEXT_PUBLIC_IS_PUBLIC_PLAYGROUND
 ARG NEXT_PUBLIC_SOCKET_URL
+ARG NEXT_PUBLIC_HOST
+ARG NEXT_PUBLIC_SENTRY_DSN
+ARG SENTRY_AUTH_TOKEN
 
 WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules

README.md

@@ -1,42 +1,61 @@
-<img src="https://github.com/openpipe/openpipe/assets/41524992/ca59596e-eb80-40f9-921f-6d67f6e6d8fa" width="72px" />
+<!-- <img src="https://github.com/openpipe/openpipe/assets/41524992/ca59596e-eb80-40f9-921f-6d67f6e6d8fa" width="72px" /> -->
 
 # OpenPipe
 
-OpenPipe is a flexible playground for comparing and optimizing LLM prompts. It lets you quickly generate, test and compare candidate prompts with realistic sample data.
+OpenPipe is a flexible playground for comparing and optimizing LLM prompts. It lets you quickly generate, test and compare candidate prompts, and can automatically [translate](#-translate-between-model-apis) those prompts between models.
 
-**Live Demo:** https://openpipe.ai
+<img src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="demo">
 
-<img src="https://github.com/openpipe/openpipe/assets/176426/fc7624c6-5b65-4d4d-82b7-4a816f3e5678" alt="demo" height="400px">
+You can use our hosted version of OpenPipe at https://openpipe.ai. You can also clone this repository and [run it locally](#running-locally).
 
-Currently there's a public playground available at [https://openpipe.ai/](https://openpipe.ai/), but the recommended approach is to [run locally](#running-locally).
+## Sample Experiments
 
-## High-Level Features
+These are simple experiments users have created that show how OpenPipe works. Feel free to fork them and start experimenting yourself.
 
-**Configure Multiple Prompts**  
+- [Twitter Sentiment Analysis](https://app.openpipe.ai/experiments/62c20a73-2012-4a64-973c-4b665ad46a57)
-Set up multiple prompt configurations and compare their output side-by-side. Each configuration can be configured independently.
+- [Reddit User Needs](https://app.openpipe.ai/experiments/22222222-2222-2222-2222-222222222222)
-
+- [OpenAI Function Calls](https://app.openpipe.ai/experiments/2ebbdcb3-ed51-456e-87dc-91f72eaf3e2b)
-**Visualize Responses**  
+- [Activity Classification](https://app.openpipe.ai/experiments/3950940f-ab6b-4b74-841d-7e9dbc4e4ff8)
-Inspect prompt completions side-by-side.
-
-**Test Many Inputs**  
-OpenPipe lets you _template_ a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broader coverage of your problem space than you'd get with manual testing.
-
-**🪄 Auto-generate Test Scenarios**  
-OpenPipe includes a tool to generate new test scenarios based on your existing prompts and scenarios. Just click "Autogenerate Scenario" to try it out!
-
-**Prompt Validation and Typeahead**  
-We use OpenAI's OpenAPI spec to automatically provide typeahead and validate prompts.
-
-<img alt="typeahead" src="https://github.com/openpipe/openpipe/assets/176426/acc638f8-d851-4742-8d01-fe6f98890840" height="300px">
-
-**Function Call Support**  
-Natively supports [OpenAI function calls](https://openai.com/blog/function-calling-and-other-api-updates) on supported models.
-
-<img height="300px" alt="function calls" src="https://github.com/openpipe/openpipe/assets/176426/48ad13fe-af2f-4294-bf32-62015597fd9b">
 
 ## Supported Models
 
-OpenPipe currently supports GPT-3.5 and GPT-4. Wider model support is planned.
+- All models available through the OpenAI [chat completion API](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
+- Llama2 [7b chat](https://replicate.com/a16z-infra/llama7b-v2-chat), [13b chat](https://replicate.com/a16z-infra/llama13b-v2-chat), [70b chat](https://replicate.com/replicate/llama70b-v2-chat).
+- Anthropic's [Claude 1 Instant](https://www.anthropic.com/index/introducing-claude) and [Claude 2](https://www.anthropic.com/index/claude-2)
+
+## Features
+
+### 🔍 Visualize Responses
+
+Inspect prompt completions side-by-side.
+
+### 🧪 Bulk-Test
+
+OpenPipe lets you _template_ a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broad coverage of your problem space.
+
+### 📟 Translate between Model APIs
+
+Write your prompt in one format and automatically convert it to work with any other model.
+
+<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/1e19ccf2-96b6-4e93-a3a5-1449710d1b5b" alt="translate between models">
+
+<br><br>
+
+### 🛠️ Refine Your Prompts Automatically
+
+Use a growing database of best-practice refinements to improve your prompts automatically.
+
+<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/87a27fe7-daef-445c-a5e2-1c82b23f9f99" alt="add function call">
+
+<br><br>
+
+### 🪄 Auto-generate Test Scenarios
+
+OpenPipe includes a tool to generate new test scenarios based on your existing prompts and scenarios. Just click "Autogenerate Scenario" to try it out!
+
+<img width="600" src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="auto-generate">
+
+<br><br>
 
 ## Running Locally
 
@@ -47,5 +66,6 @@ OpenPipe currently supports GPT-3.5 and GPT-4. Wider model support is planned.
 5. Install the dependencies: `cd openpipe && pnpm install`
 6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`.
 7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma db push` to create the database.
-8. Start the app: `pnpm dev`.
+8. Create a [GitHub OAuth App](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app) and update the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` values. (Note: a PR to make auth optional when running locally would be a great contribution!)
-9. Navigate to [http://localhost:3000](http://localhost:3000)
+9. Start the app: `pnpm dev`.
+10. Navigate to [http://localhost:3000](http://localhost:3000)

next.config.mjs

@@ -1,13 +1,14 @@
 import nextRoutes from "nextjs-routes/config";
+import { withSentryConfig } from "@sentry/nextjs";
 
 /**
  * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation. This is especially useful
  * for Docker builds.
  */
-await import("./src/env.mjs");
+const { env } = await import("./src/env.mjs");
 
 /** @type {import("next").NextConfig} */
-const config = {
+let config = {
   reactStrictMode: true,
 
   /**
@@ -21,6 +22,13 @@ const config = {
     defaultLocale: "en",
   },
 
+  rewrites: async () => [
+    {
+      source: "/ingest/:path*",
+      destination: "https://app.posthog.com/:path*",
+    },
+  ],
+
   webpack: (config) => {
     config.module.rules.push({
       test: /\.txt$/,
@@ -30,4 +38,24 @@ const config = {
   },
 };
 
-export default nextRoutes()(config);
+config = nextRoutes()(config);
+
+if (env.NEXT_PUBLIC_SENTRY_DSN && env.SENTRY_AUTH_TOKEN) {
+  // @ts-expect-error - `withSentryConfig` is not typed correctly
+  config = withSentryConfig(
+    config,
+    {
+      authToken: env.SENTRY_AUTH_TOKEN,
+      silent: true,
+      org: "openpipe",
+      project: "openpipe",
+    },
+    {
+      widenClientFileUpload: true,
+      tunnelRoute: "/monitoring",
+      disableLogger: true,
+    },
+  );
+}
+
+export default config;

package.json

@@ -3,25 +3,41 @@
   "type": "module",
   "version": "0.1.0",
   "license": "Apache-2.0",
+  "engines": {
+    "node": ">=20.0.0",
+    "pnpm": ">=8.6.1"
+  },
   "scripts": {
     "build": "next build",
     "dev:next": "next dev",
     "dev:wss": "pnpm tsx --watch src/wss-server.ts",
-    "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss'",
+    "dev:worker": "NODE_ENV='development' pnpm tsx --watch src/server/tasks/worker.ts",
+    "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss' 'pnpm dev:worker'",
     "postinstall": "prisma generate",
     "lint": "next lint",
     "start": "next start",
-    "codegen": "tsx src/codegen/export-openai-types.ts"
+    "codegen": "tsx src/codegen/export-openai-types.ts",
+    "seed": "tsx prisma/seed.ts",
+    "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
+    "test": "pnpm vitest --no-threads"
   },
   "dependencies": {
+    "@anthropic-ai/sdk": "^0.5.8",
+    "@apidevtools/json-schema-ref-parser": "^10.1.0",
+    "@babel/preset-typescript": "^7.22.5",
+    "@babel/standalone": "^7.22.9",
+    "@chakra-ui/anatomy": "^2.2.0",
     "@chakra-ui/next-js": "^2.1.4",
     "@chakra-ui/react": "^2.7.1",
+    "@chakra-ui/styled-system": "^2.9.1",
     "@emotion/react": "^11.11.1",
     "@emotion/server": "^11.11.0",
     "@emotion/styled": "^11.11.0",
+    "@fontsource/inconsolata": "^5.0.5",
     "@monaco-editor/loader": "^1.3.3",
     "@next-auth/prisma-adapter": "^1.0.5",
     "@prisma/client": "^4.14.0",
+    "@sentry/nextjs": "^7.61.0",
     "@t3-oss/env-nextjs": "^0.3.1",
     "@tabler/icons-react": "^2.22.0",
     "@tanstack/react-query": "^4.29.7",
@@ -29,6 +45,8 @@
     "@trpc/next": "^10.26.0",
     "@trpc/react-query": "^10.26.0",
     "@trpc/server": "^10.26.0",
+    "@vercel/og": "^0.5.9",
+    "ast-types": "^0.14.2",
     "chroma-js": "^2.4.2",
     "concurrently": "^8.2.0",
     "cors": "^2.8.5",
@@ -38,43 +56,65 @@
     "express": "^4.18.2",
     "framer-motion": "^10.12.17",
     "gpt-tokens": "^1.0.10",
+    "graphile-worker": "^0.13.0",
     "immer": "^10.0.2",
     "isolated-vm": "^4.5.0",
+    "json-schema-to-typescript": "^13.0.2",
     "json-stringify-pretty-compact": "^4.0.0",
-    "lodash": "^4.17.21",
+    "jsonschema": "^1.4.1",
+    "lodash-es": "^4.17.21",
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
+    "next-query-params": "^4.2.3",
     "nextjs-routes": "^2.0.1",
-    "openai": "4.0.0-beta.2",
+    "openai": "4.0.0-beta.7",
     "pluralize": "^8.0.0",
-    "posthog-js": "^1.68.4",
+    "posthog-js": "^1.75.3",
+    "posthog-node": "^3.1.1",
     "prettier": "^3.0.0",
+    "prismjs": "^1.29.0",
     "react": "18.2.0",
+    "react-diff-viewer": "^3.1.1",
     "react-dom": "18.2.0",
+    "react-github-btn": "^1.4.0",
     "react-icons": "^4.10.1",
+    "react-json-tree": "^0.18.0",
+    "react-select": "^5.7.4",
     "react-syntax-highlighter": "^15.5.0",
     "react-textarea-autosize": "^8.5.0",
+    "recast": "^0.23.3",
+    "replicate": "^0.12.3",
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
     "superjson": "1.12.2",
     "tsx": "^3.12.7",
+    "type-fest": "^4.0.0",
+    "use-query-params": "^2.2.1",
+    "uuid": "^9.0.0",
+    "vite-tsconfig-paths": "^4.2.0",
     "zod": "^3.21.4",
     "zustand": "^4.3.9"
   },
   "devDependencies": {
     "@openapi-contrib/openapi-schema-to-json-schema": "^4.0.5",
+    "@types/babel__core": "^7.20.1",
+    "@types/babel__standalone": "^7.1.4",
     "@types/chroma-js": "^2.4.0",
     "@types/cors": "^2.8.13",
     "@types/eslint": "^8.37.0",
     "@types/express": "^4.17.17",
-    "@types/lodash": "^4.14.195",
+    "@types/json-schema": "^7.0.12",
+    "@types/lodash-es": "^4.17.8",
     "@types/node": "^18.16.0",
     "@types/pluralize": "^0.0.30",
+    "@types/prismjs": "^1.26.0",
     "@types/react": "^18.2.6",
     "@types/react-dom": "^18.2.4",
     "@types/react-syntax-highlighter": "^15.5.7",
+    "@types/uuid": "^9.0.2",
     "@typescript-eslint/eslint-plugin": "^5.59.6",
     "@typescript-eslint/parser": "^5.59.6",
+    "csv-parse": "^5.4.0",
     "eslint": "^8.40.0",
     "eslint-config-next": "^13.4.2",
     "eslint-plugin-unused-imports": "^2.0.0",
@@ -90,6 +130,6 @@
     "initVersion": "7.14.0"
   },
   "prisma": {
-    "seed": "tsx prisma/seed.ts"
+    "seed": "pnpm seed"
   }
 }

prisma/datasets/validated_tweets.csv

@@ -0,0 +1,84 @@
+Text,sentiment,emotion
+@dell your customer service is horrible especially agent syedfaisal who has made this experience of purchasing a new computer downright awful and I’ll reconsider ever buying a Dell in the future @DellTech,negative,anger
+@zacokalo @Dell @DellCares @Dell give the man what he paid for!,neutral,anger
+"COOKING STREAM DAY!!! Ty to @Alienware for sponsoring this stream! I’ll be making a bunch of Japanese Alien themed foods hehe 
+
+Come check it out! https://t.co/m06tJQ06zk 
+
+#alienwarepartner #intelgaming @Dell @IntelGaming https://t.co/qOdQX2E8VD",positive,joy
+@emijuju_ @Alienware @Dell @intel Beautiful 😍❤️😻,positive,joy
+"What's your biggest data management challenge?     • Cloud complexity?   • Lengthy tech refresh cycles?   • Capital budget constraints?      Solve your challenges with as-a-Storage. Get simplicity, agility & control with @Dell #APEX. https://t.co/mCblMtH931 https://t.co/eepKNZ4Ai3",neutral,optimism
+"This week we were at the ""Top Gun"" themed @Dell Product Expo. Eddie Muñoz met Maverick look-alike, California Tom Cruise (Jerome LeBlanc)!
+
+""I feel the need, the need for speed."" - Maverick
+#topgun #topgunmaverick #dell #delltechnologies #lockncharge https://t.co/QHYH2EbMjq",positive,joy
+"Itsss been more than a week...i m following up with dell for troubleshootings...my https://t.co/lWhg2YKhQa suffering so as my hard earned money...hightly disappointed...contd..
+@DellCares @Dell",negative,sadness
+"@ashu_k7 @Dell Pathetic!!!!! I Dont mind taking legal action, this is deficency of service for which the customer is nt getting help..",negative,anger
+@ashu_k7 @Dell Making life unhappy is the new tag line of #Dell,negative,sadness
+"@Dell If you are buying a Dell, make sure you are making your life hell.
+Better buy other laptops. If you wanted to opt for Dell better opt for garbage on the streets.",negative,anger
+"MY DESK'S FINAL FORM? Seriously, I'm finally happy with my monitor setup here... and I'll keep this setup whenever I move... FOREVER. What do you think?
+https://t.co/WJZ2JXtOnX
+@Alienware @Dell cheers. https://t.co/6Whhldfpv0",positive,joy
+"@Dell Dell Alienware computer has had software problems with SupportAssist since purchase.  Dell, despite paying for Premium Support, has never fixed issues.  Latest solution was to erase everything and reload....SupportAssist still doesn't work.",negative,anger
+"HUGE congratulations to Startup Battle 3.0 winner ➡️ @Ox_Fulfillment x @cyborgcharu for being featured in @BusinessInsider & @Dell showcasing the journey at Ox! 🚀🚀🚀
+
+We love to see our portfolio companies continuing to BUILD SOMETHING FROM NOTHING! 🔥 https://t.co/awBkn5ippB",positive,joy
+@Dell happy Friday!,positive,joy
+"@intel Core i5 1135G7 - 4732 points
+@intel Core i5 1235 - 6619 points 
+@Dell Latitude 5420 x 5430.
+Cinebench R23. Good job Intel!",positive,joy
+@Dell india we purchased 52 docking station and we have around 100 users using dell laptop as well as dell monitor now they are refusing to replace my faulty product and disconnecting my every call....,negative,anger
+"It's another year ans another day But cant fill it in yet the child hood dreams.
+It's my birthdy today. Can anyone of you guys bless me with  a simplest gaming oc that can run 
+@DOTA2 ?
+@Dell @HP @VastGG @Acer @Alienware @Lenovo @toshiba @IBM @Fujitsu_Global @NEC https://t.co/69G8tL9sN8",neutral,joy
+"@idoccor @Dell That's always the decision—wait, or, look elsewhere. In this case, I think I unfortunately need to wait since there are only two monitors with these specs and I don't like the other one 😂",negative,sadness
+"@MichaelDell @Dell @DellCares For how long this will continue. It is high time you either fix the problem for good or replace the complete laptop. Spent over 60+ hours with Customer Care teams, which is not helping. Cannot keep going on like this.",negative,anger
+"@Dell @DellCares but no, not really",neutral,sadness
+"Business innovation requires insight, agility and efficiency. How do you get there? RP PRO, LLC recommends starting by proactively managing IT infrastructure with #OpenManage Systems from @Dell. https://t.co/fBcK1lfFMu https://t.co/xWHLkkHCjn",neutral,optimism
+@Dell Yessirrrrr #NationalCoffeeDay,positive,joy
+"New blog post from @Dell shared on https://t.co/EgfPChB8AT 
+
+Re-routing Our Connected and Autonomous Future https://t.co/AW8EHQrbd6
+
+#future #futuretech #techinnovation https://t.co/koX8stKPsr",neutral,joy
+"In a free-market economy, the folks @IronMountain can set prices as they see fit. Their customers are also free to find better prices at competitors like @Dell 
+@H3CGlobal @HPE
+https://t.co/reZ56DNTBI",neutral,optimism
+"Delighted to chat with many of our partners here in person at @Intel Innovation! @Dell, @Lenovo, @Supermicro_SMCI, @QuantaQCT #IntelON https://t.co/BxIeGW8deN",positive,joy
+"A special gracias to our Startup Chica San Antonio 2022 sponsors @eBay, @jcpenney, @Barbie, @HEB, @Dell, @Honda, @SouthsideSATX💜✨ https://t.co/lZ6WWkziHl",positive,joy
+"When your team decides to start supporting developers, your #ops must change too. More from @cote and @Dell Developer Community Manager @barton808: https://t.co/W6f1oMiTgV",neutral,optimism
+@EmDStowers @LASERGIANT1 @ohwormongod @Ludovician_Vega @Dell our boy snitchin,neutral,anger
+A 1st place dmi:Design Value Award goes to @Dell for a packaging modernization initiative that helped them get closer to their corporate Moonshot Sustainability Goal of 100% recycled or renewable packaging by 2030. More at https://t.co/dnhZWWLCQC #designvalue #DVA22,positive,optimism
+Reducing deployment and maintenance complexity is the goal behind @dell and @WindRiver's new collaboration. https://t.co/2PxQgPuHUU,positive,optimism
+@jaserhunter @Dell Love the sales pitch lol,positive,joy
+@Dell india we purchased 52 docking station and we have around 100 users using dell laptop as well as dell monitor now they are refusing to replace my faulty product and disconnecting my every call....,negative,anger
+@ashu_k7 @Dell One more example.. their technical support is also worse. https://t.co/20atSgI4fg,negative,anger
+*angry screeches about @Dell proprietary MBR windows 8.1 partitions not being able to save as an img in clonezilla *,negative,anger
+@socialitebooks @BBYC_Gamers @Dell @Alienware @BestBuyCanada @intelcanada Congratulations!!!,positive,joy
+"Thank you to the @dell team for coming out to volunteer today! We truly appreciate your hard work and look forward to seeing you again soon!
+
+If you and your team are interested in helping out at the UMLAUF, visit our website for more information: https://t.co/lVfsZT2ogS https://t.co/eLz0FY0y4M",positive,joy
+"@TheCaramelGamer @intel @bravadogaming @Intel_Africa @Dell @DellTech @DellTechMEA @Alienware @IntelUK we love to see it. 
+
+Also also actually actually whoever did that artwork? 🔥🔥🔥 am a fan.",positive,joy
+"LOVING MY DELL 2 IN 1 LAPTOP
+YAYY 🥳🥳
+@Dell #DellInspiron #DellLaptop https://t.co/vib96jf3tC",positive,joy
+@Azure @OracleItalia @AWS_Italy @lenovoitalia @Dell discussing the future of #HPC during the #hpcroundtable22 in Turin today #highperformancecomputing https://t.co/jJ1WqBulPF,neutral,joy
+Attracting talent @AmericanChamber. @marg_cola @Dell speaks of quality of life connectivity and the Opportunity for development being so crucial. Housing  availability is now impacting on decision making for potential candidates. #WhyCork,positive,optimism
+.@Dell partners with @WindRiver on modular cloud-native telecommunications infrastructure https://t.co/4SWATspwCP @SiliconANGLE @Mike_Wheatley @holgermu @constellationr,neutral,joy
+@Dell Not buy Dell Inspiron laptop,neutral,sadness
+"@dell #delltechforum reminding us IDC have predicted that by 2024, 50% of everything we consume in technology will be as a service https://t.co/3UBiZJX0LE",neutral,optimism
+@RachMurph @HETTShow @Dell Thank you for coming! Great evening,positive,joy
+Congratulations to Jason M of Moncton NB on winning a @Dell  @Alienware m15 R7 15.6″ gaming laptop from @BestBuyCanada and @intelcanada's gaming days #contest on the blog. Visit https://t.co/VryaY5Rvv9 to learn about tech and for chances to win new tech. https://t.co/T6n0dzF6oL,positive,joy
+@MattVisiwig @Dell Sour taste for sure 😶 But don't let ego distract you from what you really want to buy 😁,neutral,optimism
+"Massive thank you goes to sponsors @HendersonLoggie @lindsaysnews @Dell @unity, all of our fantastic judges and mentors and the team at @EGX and @ExCeLLondon. 
+
+Big congratulations also to all of our other @AbertayDare teams - an amazing year! #Dare2022 https://t.co/jYe4agO7lW",positive,joy
+"@timetcetera @rahaug Nah, I just need @Dell to start paying me comissions 😂",neutral,joy
+"""Whether you’re an engineer, a designer, or work in supply chain management or sales, there are always opportunities to think about sustainability and how you can do things more efficiently."" 👏 — Oliver Campbell, Director of Packaging Engineering, @Dell https://t.co/vUJLTWNFwP https://t.co/GJWAzGfAxJ",positive,optimism
+"Hi, my name is @listerepvp and I support @Dell, always.",positive,joy

prisma/migrations/20230714113205_create_scenariotvariant/migration.sql

@@ -0,0 +1,49 @@
+-- Drop the foreign key constraints on the original ModelOutput
+ALTER TABLE "ModelOutput" DROP CONSTRAINT "ModelOutput_promptVariantId_fkey";
+ALTER TABLE "ModelOutput" DROP CONSTRAINT "ModelOutput_testScenarioId_fkey";
+
+-- Rename the old table
+ALTER TABLE "ModelOutput" RENAME TO "ScenarioVariantCell";
+ALTER TABLE "ScenarioVariantCell" RENAME CONSTRAINT "ModelOutput_pkey" TO "ScenarioVariantCell_pkey";
+ALTER INDEX "ModelOutput_inputHash_idx" RENAME TO "ScenarioVariantCell_inputHash_idx";
+ALTER INDEX "ModelOutput_promptVariantId_testScenarioId_key" RENAME TO "ScenarioVariantCell_promptVariantId_testScenarioId_key";
+
+-- Add the new fields to the renamed table
+ALTER TABLE "ScenarioVariantCell" ADD COLUMN "retryTime" TIMESTAMP(3);
+ALTER TABLE "ScenarioVariantCell" ADD COLUMN "streamingChannel" TEXT;
+ALTER TABLE "ScenarioVariantCell" ALTER COLUMN "inputHash" DROP NOT NULL;
+ALTER TABLE "ScenarioVariantCell" ALTER COLUMN "output" DROP NOT NULL,
+ALTER COLUMN "statusCode" DROP NOT NULL,
+ALTER COLUMN "timeToComplete" DROP NOT NULL;
+
+-- Create the new table
+CREATE TABLE "ModelOutput" (
+    "id" UUID NOT NULL,
+    "inputHash" TEXT NOT NULL,
+    "output" JSONB NOT NULL,
+    "timeToComplete" INTEGER NOT NULL DEFAULT 0,
+    "promptTokens" INTEGER,
+    "completionTokens" INTEGER,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "scenarioVariantCellId" UUID
+);
+
+-- Move inputHash index
+DROP INDEX "ScenarioVariantCell_inputHash_idx";
+CREATE INDEX "ModelOutput_inputHash_idx" ON "ModelOutput"("inputHash");
+
+CREATE UNIQUE INDEX "ModelOutput_scenarioVariantCellId_key" ON "ModelOutput"("scenarioVariantCellId");
+ALTER TABLE "ModelOutput" ADD CONSTRAINT "ModelOutput_scenarioVariantCellId_fkey" FOREIGN KEY ("scenarioVariantCellId") REFERENCES "ScenarioVariantCell"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+ALTER TABLE "ModelOutput" ALTER COLUMN "scenarioVariantCellId" SET NOT NULL,
+ADD CONSTRAINT "ModelOutput_pkey" PRIMARY KEY ("id");
+
+ALTER TABLE "ScenarioVariantCell" ADD CONSTRAINT "ScenarioVariantCell_promptVariantId_fkey" FOREIGN KEY ("promptVariantId") REFERENCES "PromptVariant"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ScenarioVariantCell" ADD CONSTRAINT "ScenarioVariantCell_testScenarioId_fkey" FOREIGN KEY ("testScenarioId") REFERENCES "TestScenario"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- CreateEnum
+CREATE TYPE "CellRetrievalStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETE', 'ERROR');
+
+-- AlterTable
+ALTER TABLE "ScenarioVariantCell" ADD COLUMN     "retrievalStatus" "CellRetrievalStatus" NOT NULL DEFAULT 'COMPLETE';

prisma/migrations/20230714182259_add_model_to_variant/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "PromptVariant" ADD COLUMN     "model" TEXT NOT NULL DEFAULT 'gpt-3.5-turbo';

prisma/migrations/20230715004031_remove_default_value_for_model/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "PromptVariant" ALTER COLUMN "model" DROP DEFAULT;

prisma/migrations/20230717203031_add_gpt4_eval/migration.sql

@@ -0,0 +1,24 @@
+/*
+  Warnings:
+
+  - You are about to rename the column `matchString` on the `Evaluation` table. If there is any code or views referring to the old name, they will break.
+  - You are about to rename the column `matchType` on the `Evaluation` table. If there is any code or views referring to the old name, they will break.
+  - You are about to rename the column `name` on the `Evaluation` table. If there is any code or views referring to the old name, they will break.
+  - You are about to rename the enum `EvaluationMatchType` to `EvalType`. If there is any code or views referring to the old name, they will break.
+*/
+
+-- RenameEnum
+ALTER TYPE "EvaluationMatchType" RENAME TO "EvalType";
+
+-- AlterTable
+ALTER TABLE "Evaluation" RENAME COLUMN "matchString" TO "value";
+ALTER TABLE "Evaluation" RENAME COLUMN "matchType" TO "evalType";
+ALTER TABLE "Evaluation" RENAME COLUMN "name" TO "label";
+
+-- AlterColumnType
+ALTER TABLE "Evaluation" ALTER COLUMN "evalType" TYPE "EvalType" USING "evalType"::text::"EvalType";
+
+-- SetNotNullConstraint
+ALTER TABLE "Evaluation" ALTER COLUMN "evalType" SET NOT NULL;
+ALTER TABLE "Evaluation" ALTER COLUMN "label" SET NOT NULL;
+ALTER TABLE "Evaluation" ALTER COLUMN "value" SET NOT NULL;

prisma/migrations/20230718005301_add_gpt4_eval_type/migration.sql

@@ -0,0 +1,39 @@
+/*
+  Warnings:
+
+  - You are about to drop the `EvaluationResult` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- AlterEnum
+ALTER TYPE "EvalType" ADD VALUE 'GPT4_EVAL';
+
+-- DropForeignKey
+ALTER TABLE "EvaluationResult" DROP CONSTRAINT "EvaluationResult_evaluationId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "EvaluationResult" DROP CONSTRAINT "EvaluationResult_promptVariantId_fkey";
+
+-- DropTable
+DROP TABLE "EvaluationResult";
+
+-- CreateTable
+CREATE TABLE "OutputEvaluation" (
+    "id" UUID NOT NULL,
+    "result" DOUBLE PRECISION NOT NULL,
+    "details" TEXT,
+    "modelOutputId" UUID NOT NULL,
+    "evaluationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "OutputEvaluation_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "OutputEvaluation_modelOutputId_evaluationId_key" ON "OutputEvaluation"("modelOutputId", "evaluationId");
+
+-- AddForeignKey
+ALTER TABLE "OutputEvaluation" ADD CONSTRAINT "OutputEvaluation_modelOutputId_fkey" FOREIGN KEY ("modelOutputId") REFERENCES "ModelOutput"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "OutputEvaluation" ADD CONSTRAINT "OutputEvaluation_evaluationId_fkey" FOREIGN KEY ("evaluationId") REFERENCES "Evaluation"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/migrations/20230718201303_add_users_and_orgs/migration.sql

@@ -0,0 +1,124 @@
+DROP TABLE "Account";
+DROP TABLE "Session";
+DROP TABLE "User";
+DROP TABLE "VerificationToken";
+
+CREATE TYPE "OrganizationUserRole" AS ENUM ('ADMIN', 'MEMBER', 'VIEWER');
+
+-- CreateTable
+CREATE TABLE "Organization" (
+    "id" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "personalOrgUserId" UUID,
+
+    CONSTRAINT "Organization_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "OrganizationUser" (
+    "id" UUID NOT NULL,
+    "role" "OrganizationUserRole" NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "userId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "OrganizationUser_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Account" (
+    "id" UUID NOT NULL,
+    "userId" UUID NOT NULL,
+    "type" TEXT NOT NULL,
+    "provider" TEXT NOT NULL,
+    "providerAccountId" TEXT NOT NULL,
+    "refresh_token" TEXT,
+    "refresh_token_expires_in" INTEGER,
+    "access_token" TEXT,
+    "expires_at" INTEGER,
+    "token_type" TEXT,
+    "scope" TEXT,
+    "id_token" TEXT,
+    "session_state" TEXT,
+
+    CONSTRAINT "Account_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Session" (
+    "id" UUID NOT NULL,
+    "sessionToken" TEXT NOT NULL,
+    "userId" UUID NOT NULL,
+    "expires" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "Session_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "User" (
+    "id" UUID NOT NULL,
+    "name" TEXT,
+    "email" TEXT,
+    "emailVerified" TIMESTAMP(3),
+    "image" TEXT,
+
+    CONSTRAINT "User_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "VerificationToken" (
+    "identifier" TEXT NOT NULL,
+    "token" TEXT NOT NULL,
+    "expires" TIMESTAMP(3) NOT NULL
+);
+
+INSERT INTO "Organization" ("id", "updatedAt") VALUES ('11111111-1111-1111-1111-111111111111', CURRENT_TIMESTAMP);
+
+-- AlterTable add organizationId as NULLABLE
+ALTER TABLE "Experiment" ADD COLUMN "organizationId" UUID;
+
+-- Set default organization for existing experiments
+UPDATE "Experiment" SET "organizationId" = '11111111-1111-1111-1111-111111111111';
+
+-- AlterTable set organizationId as NOT NULL
+ALTER TABLE "Experiment" ALTER COLUMN "organizationId" SET NOT NULL;
+
+
+-- CreateIndex
+CREATE UNIQUE INDEX "OrganizationUser_organizationId_userId_key" ON "OrganizationUser"("organizationId", "userId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Account_provider_providerAccountId_key" ON "Account"("provider", "providerAccountId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Session_sessionToken_key" ON "Session"("sessionToken");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "User_email_key" ON "User"("email");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "VerificationToken_token_key" ON "VerificationToken"("token");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "VerificationToken_identifier_token_key" ON "VerificationToken"("identifier", "token");
+
+-- AddForeignKey
+ALTER TABLE "Experiment" ADD CONSTRAINT "Experiment_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "OrganizationUser" ADD CONSTRAINT "OrganizationUser_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "OrganizationUser" ADD CONSTRAINT "OrganizationUser_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Account" ADD CONSTRAINT "Account_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Session" ADD CONSTRAINT "Session_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+CREATE UNIQUE INDEX "Organization_personalOrgUserId_key" ON "Organization"("personalOrgUserId");
+
+ALTER TABLE "Organization" ADD CONSTRAINT "Organization_personalOrgUserId_fkey" FOREIGN KEY ("personalOrgUserId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/migrations/20230719181731_remove_duplicate_scenario_variant_cell_fields/migration.sql

@@ -0,0 +1,16 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `completionTokens` on the `ScenarioVariantCell` table. All the data in the column will be lost.
+  - You are about to drop the column `inputHash` on the `ScenarioVariantCell` table. All the data in the column will be lost.
+  - You are about to drop the column `output` on the `ScenarioVariantCell` table. All the data in the column will be lost.
+  - You are about to drop the column `promptTokens` on the `ScenarioVariantCell` table. All the data in the column will be lost.
+  - You are about to drop the column `timeToComplete` on the `ScenarioVariantCell` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "ScenarioVariantCell" DROP COLUMN "completionTokens",
+DROP COLUMN "inputHash",
+DROP COLUMN "output",
+DROP COLUMN "promptTokens",
+DROP COLUMN "timeToComplete";

prisma/migrations/20230719224611_add_cost_to_model_output/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `model` on the `PromptVariant` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "ModelOutput" ADD COLUMN     "cost" DOUBLE PRECISION;

prisma/migrations/20230720213215_add_model_providers/migration.sql

@@ -0,0 +1,17 @@
+-- Add new columns allowing NULL values
+ALTER TABLE "PromptVariant" 
+ADD COLUMN "constructFnVersion" INTEGER, 
+ADD COLUMN "modelProvider" TEXT;
+
+-- Update existing records to have the default values
+UPDATE "PromptVariant" 
+SET "constructFnVersion" = 1, 
+"modelProvider" = 'openai/ChatCompletion' 
+WHERE "constructFnVersion" IS NULL OR "modelProvider" IS NULL;
+
+-- Alter table to set NOT NULL constraint
+ALTER TABLE "PromptVariant" 
+ALTER COLUMN "constructFnVersion" SET NOT NULL, 
+ALTER COLUMN "modelProvider" SET NOT NULL;
+
+ALTER TABLE "ScenarioVariantCell" ADD COLUMN     "prompt" JSONB;

prisma/migrations/20230725005817_use_id_as_streaming_channel/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `streamingChannel` on the `ScenarioVariantCell` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "ScenarioVariantCell" DROP COLUMN "streamingChannel";

prisma/migrations/20230725191512_migrate_model_response/migration.sql

@@ -0,0 +1,52 @@
+-- DropForeignKey
+ALTER TABLE "ModelOutput" DROP CONSTRAINT "ModelOutput_scenarioVariantCellId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "OutputEvaluation" DROP CONSTRAINT "OutputEvaluation_modelOutputId_fkey";
+
+-- DropIndex
+DROP INDEX "OutputEvaluation_modelOutputId_evaluationId_key";
+
+-- AlterTable
+ALTER TABLE "OutputEvaluation" RENAME COLUMN "modelOutputId" TO "modelResponseId";
+
+-- AlterTable
+ALTER TABLE "ScenarioVariantCell" DROP COLUMN "retryTime",
+DROP COLUMN "statusCode",
+ADD COLUMN     "jobQueuedAt" TIMESTAMP(3),
+ADD COLUMN     "jobStartedAt" TIMESTAMP(3);
+
+ALTER TABLE "ModelOutput" RENAME TO "ModelResponse";
+
+ALTER TABLE "ModelResponse" 
+ADD COLUMN "requestedAt" TIMESTAMP(3),
+ADD COLUMN "receivedAt" TIMESTAMP(3),
+ADD COLUMN "statusCode" INTEGER,
+ADD COLUMN "errorMessage" TEXT,
+ADD COLUMN "retryTime" TIMESTAMP(3),
+ADD COLUMN "outdated" BOOLEAN NOT NULL DEFAULT false;
+
+-- 3. Remove the unnecessary column
+ALTER TABLE "ModelResponse"
+DROP COLUMN "timeToComplete";
+
+-- AlterTable
+ALTER TABLE "ModelResponse" RENAME CONSTRAINT "ModelOutput_pkey" TO "ModelResponse_pkey";
+ALTER TABLE "ModelResponse" ALTER COLUMN "output" DROP NOT NULL;
+
+-- DropIndex
+DROP INDEX "ModelOutput_scenarioVariantCellId_key";
+
+-- AddForeignKey
+ALTER TABLE "ModelResponse" ADD CONSTRAINT "ModelResponse_scenarioVariantCellId_fkey" FOREIGN KEY ("scenarioVariantCellId") REFERENCES "ScenarioVariantCell"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- RenameIndex
+ALTER INDEX "ModelOutput_inputHash_idx" RENAME TO "ModelResponse_inputHash_idx";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "OutputEvaluation_modelResponseId_evaluationId_key" ON "OutputEvaluation"("modelResponseId", "evaluationId");
+
+-- AddForeignKey
+ALTER TABLE "OutputEvaluation" ADD CONSTRAINT "OutputEvaluation_modelResponseId_fkey" FOREIGN KEY ("modelResponseId") REFERENCES "ModelResponse"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+

prisma/migrations/20230801195916_add_world_champs/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "WorldChampEntrant" (
+    "id" UUID NOT NULL,
+    "userId" UUID NOT NULL,
+    "approved" BOOLEAN NOT NULL DEFAULT false,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "WorldChampEntrant_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "WorldChampEntrant_userId_key" ON "WorldChampEntrant"("userId");
+
+-- AddForeignKey
+ALTER TABLE "WorldChampEntrant" ADD CONSTRAINT "WorldChampEntrant_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/migrations/20230801234208_add_created_at_and_updated_at_to_users_table/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "User" ADD COLUMN     "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

prisma/migrations/20230803163242_add_admin_role/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "UserRole" AS ENUM ('ADMIN', 'USER');
+
+-- AlterTable
+ALTER TABLE "User" ADD COLUMN     "role" "UserRole" NOT NULL DEFAULT 'USER';

prisma/migrations/20230804042305_add_datasets/migration.sql

@@ -0,0 +1,28 @@
+-- CreateTable
+CREATE TABLE "Dataset" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "Dataset_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "DatasetEntry" (
+    "id" UUID NOT NULL,
+    "input" TEXT NOT NULL,
+    "output" TEXT,
+    "datasetId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "DatasetEntry_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "Dataset" ADD CONSTRAINT "Dataset_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "DatasetEntry" ADD CONSTRAINT "DatasetEntry_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/migrations/20230805054033_migrate_construct_fn_to_prompt_constructor/migration.sql

@@ -0,0 +1,13 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `constructFn` on the `PromptVariant` table. All the data in the column will be lost.
+  - You are about to drop the column `constructFnVersion` on the `PromptVariant` table. All the data in the column will be lost.
+  - Added the required column `promptConstructor` to the `PromptVariant` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `promptConstructorVersion` to the `PromptVariant` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+
+ALTER TABLE "PromptVariant" RENAME COLUMN "constructFn" TO "promptConstructor";
+ALTER TABLE "PromptVariant" RENAME COLUMN "constructFnVersion" TO "promptConstructorVersion";

prisma/schema.prisma

@@ -2,8 +2,7 @@
 // learn more about it in the docs: https://pris.ly/d/prisma-schema
 
 generator client {
-    provider        = "prisma-client-js"
+    provider = "prisma-client-js"
-    previewFeatures = ["jsonProtocol"]
 }
 
 datasource db {
@@ -17,19 +16,26 @@ model Experiment {
 
     sortIndex Int @default(0)
 
-    createdAt        DateTime           @default(now())
+    organizationId String        @db.Uuid
-    updatedAt        DateTime           @updatedAt
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
-    TemplateVariable TemplateVariable[]
+
-    PromptVariant    PromptVariant[]
+    createdAt DateTime @default(now())
-    TestScenario     TestScenario[]
+    updatedAt DateTime @updatedAt
-    Evaluation       Evaluation[]
+
+    templateVariables TemplateVariable[]
+    promptVariants    PromptVariant[]
+    testScenarios     TestScenario[]
+    evaluations       Evaluation[]
 }
 
 model PromptVariant {
-    id    String @id @default(uuid()) @db.Uuid
+    id String @id @default(uuid()) @db.Uuid
-    label String
 
-    constructFn String
+    label                    String
+    promptConstructor        String
+    promptConstructorVersion Int
+    model                    String
+    modelProvider            String
 
     uiId      String  @default(uuid()) @db.Uuid
     visible   Boolean @default(true)
@@ -38,10 +44,9 @@ model PromptVariant {
     experimentId String     @db.Uuid
     experiment   Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade)
 
-    createdAt        DateTime           @default(now())
+    createdAt            DateTime              @default(now())
-    updatedAt        DateTime           @updatedAt
+    updatedAt            DateTime              @updatedAt
-    ModelOutput      ModelOutput[]
+    scenarioVariantCells ScenarioVariantCell[]
-    EvaluationResult EvaluationResult[]
 
     @@index([uiId])
 }
@@ -58,9 +63,9 @@ model TestScenario {
     experimentId String     @db.Uuid
     experiment   Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade)
 
-    createdAt   DateTime      @default(now())
+    createdAt            DateTime              @default(now())
-    updatedAt   DateTime      @updatedAt
+    updatedAt            DateTime              @updatedAt
-    ModelOutput ModelOutput[]
+    scenarioVariantCells ScenarioVariantCell[]
 }
 
 model TemplateVariable {
@@ -75,20 +80,25 @@ model TemplateVariable {
     updatedAt DateTime @updatedAt
 }
 
-model ModelOutput {
+enum CellRetrievalStatus {
+    PENDING
+    IN_PROGRESS
+    COMPLETE
+    ERROR
+}
+
+model ScenarioVariantCell {
     id String @id @default(uuid()) @db.Uuid
 
-    inputHash      String
+    retrievalStatus CellRetrievalStatus @default(COMPLETE)
-    output         Json
+    jobQueuedAt     DateTime?
-    statusCode     Int
+    jobStartedAt    DateTime?
-    errorMessage   String?
+    modelResponses  ModelResponse[]
-    timeToComplete Int     @default(0)
+    errorMessage    String? // Contains errors that occurred independently of model responses
-
-    promptTokens     Int? // Added promptTokens field
-    completionTokens Int? // Added completionTokens field
 
     promptVariantId String        @db.Uuid
     promptVariant   PromptVariant @relation(fields: [promptVariantId], references: [id], onDelete: Cascade)
+    prompt          Json?
 
     testScenarioId String       @db.Uuid
     testScenario   TestScenario @relation(fields: [testScenarioId], references: [id], onDelete: Cascade)
@@ -97,82 +107,197 @@ model ModelOutput {
     updatedAt DateTime @updatedAt
 
     @@unique([promptVariantId, testScenarioId])
+}
+
+model ModelResponse {
+    id String @id @default(uuid()) @db.Uuid
+
+    inputHash        String
+    requestedAt      DateTime?
+    receivedAt       DateTime?
+    output           Json?
+    cost             Float?
+    promptTokens     Int?
+    completionTokens Int?
+    statusCode       Int?
+    errorMessage     String?
+    retryTime        DateTime?
+    outdated         Boolean   @default(false)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+
+    scenarioVariantCellId String              @db.Uuid
+    scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
+    outputEvaluations     OutputEvaluation[]
+
     @@index([inputHash])
 }
 
-enum EvaluationMatchType {
+enum EvalType {
     CONTAINS
     DOES_NOT_CONTAIN
+    GPT4_EVAL
 }
 
 model Evaluation {
     id String @id @default(uuid()) @db.Uuid
 
-    name        String
+    label    String
-    matchString String
+    evalType EvalType
-    matchType   EvaluationMatchType
+    value    String
 
     experimentId String     @db.Uuid
     experiment   Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade)
 
-    createdAt        DateTime           @default(now())
+    createdAt         DateTime           @default(now())
-    updatedAt        DateTime           @updatedAt
+    updatedAt         DateTime           @updatedAt
-    EvaluationResult EvaluationResult[]
+    outputEvaluations OutputEvaluation[]
 }
 
-model EvaluationResult {
+model OutputEvaluation {
     id String @id @default(uuid()) @db.Uuid
 
-    passCount Int
+    // Number between 0 (fail) and 1 (pass)
-    failCount Int
+    result  Float
+    details String?
+
+    modelResponseId String        @db.Uuid
+    modelResponse   ModelResponse @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
 
     evaluationId String     @db.Uuid
     evaluation   Evaluation @relation(fields: [evaluationId], references: [id], onDelete: Cascade)
 
-    promptVariantId String        @db.Uuid
+    createdAt DateTime @default(now())
-    promptVariant   PromptVariant @relation(fields: [promptVariantId], references: [id], onDelete: Cascade)
+    updatedAt DateTime @updatedAt
+
+    @@unique([modelResponseId, evaluationId])
+}
+
+model Dataset {
+    id String @id @default(uuid()) @db.Uuid
+
+    name           String
+    datasetEntries DatasetEntry[]
+
+    organizationId String       @db.Uuid
+    organization   Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
+model DatasetEntry {
+    id String @id @default(uuid()) @db.Uuid
+
+    input  String
+    output String?
+
+    datasetId String   @db.Uuid
+    dataset   Dataset? @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
+model Organization {
+    id                String  @id @default(uuid()) @db.Uuid
+    personalOrgUserId String? @unique @db.Uuid
+    PersonalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+
+    createdAt         DateTime           @default(now())
+    updatedAt         DateTime           @updatedAt
+    organizationUsers OrganizationUser[]
+    experiments       Experiment[]
+    datasets          Dataset[]
+}
+
+enum OrganizationUserRole {
+    ADMIN
+    MEMBER
+    VIEWER
+}
+
+model OrganizationUser {
+    id String @id @default(uuid()) @db.Uuid
+
+    role OrganizationUserRole
+
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+
+    userId String @db.Uuid
+    user   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    @@unique([evaluationId, promptVariantId])
+    @@unique([organizationId, userId])
+}
+
+model WorldChampEntrant {
+    id String @id @default(uuid()) @db.Uuid
+
+    userId String @db.Uuid
+    user   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
+
+    approved Boolean @default(false)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+
+    @@unique([userId])
 }
 
-// Necessary for Next auth
 model Account {
-    id                String  @id @default(cuid())
+    id                       String  @id @default(uuid()) @db.Uuid
-    userId            String
+    userId                   String  @db.Uuid
-    type              String
+    type                     String
-    provider          String
+    provider                 String
-    providerAccountId String
+    providerAccountId        String
-    refresh_token     String? // @db.Text
+    refresh_token            String? @db.Text
-    access_token      String? // @db.Text
+    refresh_token_expires_in Int?
-    expires_at        Int?
+    access_token             String? @db.Text
-    token_type        String?
+    expires_at               Int?
-    scope             String?
+    token_type               String?
-    id_token          String? // @db.Text
+    scope                    String?
-    session_state     String?
+    id_token                 String? @db.Text
-    user              User    @relation(fields: [userId], references: [id], onDelete: Cascade)
+    session_state            String?
+    user                     User    @relation(fields: [userId], references: [id], onDelete: Cascade)
 
     @@unique([provider, providerAccountId])
 }
 
 model Session {
-    id           String   @id @default(cuid())
+    id           String   @id @default(uuid()) @db.Uuid
     sessionToken String   @unique
-    userId       String
+    userId       String   @db.Uuid
     expires      DateTime
     user         User     @relation(fields: [userId], references: [id], onDelete: Cascade)
 }
 
+enum UserRole {
+    ADMIN
+    USER
+}
+
 model User {
-    id            String    @id @default(cuid())
+    id            String    @id @default(uuid()) @db.Uuid
     name          String?
     email         String?   @unique
     emailVerified DateTime?
     image         String?
-    accounts      Account[]
+
-    sessions      Session[]
+    role UserRole @default(USER)
+
+    accounts          Account[]
+    sessions          Session[]
+    organizationUsers OrganizationUser[]
+    organizations     Organization[]
+    worldChampEntrant WorldChampEntrant?
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @default(now()) @updatedAt
 }
 
 model VerificationToken {

prisma/seed.ts

@@ -1,76 +1,102 @@
 import { prisma } from "~/server/db";
+import dedent from "dedent";
+import { generateNewCell } from "~/server/utils/generateNewCell";
+import { promptConstructorVersion } from "~/promptConstructor/version";
 
-const experimentId = "11111111-1111-1111-1111-111111111111";
+const defaultId = "11111111-1111-1111-1111-111111111111";
+
+await prisma.organization.deleteMany({
+  where: { id: defaultId },
+});
+
+// If there's an existing org, just seed into it
+const org =
+  (await prisma.organization.findFirst({})) ??
+  (await prisma.organization.create({
+    data: { id: defaultId },
+  }));
 
-// Delete the existing experiment
 await prisma.experiment.deleteMany({
   where: {
-    id: experimentId,
+    id: defaultId,
   },
 });
 
-const experiment = await prisma.experiment.create({
+await prisma.experiment.create({
   data: {
-    id: experimentId,
+    id: defaultId,
     label: "Country Capitals Example",
+    organizationId: org.id,
   },
 });
 
-await prisma.modelOutput.deleteMany({
+await prisma.scenarioVariantCell.deleteMany({
   where: {
     promptVariant: {
-      experimentId,
+      experimentId: defaultId,
     },
   },
 });
 
 await prisma.promptVariant.deleteMany({
   where: {
-    experimentId,
+    experimentId: defaultId,
   },
 });
 
 await prisma.promptVariant.createMany({
   data: [
     {
-      experimentId,
+      experimentId: defaultId,
       label: "Prompt Variant 1",
       sortIndex: 0,
-      constructFn: `prompt = {
+      model: "gpt-3.5-turbo-0613",
-        model: "gpt-3.5-turbo-0613",
+      modelProvider: "openai/ChatCompletion",
-        messages: [{ role: "user", content: "What is the capital of {{country}}?" }],
+      promptConstructorVersion,
-        temperature: 0,
+      promptConstructor: dedent`
-      }`,
+        definePrompt("openai/ChatCompletion", {
+          model: "gpt-3.5-turbo-0613",
+          messages: [
+            {
+              role: "user",
+              content: \`What is the capital of ${"$"}{scenario.country}?\`
+            }
+          ],
+          temperature: 0,
+        })`,
     },
     {
-      experimentId,
+      experimentId: defaultId,
       label: "Prompt Variant 2",
       sortIndex: 1,
-      constructFn: `prompt = {
+      model: "gpt-3.5-turbo-0613",
-        model: "gpt-3.5-turbo-0613",
+      modelProvider: "openai/ChatCompletion",
-        messages: [
+      promptConstructorVersion,
-          {
+      promptConstructor: dedent`
-            role: "user",
+        definePrompt("openai/ChatCompletion", {
-            content:
+          model: "gpt-3.5-turbo-0613",
-              "What is the capital of {{country}}? Return just the city name and nothing else.",
+          messages: [
-          },
+            {
-        ],
+              role: "user",
-        temperature: 0,
+              content: \`What is the capital of ${"$"}{scenario.country}? Return just the city name and nothing else.\`
-      }`,
+            }
+          ],
+          temperature: 0,
+        })`,
     },
   ],
 });
 
 await prisma.templateVariable.deleteMany({
   where: {
-    experimentId,
+    experimentId: defaultId,
   },
 });
 
 await prisma.templateVariable.createMany({
   data: [
     {
-      experimentId,
+      experimentId: defaultId,
       label: "country",
     },
   ],
@@ -78,32 +104,66 @@ await prisma.templateVariable.createMany({
 
 await prisma.testScenario.deleteMany({
   where: {
-    experimentId,
+    experimentId: defaultId,
   },
 });
 
+const countries = [
+  "Afghanistan",
+  "Albania",
+  "Algeria",
+  "Andorra",
+  "Angola",
+  "Antigua and Barbuda",
+  "Argentina",
+  "Armenia",
+  "Australia",
+  "Austria",
+  "Austrian Empire",
+  "Azerbaijan",
+  "Baden",
+  "Bahamas, The",
+  "Bahrain",
+  "Bangladesh",
+  "Barbados",
+  "Bavaria",
+  "Belarus",
+  "Belgium",
+  "Belize",
+  "Benin (Dahomey)",
+  "Bolivia",
+  "Bosnia and Herzegovina",
+  "Botswana",
+];
 await prisma.testScenario.createMany({
-  data: [
+  data: countries.map((country, i) => ({
-    {
+    experimentId: defaultId,
-      experimentId,
+    sortIndex: i,
-      sortIndex: 0,
+    variableValues: {
-      variableValues: {
+      country: country,
-        country: "Spain",
-      },
     },
-    {
+  })),
-      experimentId,
-      sortIndex: 1,
-      variableValues: {
-        country: "USA",
-      },
-    },
-    {
-      experimentId,
-      sortIndex: 2,
-      variableValues: {
-        country: "Chile",
-      },
-    },
-  ],
 });
+
+const variants = await prisma.promptVariant.findMany({
+  where: {
+    experimentId: defaultId,
+  },
+});
+
+const scenarios = await prisma.testScenario.findMany({
+  where: {
+    experimentId: defaultId,
+  },
+});
+
+await Promise.all(
+  variants
+    .flatMap((variant) =>
+      scenarios.map((scenario) => ({
+        promptVariantId: variant.id,
+        testScenarioId: scenario.id,
+      })),
+    )
+    .map((cell) => generateNewCell(cell.promptVariantId, cell.testScenarioId, { stream: false })),
+);

prisma/seedAgiEval.ts

@@ -0,0 +1,128 @@
+import { prisma } from "~/server/db";
+import { generateNewCell } from "~/server/utils/generateNewCell";
+import dedent from "dedent";
+import { execSync } from "child_process";
+import fs from "fs";
+import { promptConstructorVersion } from "~/promptConstructor/version";
+
+const defaultId = "11111111-1111-1111-1111-111111111112";
+
+await prisma.organization.deleteMany({
+  where: { id: defaultId },
+});
+
+// If there's an existing org, just seed into it
+const org =
+  (await prisma.organization.findFirst({})) ??
+  (await prisma.organization.create({
+    data: { id: defaultId },
+  }));
+
+// Clone the repo from git@github.com:microsoft/AGIEval.git into a tmp dir if it doesn't exist
+const tmpDir = "/tmp/agi-eval";
+if (!fs.existsSync(tmpDir)) {
+  execSync(`git clone git@github.com:microsoft/AGIEval.git ${tmpDir}`);
+}
+
+const datasets = [
+  "sat-en",
+  "sat-math",
+  "lsat-rc",
+  "lsat-ar",
+  "aqua-rat",
+  "logiqa-en",
+  "lsat-lr",
+  "math",
+];
+
+type Scenario = {
+  passage: string | null;
+  question: string;
+  options: string[] | null;
+  label: string;
+};
+
+for (const dataset of datasets) {
+  const experimentName = `AGI-Eval: ${dataset}`;
+  const oldExperiment = await prisma.experiment.findFirst({
+    where: {
+      label: experimentName,
+      organizationId: org.id,
+    },
+  });
+  if (oldExperiment) {
+    await prisma.experiment.deleteMany({
+      where: { id: oldExperiment.id },
+    });
+  }
+
+  const experiment = await prisma.experiment.create({
+    data: {
+      id: oldExperiment?.id ?? undefined,
+      label: experimentName,
+      organizationId: org.id,
+    },
+  });
+
+  const scenarios: Scenario[] = fs
+    .readFileSync(`${tmpDir}/data/v1/${dataset}.jsonl`, "utf8")
+    .split("\n")
+    .filter((line) => line.length > 0)
+    .map((line) => JSON.parse(line) as Scenario);
+  console.log("scenarios", scenarios.length);
+
+  await prisma.testScenario.createMany({
+    data: scenarios.slice(0, 30).map((scenario, i) => ({
+      experimentId: experiment.id,
+      sortIndex: i,
+      variableValues: {
+        passage: scenario.passage,
+        question: scenario.question,
+        options: scenario.options?.join("\n"),
+        label: scenario.label,
+      },
+    })),
+  });
+
+  await prisma.templateVariable.createMany({
+    data: ["passage", "question", "options", "label"].map((label) => ({
+      experimentId: experiment.id,
+      label,
+    })),
+  });
+
+  await prisma.promptVariant.createMany({
+    data: [
+      {
+        experimentId: experiment.id,
+        label: "Prompt Variant 1",
+        sortIndex: 0,
+        model: "gpt-3.5-turbo-0613",
+        modelProvider: "openai/ChatCompletion",
+        promptConstructorVersion,
+        promptConstructor: dedent`
+          definePrompt("openai/ChatCompletion", {
+            model: "gpt-3.5-turbo-0613",
+            messages: [
+              {
+                role: "user",
+                content: \`Passage: ${"$"}{scenario.passage}\n\nQuestion: ${"$"}{scenario.question}\n\nOptions: ${"$"}{scenario.options}\n\n Respond with just the letter of the best option in the format Answer: (A).\`
+              }
+            ],
+            temperature: 0,
+          })`,
+      },
+    ],
+  });
+
+  await prisma.evaluation.createMany({
+    data: [
+      {
+        experimentId: experiment.id,
+        label: "Eval",
+        evalType: "CONTAINS",
+        value: "Answer: ({{label}})",
+      },
+    ],
+  });
+}

prisma/seedTwitterSentiment.ts

@@ -0,0 +1,114 @@
+import { prisma } from "~/server/db";
+import dedent from "dedent";
+import fs from "fs";
+import { parse } from "csv-parse/sync";
+import { promptConstructorVersion } from "~/promptConstructor/version";
+
+const defaultId = "11111111-1111-1111-1111-111111111112";
+
+await prisma.organization.deleteMany({
+  where: { id: defaultId },
+});
+
+// If there's an existing org, just seed into it
+const org =
+  (await prisma.organization.findFirst({})) ??
+  (await prisma.organization.create({
+    data: { id: defaultId },
+  }));
+
+type Scenario = {
+  text: string;
+  sentiment: string;
+  emotion: string;
+};
+
+const experimentName = `Twitter Sentiment Analysis`;
+const oldExperiment = await prisma.experiment.findFirst({
+  where: {
+    label: experimentName,
+    organizationId: org.id,
+  },
+});
+if (oldExperiment) {
+  await prisma.experiment.deleteMany({
+    where: { id: oldExperiment.id },
+  });
+}
+
+const experiment = await prisma.experiment.create({
+  data: {
+    id: oldExperiment?.id ?? undefined,
+    label: experimentName,
+    organizationId: org.id,
+  },
+});
+
+const content = fs.readFileSync("./prisma/datasets/validated_tweets.csv", "utf8");
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const records: any[] = parse(content, { delimiter: ",", from_line: 2 });
+
+console.log("records", records);
+
+const scenarios: Scenario[] = records.map((row) => ({
+  text: row[0],
+  sentiment: row[1],
+  emotion: row[2],
+}));
+
+console.log("scenarios", scenarios.length);
+
+await prisma.testScenario.createMany({
+  data: scenarios.slice(0, 30).map((scenario, i) => ({
+    experimentId: experiment.id,
+    sortIndex: i,
+    variableValues: {
+      text: scenario.text,
+      sentiment: scenario.sentiment,
+      emotion: scenario.emotion,
+    },
+  })),
+});
+
+await prisma.templateVariable.createMany({
+  data: ["text", "sentiment", "emotion"].map((label) => ({
+    experimentId: experiment.id,
+    label,
+  })),
+});
+
+await prisma.promptVariant.createMany({
+  data: [
+    {
+      experimentId: experiment.id,
+      label: "Prompt Variant 1",
+      sortIndex: 0,
+      model: "gpt-3.5-turbo-0613",
+      modelProvider: "openai/ChatCompletion",
+      promptConstructorVersion,
+      promptConstructor: dedent`
+          definePrompt("openai/ChatCompletion", {
+            model: "gpt-3.5-turbo-0613",
+            messages: [
+              {
+                role: "user",
+                content: \`Text: ${"$"}{scenario.text}\n\nRespond with the sentiment (negative|neutral|positive) and emotion (optimism|joy|anger|sadness) of the tweet in this format: "answer: <sentiment>-<emotion>".\`
+              }
+            ],
+            temperature: 0,
+          })`,
+    },
+  ],
+});
+
+await prisma.evaluation.createMany({
+  data: [
+    {
+      experimentId: experiment.id,
+      label: "Eval",
+      evalType: "CONTAINS",
+      value: "answer: {{sentiment}}-{{emotion}}",
+    },
+  ],
+});

render.yaml

@@ -12,7 +12,7 @@ services:
     dockerContext: .
     plan: standard
     domains:
-      - openpipe.ai
+      - app.openpipe.ai
     envVars:
       - key: NODE_ENV
         value: production

run-prod.sh

@@ -5,5 +5,11 @@ set -e
 echo "Migrating the database"
 pnpm prisma migrate deploy
 
+echo "Migrating promptConstructors"
+pnpm tsx src/promptConstructor/migrate.ts
+
 echo "Starting the server"
-pnpm start
+
+pnpm concurrently --kill-others \
+  "pnpm start" \
+  "pnpm tsx src/server/tasks/worker.ts"

sentry.client.config.ts

@@ -0,0 +1,33 @@
+// This file configures the initialization of Sentry on the client.
+// The config you add here will be used whenever a users loads a page in their browser.
+// https://docs.sentry.io/platforms/javascript/guides/nextjs/
+
+import * as Sentry from "@sentry/nextjs";
+import { env } from "~/env.mjs";
+
+if (env.NEXT_PUBLIC_SENTRY_DSN) {
+  Sentry.init({
+    dsn: env.NEXT_PUBLIC_SENTRY_DSN,
+
+    // Adjust this value in production, or use tracesSampler for greater control
+    tracesSampleRate: 1,
+
+    // Setting this option to true will print useful information to the console while you're setting up Sentry.
+    debug: false,
+
+    replaysOnErrorSampleRate: 1.0,
+
+    // This sets the sample rate to be 10%. You may want this to be 100% while
+    // in development and sample at a lower rate in production
+    replaysSessionSampleRate: 0.1,
+
+    // You can remove this option if you're not planning to use the Sentry Session Replay feature:
+    integrations: [
+      new Sentry.Replay({
+        // Additional Replay configuration goes in here, for example:
+        maskAllText: true,
+        blockAllMedia: true,
+      }),
+    ],
+  });
+}

sentry.edge.config.ts

@@ -0,0 +1,19 @@
+// This file configures the initialization of Sentry for edge features (middleware, edge routes, and so on).
+// The config you add here will be used whenever one of the edge features is loaded.
+// Note that this config is unrelated to the Vercel Edge Runtime and is also required when running locally.
+// https://docs.sentry.io/platforms/javascript/guides/nextjs/
+
+import * as Sentry from "@sentry/nextjs";
+import { env } from "~/env.mjs";
+
+if (env.NEXT_PUBLIC_SENTRY_DSN) {
+  Sentry.init({
+    dsn: env.NEXT_PUBLIC_SENTRY_DSN,
+
+    // Adjust this value in production, or use tracesSampler for greater control
+    tracesSampleRate: 1,
+
+    // Setting this option to true will print useful information to the console while you're setting up Sentry.
+    debug: false,
+  });
+}

sentry.server.config.ts

@@ -0,0 +1,18 @@
+// This file configures the initialization of Sentry on the server.
+// The config you add here will be used whenever the server handles a request.
+// https://docs.sentry.io/platforms/javascript/guides/nextjs/
+
+import * as Sentry from "@sentry/nextjs";
+import { env } from "~/env.mjs";
+
+if (env.NEXT_PUBLIC_SENTRY_DSN) {
+  Sentry.init({
+    dsn: env.NEXT_PUBLIC_SENTRY_DSN,
+
+    // Adjust this value in production, or use tracesSampler for greater control
+    tracesSampleRate: 1,
+
+    // Setting this option to true will print useful information to the console while you're setting up Sentry.
+    debug: false,
+  });
+}

src/codegen/export-openai-schema.ts

@@ -1,48 +0,0 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-
-import YAML from "yaml";
-import fs from "fs";
-import path from "path";
-import { openapiSchemaToJsonSchema } from "@openapi-contrib/openapi-schema-to-json-schema";
-import assert from "assert";
-import { type AcceptibleInputSchema } from "@openapi-contrib/openapi-schema-to-json-schema/dist/mjs/openapi-schema-types";
-
-const OPENAPI_URL =
-  "https://raw.githubusercontent.com/openai/openai-openapi/0c432eb66fd0c758fd8b9bd69db41c1096e5f4db/openapi.yaml";
-
-const convertOpenApiToJsonSchema = async (url: string) => {
-  // Fetch the openapi document
-  const response = await fetch(url);
-  const openApiYaml = await response.text();
-
-  // Parse the yaml document
-  const openApiDocument = YAML.parse(openApiYaml) as AcceptibleInputSchema;
-
-  // Convert the openapi schema to json schema
-  const jsonSchema = openapiSchemaToJsonSchema(openApiDocument);
-
-  const modelProperty = jsonSchema.components.schemas.CreateChatCompletionRequest.properties.model;
-
-  assert(modelProperty.oneOf.length === 2, "Expected model to have oneOf length of 2");
-
-  // We need to do a bit of surgery here since the Monaco editor doesn't like
-  // the fact that the schema says `model` can be either a string or an enum,
-  // and displays a warning in the editor. Let's stick with just an enum for
-  // now and drop the string option.
-  modelProperty.type = "string";
-  modelProperty.enum = modelProperty.oneOf[1].enum;
-  modelProperty.oneOf = undefined;
-
-  // Get the directory of the current script
-  const currentDirectory = path.dirname(import.meta.url).replace("file://", "");
-
-  // Write the JSON schema to a file in the current directory
-  fs.writeFileSync(
-    path.join(currentDirectory, "openai.schema.json"),
-    JSON.stringify(jsonSchema, null, 2),
-  );
-};
-
-convertOpenApiToJsonSchema(OPENAPI_URL)
-  .then(() => console.log("JSON schema has been written successfully."))
-  .catch((err) => console.error(err));

src/codegen/export-openai-types.ts

@@ -1,52 +0,0 @@
-import fs from "fs";
-import path from "path";
-import openapiTS, { type OpenAPI3 } from "openapi-typescript";
-import YAML from "yaml";
-import _ from "lodash";
-import assert from "assert";
-
-const OPENAPI_URL =
-  "https://raw.githubusercontent.com/openai/openai-openapi/0c432eb66fd0c758fd8b9bd69db41c1096e5f4db/openapi.yaml";
-
-// Generate TypeScript types from OpenAPI
-
-const schema = await fetch(OPENAPI_URL)
-  .then((res) => res.text())
-  .then((txt) => YAML.parse(txt) as OpenAPI3);
-
-console.log(schema.components?.schemas?.CreateChatCompletionRequest);
-
-// @ts-expect-error just assume this works, the assert will catch it if it doesn't
-const modelProperty = schema.components?.schemas?.CreateChatCompletionRequest?.properties?.model;
-
-assert(modelProperty.oneOf.length === 2, "Expected model to have oneOf length of 2");
-
-// We need to do a bit of surgery here since the Monaco editor doesn't like
-// the fact that the schema says `model` can be either a string or an enum,
-// and displays a warning in the editor. Let's stick with just an enum for
-// now and drop the string option.
-modelProperty.type = "string";
-modelProperty.enum = modelProperty.oneOf[1].enum;
-modelProperty.oneOf = undefined;
-
-delete schema["paths"];
-assert(schema.components?.schemas);
-schema.components.schemas = _.pick(schema.components?.schemas, [
-  "CreateChatCompletionRequest",
-  "ChatCompletionRequestMessage",
-  "ChatCompletionFunctions",
-  "ChatCompletionFunctionParameters",
-]);
-console.log(schema);
-
-let openApiTypes = await openapiTS(schema);
-
-// Remove the `export` from any line that starts with `export`
-openApiTypes = openApiTypes.replaceAll("\nexport ", "\n");
-
-// Get the directory of the current script
-const currentDirectory = path.dirname(import.meta.url).replace("file://", "");
-
-// Write the TypeScript types. We only want to use this in our in-app editor, so
-// save as a .txt so VS Code doesn't try to auto-import definitions from it.
-fs.writeFileSync(path.join(currentDirectory, "openai.types.ts.txt"), openApiTypes);

src/codegen/openai.types.ts.txt

@@ -1,148 +0,0 @@
-/**
- * This file was auto-generated by openapi-typescript.
- * Do not make direct changes to the file.
- */
-
-
-/** OneOf type helpers */
-type Without<T, U> = { [P in Exclude<keyof T, keyof U>]?: never };
-type XOR<T, U> = (T | U) extends object ? (Without<T, U> & U) | (Without<U, T> & T) : T | U;
-type OneOf<T extends any[]> = T extends [infer Only] ? Only : T extends [infer A, infer B, ...infer Rest] ? OneOf<[XOR<A, B>, ...Rest]> : never;
-
-type paths = Record<string, never>;
-
-type webhooks = Record<string, never>;
-
-interface components {
-  schemas: {
-    CreateChatCompletionRequest: {
-      /**
-       * @description ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API. 
-       * @example gpt-3.5-turbo 
-       * @enum {string}
-       */
-      model: "gpt-4" | "gpt-4-0613" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-3.5-turbo" | "gpt-3.5-turbo-16k" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613";
-      /** @description A list of messages comprising the conversation so far. [Example Python code](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb). */
-      messages: (components["schemas"]["ChatCompletionRequestMessage"])[];
-      /** @description A list of functions the model may generate JSON inputs for. */
-      functions?: (components["schemas"]["ChatCompletionFunctions"])[];
-      /** @description Controls how the model responds to function calls. "none" means the model does not call a function, and responds to the end-user. "auto" means the model can pick between an end-user or calling a function.  Specifying a particular function via `{"name":\ "my_function"}` forces the model to call that function. "none" is the default when no functions are present. "auto" is the default if functions are present. */
-      function_call?: OneOf<["none" | "auto", {
-        /** @description The name of the function to call. */
-        name: string;
-      }]>;
-      /**
-       * @description What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.
-       * 
-       * We generally recommend altering this or `top_p` but not both.
-       *  
-       * @default 1 
-       * @example 1
-       */
-      temperature?: number | null;
-      /**
-       * @description An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.
-       * 
-       * We generally recommend altering this or `temperature` but not both.
-       *  
-       * @default 1 
-       * @example 1
-       */
-      top_p?: number | null;
-      /**
-       * @description How many chat completion choices to generate for each input message. 
-       * @default 1 
-       * @example 1
-       */
-      n?: number | null;
-      /**
-       * @description If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb).
-       *  
-       * @default false
-       */
-      stream?: boolean | null;
-      /**
-       * @description Up to 4 sequences where the API will stop generating further tokens.
-       *  
-       * @default null
-       */
-      stop?: (string | null) | (string)[];
-      /**
-       * @description The maximum number of [tokens](/tokenizer) to generate in the chat completion.
-       * 
-       * The total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb) for counting tokens.
-       *  
-       * @default inf
-       */
-      max_tokens?: number;
-      /**
-       * @description Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.
-       * 
-       * [See more information about frequency and presence penalties.](/docs/api-reference/parameter-details)
-       *  
-       * @default 0
-       */
-      presence_penalty?: number | null;
-      /**
-       * @description Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.
-       * 
-       * [See more information about frequency and presence penalties.](/docs/api-reference/parameter-details)
-       *  
-       * @default 0
-       */
-      frequency_penalty?: number | null;
-      /**
-       * @description Modify the likelihood of specified tokens appearing in the completion.
-       * 
-       * Accepts a json object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token.
-       *  
-       * @default null
-       */
-      logit_bias?: Record<string, unknown> | null;
-      /**
-       * @description A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids).
-       *  
-       * @example user-1234
-       */
-      user?: string;
-    };
-    ChatCompletionRequestMessage: {
-      /**
-       * @description The role of the messages author. One of `system`, `user`, `assistant`, or `function`. 
-       * @enum {string}
-       */
-      role: "system" | "user" | "assistant" | "function";
-      /** @description The contents of the message. `content` is required for all messages except assistant messages with function calls. */
-      content?: string;
-      /** @description The name of the author of this message. `name` is required if role is `function`, and it should be the name of the function whose response is in the `content`. May contain a-z, A-Z, 0-9, and underscores, with a maximum length of 64 characters. */
-      name?: string;
-      /** @description The name and arguments of a function that should be called, as generated by the model. */
-      function_call?: {
-        /** @description The name of the function to call. */
-        name?: string;
-        /** @description The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. */
-        arguments?: string;
-      };
-    };
-    ChatCompletionFunctions: {
-      /** @description The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. */
-      name: string;
-      /** @description The description of what the function does. */
-      description?: string;
-      parameters?: components["schemas"]["ChatCompletionFunctionParameters"];
-    };
-    /** @description The parameters the functions accepts, described as a JSON Schema object. See the [guide](/docs/guides/gpt/function-calling) for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for documentation about the format. */
-    ChatCompletionFunctionParameters: {
-      [key: string]: unknown;
-    };
-  };
-  responses: never;
-  parameters: never;
-  requestBodies: never;
-  headers: never;
-  pathItems: never;
-}
-
-type external = Record<string, never>;
-
-type operations = Record<string, never>;

src/codegen/tsconfig.json

@@ -1,6 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "esnext",
-    "moduleResolution": "nodenext"
-  }
-}

src/components/AutoResizeTextArea.tsx

@@ -1,19 +1,22 @@
 import { Textarea, type TextareaProps } from "@chakra-ui/react";
 import ResizeTextarea from "react-textarea-autosize";
-import React from "react";
+import React, { useLayoutEffect, useState } from "react";
 
 export const AutoResizeTextarea: React.ForwardRefRenderFunction<
   HTMLTextAreaElement,
-  TextareaProps
+  TextareaProps & { minRows?: number }
-> = (props, ref) => {
+> = ({ minRows = 1, overflowY = "hidden", ...props }, ref) => {
+  const [isRerendered, setIsRerendered] = useState(false);
+  useLayoutEffect(() => setIsRerendered(true), []);
+
   return (
     <Textarea
       minH="unset"
-      overflow="hidden"
+      minRows={minRows}
+      overflowY={isRerendered ? overflowY : "hidden"}
       w="100%"
       resize="none"
       ref={ref}
-      minRows={1}
       transition="height none"
       as={ResizeTextarea}
       {...props}

src/components/ChangeModelModal/ChangeModelModal.tsx

@@ -0,0 +1,142 @@
+import {
+  Button,
+  HStack,
+  Icon,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  Spinner,
+  Text,
+  VStack,
+} from "@chakra-ui/react";
+import { type PromptVariant } from "@prisma/client";
+import { isObject, isString } from "lodash-es";
+import { useState } from "react";
+import { RiExchangeFundsFill } from "react-icons/ri";
+import { type ProviderModel } from "~/modelProviders/types";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { lookupModel, modelLabel } from "~/utils/utils";
+import CompareFunctions from "../RefinePromptModal/CompareFunctions";
+import { ModelSearch } from "./ModelSearch";
+import { ModelStatsCard } from "./ModelStatsCard";
+
+export const ChangeModelModal = ({
+  variant,
+  onClose,
+}: {
+  variant: PromptVariant;
+  onClose: () => void;
+}) => {
+  const originalModel = lookupModel(variant.modelProvider, variant.model);
+  const [selectedModel, setSelectedModel] = useState({
+    provider: variant.modelProvider,
+    model: variant.model,
+  } as ProviderModel);
+  const [convertedModel, setConvertedModel] = useState<ProviderModel | undefined>();
+  const visibleScenarios = useVisibleScenarioIds();
+
+  const utils = api.useContext();
+
+  const experiment = useExperiment();
+
+  const { mutateAsync: getModifiedPromptMutateAsync, data: modifiedPromptFn } =
+    api.promptVariants.getModifiedPromptFn.useMutation();
+
+  const [getModifiedPromptFn, modificationInProgress] = useHandledAsyncCallback(async () => {
+    if (!experiment) return;
+
+    await getModifiedPromptMutateAsync({
+      id: variant.id,
+      newModel: selectedModel,
+    });
+    setConvertedModel(selectedModel);
+  }, [getModifiedPromptMutateAsync, onClose, experiment, variant, selectedModel]);
+
+  const replaceVariantMutation = api.promptVariants.replaceVariant.useMutation();
+
+  const [replaceVariant, replacementInProgress] = useHandledAsyncCallback(async () => {
+    if (
+      !variant.experimentId ||
+      !modifiedPromptFn ||
+      (isObject(modifiedPromptFn) && "status" in modifiedPromptFn)
+    )
+      return;
+    await replaceVariantMutation.mutateAsync({
+      id: variant.id,
+      promptConstructor: modifiedPromptFn,
+      streamScenarios: visibleScenarios,
+    });
+    await utils.promptVariants.list.invalidate();
+    onClose();
+  }, [replaceVariantMutation, variant, onClose, modifiedPromptFn]);
+
+  const originalLabel = modelLabel(variant.modelProvider, variant.model);
+  const selectedLabel = modelLabel(selectedModel.provider, selectedModel.model);
+  const convertedLabel =
+    convertedModel && modelLabel(convertedModel.provider, convertedModel.model);
+
+  return (
+    <Modal
+      isOpen
+      onClose={onClose}
+      size={{ base: "xl", sm: "2xl", md: "3xl", lg: "5xl", xl: "7xl" }}
+    >
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={RiExchangeFundsFill} />
+            <Text>Change Model</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack spacing={8}>
+            <ModelStatsCard label="Original Model" model={originalModel} />
+            {originalLabel !== selectedLabel && (
+              <ModelStatsCard
+                label="New Model"
+                model={lookupModel(selectedModel.provider, selectedModel.model)}
+              />
+            )}
+            <ModelSearch selectedModel={selectedModel} setSelectedModel={setSelectedModel} />
+            {isString(modifiedPromptFn) && (
+              <CompareFunctions
+                originalFunction={variant.promptConstructor}
+                newFunction={modifiedPromptFn}
+                leftTitle={originalLabel}
+                rightTitle={convertedLabel}
+              />
+            )}
+          </VStack>
+        </ModalBody>
+
+        <ModalFooter>
+          <HStack>
+            <Button
+              colorScheme="gray"
+              onClick={getModifiedPromptFn}
+              minW={24}
+              isDisabled={originalLabel === selectedLabel || modificationInProgress}
+            >
+              {modificationInProgress ? <Spinner boxSize={4} /> : <Text>Convert</Text>}
+            </Button>
+            <Button
+              colorScheme="blue"
+              onClick={replaceVariant}
+              minW={24}
+              isDisabled={!convertedModel || modificationInProgress || replacementInProgress}
+            >
+              {replacementInProgress ? <Spinner boxSize={4} /> : <Text>Accept</Text>}
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

src/components/ChangeModelModal/ModelSearch.tsx

@@ -0,0 +1,36 @@
+import { Text, VStack } from "@chakra-ui/react";
+import { type LegacyRef } from "react";
+import Select from "react-select";
+import { useElementDimensions } from "~/utils/hooks";
+
+import { flatMap } from "lodash-es";
+import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { type ProviderModel } from "~/modelProviders/types";
+import { modelLabel } from "~/utils/utils";
+
+const modelOptions = flatMap(Object.entries(frontendModelProviders), ([providerId, provider]) =>
+  Object.entries(provider.models).map(([modelId]) => ({
+    provider: providerId,
+    model: modelId,
+  })),
+) as ProviderModel[];
+
+export const ModelSearch = (props: {
+  selectedModel: ProviderModel;
+  setSelectedModel: (model: ProviderModel) => void;
+}) => {
+  const [containerRef, containerDimensions] = useElementDimensions();
+
+  return (
+    <VStack ref={containerRef as LegacyRef<HTMLDivElement>} w="full" fontFamily="inconsolata">
+      <Text fontWeight="bold">Browse Models</Text>
+      <Select<ProviderModel>
+        styles={{ control: (provided) => ({ ...provided, width: containerDimensions?.width }) }}
+        getOptionLabel={(data) => modelLabel(data.provider, data.model)}
+        getOptionValue={(data) => modelLabel(data.provider, data.model)}
+        options={modelOptions}
+        onChange={(option) => option && props.setSelectedModel(option)}
+      />
+    </VStack>
+  );
+};

src/components/ChangeModelModal/ModelStatsCard.tsx

@@ -0,0 +1,117 @@
+import {
+  GridItem,
+  HStack,
+  Link,
+  SimpleGrid,
+  Text,
+  VStack,
+  type StackProps,
+} from "@chakra-ui/react";
+import { type lookupModel } from "~/utils/utils";
+
+export const ModelStatsCard = ({
+  label,
+  model,
+}: {
+  label: string;
+  model: ReturnType<typeof lookupModel>;
+}) => {
+  if (!model) return null;
+  return (
+    <VStack w="full" align="start">
+      <Text fontWeight="bold" fontSize="sm" textTransform="uppercase">
+        {label}
+      </Text>
+
+      <VStack
+        w="full"
+        spacing={6}
+        borderWidth={1}
+        borderColor="gray.300"
+        p={4}
+        borderRadius={8}
+        fontFamily="inconsolata"
+      >
+        <HStack w="full" align="flex-start">
+          <VStack flex={1} fontSize="lg" alignItems="flex-start">
+            <Text as="span" fontWeight="bold" color="gray.900">
+              {model.name}
+            </Text>
+            <Text as="span" color="gray.600" fontSize="sm">
+              Provider: {model.provider}
+            </Text>
+          </VStack>
+          <Link
+            href={model.learnMoreUrl}
+            isExternal
+            color="blue.500"
+            fontWeight="bold"
+            fontSize="sm"
+            ml={2}
+          >
+            Learn More
+          </Link>
+        </HStack>
+        <SimpleGrid
+          w="full"
+          justifyContent="space-between"
+          alignItems="flex-start"
+          fontSize="sm"
+          columns={{ base: 2, md: 4 }}
+        >
+          <SelectedModelLabeledInfo label="Context Window" info={model.contextWindow} />
+          {model.promptTokenPrice && (
+            <SelectedModelLabeledInfo
+              label="Input"
+              info={
+                <Text>
+                  ${(model.promptTokenPrice * 1000).toFixed(3)}
+                  <Text color="gray.500"> / 1K tokens</Text>
+                </Text>
+              }
+            />
+          )}
+          {model.completionTokenPrice && (
+            <SelectedModelLabeledInfo
+              label="Output"
+              info={
+                <Text>
+                  ${(model.completionTokenPrice * 1000).toFixed(3)}
+                  <Text color="gray.500"> / 1K tokens</Text>
+                </Text>
+              }
+            />
+          )}
+          {model.pricePerSecond && (
+            <SelectedModelLabeledInfo
+              label="Price"
+              info={
+                <Text>
+                  ${model.pricePerSecond.toFixed(3)}
+                  <Text color="gray.500"> / second</Text>
+                </Text>
+              }
+            />
+          )}
+          <SelectedModelLabeledInfo label="Speed" info={<Text>{model.speed}</Text>} />
+        </SimpleGrid>
+      </VStack>
+    </VStack>
+  );
+};
+
+const SelectedModelLabeledInfo = ({
+  label,
+  info,
+  ...props
+}: {
+  label: string;
+  info: string | number | React.ReactElement;
+} & StackProps) => (
+  <GridItem>
+    <VStack alignItems="flex-start" {...props}>
+      <Text fontWeight="bold">{label}</Text>
+      <Text>{info}</Text>
+    </VStack>
+  </GridItem>
+);

src/components/CustomInstructionsInput.tsx

@@ -0,0 +1,86 @@
+import {
+  Button,
+  Spinner,
+  InputGroup,
+  InputRightElement,
+  Icon,
+  HStack,
+  type InputGroupProps,
+} from "@chakra-ui/react";
+import { IoMdSend } from "react-icons/io";
+import AutoResizeTextArea from "./AutoResizeTextArea";
+
+export const CustomInstructionsInput = ({
+  instructions,
+  setInstructions,
+  loading,
+  onSubmit,
+  placeholder = "Send custom instructions",
+  ...props
+}: {
+  instructions: string;
+  setInstructions: (instructions: string) => void;
+  loading: boolean;
+  onSubmit: () => void;
+  placeholder?: string;
+} & InputGroupProps) => {
+  return (
+    <InputGroup
+      size="md"
+      w="full"
+      maxW="600"
+      boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
+      borderRadius={8}
+      alignItems="center"
+      colorScheme="orange"
+      {...props}
+    >
+      <AutoResizeTextArea
+        value={instructions}
+        onChange={(e) => setInstructions(e.target.value)}
+        onKeyDown={(e) => {
+          if (e.key === "Enter" && !e.metaKey && !e.ctrlKey && !e.shiftKey) {
+            e.preventDefault();
+            e.currentTarget.blur();
+            onSubmit();
+          }
+        }}
+        placeholder={placeholder}
+        py={4}
+        pl={4}
+        pr={12}
+        colorScheme="orange"
+        borderColor="gray.300"
+        borderWidth={1}
+        _hover={{
+          borderColor: "gray.300",
+        }}
+        _focus={{
+          borderColor: "gray.300",
+        }}
+        isDisabled={loading}
+      />
+      <HStack></HStack>
+      <InputRightElement width="8" height="full">
+        <Button
+          h="8"
+          w="8"
+          minW="unset"
+          size="sm"
+          onClick={() => onSubmit()}
+          variant={instructions ? "solid" : "ghost"}
+          mr={4}
+          borderRadius="8"
+          bgColor={instructions ? "orange.400" : "transparent"}
+          colorScheme="orange"
+        >
+          {loading ? (
+            <Spinner boxSize={4} />
+          ) : (
+            <Icon as={IoMdSend} color={instructions ? "white" : "gray.500"} boxSize={5} />
+          )}
+        </Button>
+      </InputRightElement>
+    </InputGroup>
+  );
+};

src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -0,0 +1,69 @@
+import {
+  Button,
+  Icon,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+  useDisclosure,
+  Text,
+} from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import { useRef } from "react";
+import { BsTrash } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+
+export const DeleteButton = () => {
+  const experiment = useExperiment();
+  const mutation = api.experiments.delete.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const { isOpen, onOpen, onClose } = useDisclosure();
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    await mutation.mutateAsync({ id: experiment.data.id });
+    await utils.experiments.list.invalidate();
+    await router.push({ pathname: "/experiments" });
+    onClose();
+  }, [mutation, experiment.data?.id, router]);
+
+  return (
+    <>
+      <Button size="sm" variant="ghost" colorScheme="red" fontWeight="normal" onClick={onOpen}>
+        <Icon as={BsTrash} boxSize={4} />
+        <Text ml={2}>Delete Experiment</Text>
+      </Button>
+
+      <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+        <AlertDialogOverlay>
+          <AlertDialogContent>
+            <AlertDialogHeader fontSize="lg" fontWeight="bold">
+              Delete Experiment
+            </AlertDialogHeader>
+
+            <AlertDialogBody>
+              If you delete this experiment all the associated prompts and scenarios will be deleted
+              as well. Are you sure?
+            </AlertDialogBody>
+
+            <AlertDialogFooter>
+              <Button ref={cancelRef} onClick={onClose}>
+                Cancel
+              </Button>
+              <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
+                Delete
+              </Button>
+            </AlertDialogFooter>
+          </AlertDialogContent>
+        </AlertDialogOverlay>
+      </AlertDialog>
+    </>
+  );
+};

src/components/ExperimentSettingsDrawer/ExperimentSettingsDrawer.tsx

@@ -6,13 +6,14 @@ import {
   DrawerHeader,
   DrawerOverlay,
   Heading,
-  Stack,
+  VStack,
 } from "@chakra-ui/react";
-import EditScenarioVars from "./EditScenarioVars";
+import EditScenarioVars from "../OutputsTable/EditScenarioVars";
-import EditEvaluations from "./EditEvaluations";
+import EditEvaluations from "../OutputsTable/EditEvaluations";
 import { useAppStore } from "~/state/store";
+import { DeleteButton } from "./DeleteButton";
 
-export default function SettingsDrawer() {
+export default function ExperimentSettingsDrawer() {
   const isOpen = useAppStore((state) => state.drawerOpen);
   const closeDrawer = useAppStore((state) => state.closeDrawer);
 
@@ -22,13 +23,16 @@ export default function SettingsDrawer() {
       <DrawerContent>
         <DrawerCloseButton />
         <DrawerHeader>
-          <Heading size="md">Settings</Heading>
+          <Heading size="md">Experiment Settings</Heading>
         </DrawerHeader>
-        <DrawerBody>
+        <DrawerBody h="full" pb={4}>
-          <Stack spacing={6}>
+          <VStack h="full" justifyContent="space-between">
-            <EditScenarioVars />
+            <VStack spacing={6}>
-            <EditEvaluations />
+              <EditScenarioVars />
-          </Stack>
+              <EditEvaluations />
+            </VStack>
+            <DeleteButton />
+          </VStack>
         </DrawerBody>
       </DrawerContent>
     </Drawer>

src/components/OutputsTable/AddVariantButton.tsx

@@ -0,0 +1,57 @@
+import { Box, Flex, Icon, Spinner } from "@chakra-ui/react";
+import { BsPlus } from "react-icons/bs";
+import { Text } from "@chakra-ui/react";
+import { api } from "~/utils/api";
+import {
+  useExperiment,
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useVisibleScenarioIds,
+} from "~/utils/hooks";
+import { cellPadding } from "../constants";
+import { ActionButton } from "./ScenariosHeader";
+
+export default function AddVariantButton() {
+  const experiment = useExperiment();
+  const mutation = api.promptVariants.create.useMutation();
+  const utils = api.useContext();
+  const visibleScenarios = useVisibleScenarioIds();
+
+  const [onClick, loading] = useHandledAsyncCallback(async () => {
+    if (!experiment.data) return;
+    await mutation.mutateAsync({
+      experimentId: experiment.data.id,
+      streamScenarios: visibleScenarios,
+    });
+    await utils.promptVariants.list.invalidate();
+  }, [mutation]);
+
+  const { canModify } = useExperimentAccess();
+  if (!canModify) return <Box w={cellPadding.x} />;
+
+  return (
+    <Flex w="100%" justifyContent="flex-end">
+      <ActionButton
+        onClick={onClick}
+        py={5}
+        leftIcon={<Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />}
+      >
+        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
+      </ActionButton>
+      {/* <Button
+        alignItems="center"
+        justifyContent="center"
+        fontWeight="normal"
+        bgColor="transparent"
+        _hover={{ bgColor: "gray.100" }}
+        px={cellPadding.x}
+        onClick={onClick}
+        height="unset"
+        minH={headerMinHeight}
+      >
+        <Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />
+        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
+      </Button> */}
+    </Flex>
+  );
+}

src/components/OutputsTable/EditEvaluations.tsx

@@ -11,14 +11,16 @@ import {
   FormLabel,
   Select,
   FormHelperText,
+  Code,
 } from "@chakra-ui/react";
-import { type Evaluation, EvaluationMatchType } from "@prisma/client";
+import { type Evaluation, EvalType } from "@prisma/client";
 import { useCallback, useState } from "react";
 import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import AutoResizeTextArea from "../AutoResizeTextArea";
 
-type EvalValues = Pick<Evaluation, "name" | "matchString" | "matchType">;
+type EvalValues = Pick<Evaluation, "label" | "value" | "evalType">;
 
 export function EvaluationEditor(props: {
   evaluation: Evaluation | null;
@@ -27,35 +29,35 @@ export function EvaluationEditor(props: {
   onCancel: () => void;
 }) {
   const [values, setValues] = useState<EvalValues>({
-    name: props.evaluation?.name ?? props.defaultName ?? "",
+    label: props.evaluation?.label ?? props.defaultName ?? "",
-    matchString: props.evaluation?.matchString ?? "",
+    value: props.evaluation?.value ?? "",
-    matchType: props.evaluation?.matchType ?? "CONTAINS",
+    evalType: props.evaluation?.evalType ?? "CONTAINS",
   });
 
   return (
     <VStack borderTopWidth={1} borderColor="gray.200" py={4}>
       <HStack w="100%">
         <FormControl flex={1}>
-          <FormLabel fontSize="sm">Evaluation Name</FormLabel>
+          <FormLabel fontSize="sm">Eval Name</FormLabel>
           <Input
             size="sm"
-            value={values.name}
+            value={values.label}
-            onChange={(e) => setValues((values) => ({ ...values, name: e.target.value }))}
+            onChange={(e) => setValues((values) => ({ ...values, label: e.target.value }))}
           />
         </FormControl>
         <FormControl flex={1}>
-          <FormLabel fontSize="sm">Match Type</FormLabel>
+          <FormLabel fontSize="sm">Eval Type</FormLabel>
           <Select
             size="sm"
-            value={values.matchType}
+            value={values.evalType}
             onChange={(e) =>
               setValues((values) => ({
                 ...values,
-                matchType: e.target.value as EvaluationMatchType,
+                evalType: e.target.value as EvalType,
               }))
             }
           >
-            {Object.values(EvaluationMatchType).map((type) => (
+            {Object.values(EvalType).map((type) => (
               <option key={type} value={type}>
                 {type}
               </option>
@@ -63,17 +65,37 @@ export function EvaluationEditor(props: {
           </Select>
         </FormControl>
       </HStack>
-      <FormControl>
+      {["CONTAINS", "DOES_NOT_CONTAIN"].includes(values.evalType) && (
-        <FormLabel fontSize="sm">Match String</FormLabel>
+        <FormControl>
-        <Input
+          <FormLabel fontSize="sm">Match String</FormLabel>
-          size="sm"
+          <Input
-          value={values.matchString}
+            size="sm"
-          onChange={(e) => setValues((values) => ({ ...values, matchString: e.target.value }))}
+            value={values.value}
-        />
+            onChange={(e) => setValues((values) => ({ ...values, value: e.target.value }))}
-        <FormHelperText>
+          />
-          This string will be interpreted as a regex and checked against each model output.
+          <FormHelperText>
-        </FormHelperText>
+            This string will be interpreted as a regex and checked against each model output. You
-      </FormControl>
+            can include scenario variables using <Code>{"{{curly_braces}}"}</Code>
+          </FormHelperText>
+        </FormControl>
+      )}
+      {values.evalType === "GPT4_EVAL" && (
+        <FormControl pt={2}>
+          <FormLabel fontSize="sm">GPT4 Instructions</FormLabel>
+          <AutoResizeTextArea
+            size="sm"
+            value={values.value}
+            onChange={(e) => setValues((values) => ({ ...values, value: e.target.value }))}
+            minRows={3}
+          />
+          <FormHelperText>
+            Give instructions to GPT-4 for how to evaluate your prompt. It will have access to the
+            full scenario as well as the output it is evaluating. It will <strong>not</strong> have
+            access to the specific prompt variant, so be sure to be clear about the task you want it
+            to perform.
+          </FormHelperText>
+        </FormControl>
+      )}
       <HStack alignSelf="flex-end">
         <Button size="sm" onClick={props.onCancel} colorScheme="gray">
           Cancel
@@ -125,6 +147,7 @@ export default function EditEvaluations() {
     }
     await utils.evaluations.list.invalidate();
     await utils.promptVariants.stats.invalidate();
+    await utils.scenarioVariantCells.get.invalidate();
   }, []);
 
   const onCancel = useCallback(() => {
@@ -156,9 +179,9 @@ export default function EditEvaluations() {
                 align="center"
                 key={evaluation.id}
               >
-                <Text fontWeight="bold">{evaluation.name}</Text>
+                <Text fontWeight="bold">{evaluation.label}</Text>
                 <Text flex={1}>
-                  {evaluation.matchType}: &quot;{evaluation.matchString}&quot;
+                  {evaluation.evalType}: &quot;{evaluation.value}&quot;
                 </Text>
                 <Button
                   variant="unstyled"

src/components/OutputsTable/EditScenarioVars.tsx

@@ -1,4 +1,4 @@
-import { Text, Button, HStack, Heading, Icon, Input, Stack, Code } from "@chakra-ui/react";
+import { Text, Button, HStack, Heading, Icon, Input, Stack } from "@chakra-ui/react";
 import { useState } from "react";
 import { BsCheck, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
@@ -36,8 +36,7 @@ export default function EditScenarioVars() {
       <Heading size="sm">Scenario Variables</Heading>
       <Stack spacing={2}>
         <Text fontSize="sm">
-          Scenario variables can be used in your prompt variants as well as evaluations. Reference
+          Scenario variables can be used in your prompt variants as well as evaluations.
-          them using <Code>{"{{curly_braces}}"}</Code>.
         </Text>
         <HStack spacing={0}>
           <Input

src/components/OutputsTable/FloatingLabelInput.tsx

@@ -0,0 +1,46 @@
+import { FormLabel, FormControl, type TextareaProps } from "@chakra-ui/react";
+import { useState } from "react";
+import AutoResizeTextArea from "../AutoResizeTextArea";
+
+export const FloatingLabelInput = ({
+  label,
+  value,
+  ...props
+}: { label: string; value: string } & TextareaProps) => {
+  const [isFocused, setIsFocused] = useState(false);
+
+  return (
+    <FormControl position="relative">
+      <FormLabel
+        position="absolute"
+        left="10px"
+        top={isFocused || !!value ? 0 : 3}
+        transform={isFocused || !!value ? "translateY(-50%)" : "translateY(0)"}
+        fontSize={isFocused || !!value ? "12px" : "16px"}
+        transition="all 0.15s"
+        zIndex="5"
+        bg="white"
+        px={1}
+        lineHeight="1"
+        pointerEvents="none"
+        color={isFocused ? "blue.500" : "gray.500"}
+      >
+        {label}
+      </FormLabel>
+      <AutoResizeTextArea
+        px={3}
+        pt={3}
+        pb={2}
+        onFocus={() => setIsFocused(true)}
+        onBlur={() => setIsFocused(false)}
+        borderRadius="md"
+        borderColor={isFocused ? "blue.500" : "gray.400"}
+        autoComplete="off"
+        value={value}
+        overflowY="auto"
+        overflowX="hidden"
+        {...props}
+      />
+    </FormControl>
+  );
+};

src/components/OutputsTable/NewScenarioButton.tsx

@@ -1,53 +0,0 @@
-import { Button, type ButtonProps, HStack, Spinner, Icon } from "@chakra-ui/react";
-import { BsPlus } from "react-icons/bs";
-import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-
-// Extracted Button styling into reusable component
-const StyledButton = ({ children, onClick }: ButtonProps) => (
-  <Button
-    fontWeight="normal"
-    bgColor="transparent"
-    _hover={{ bgColor: "gray.100" }}
-    px={2}
-    onClick={onClick}
-  >
-    {children}
-  </Button>
-);
-
-export default function NewScenarioButton() {
-  const experiment = useExperiment();
-  const mutation = api.scenarios.create.useMutation();
-  const utils = api.useContext();
-
-  const [onClick] = useHandledAsyncCallback(async () => {
-    if (!experiment.data) return;
-    await mutation.mutateAsync({
-      experimentId: experiment.data.id,
-    });
-    await utils.scenarios.list.invalidate();
-  }, [mutation]);
-
-  const [onAutogenerate, autogenerating] = useHandledAsyncCallback(async () => {
-    if (!experiment.data) return;
-    await mutation.mutateAsync({
-      experimentId: experiment.data.id,
-      autogenerate: true,
-    });
-    await utils.scenarios.list.invalidate();
-  }, [mutation]);
-
-  return (
-    <HStack spacing={2}>
-      <StyledButton onClick={onClick}>
-        <Icon as={BsPlus} boxSize={6} />
-        Add Scenario
-      </StyledButton>
-      <StyledButton onClick={onAutogenerate}>
-        <Icon as={autogenerating ? Spinner : BsPlus} boxSize={6} mr={autogenerating ? 1 : 0} />
-        Autogenerate Scenario
-      </StyledButton>
-    </HStack>
-  );
-}

src/components/OutputsTable/NewVariantButton.tsx

@@ -1,37 +0,0 @@
-import { Button, Icon, Spinner } from "@chakra-ui/react";
-import { BsPlus } from "react-icons/bs";
-import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-import { cellPadding, headerMinHeight } from "../constants";
-
-export default function NewVariantButton() {
-  const experiment = useExperiment();
-  const mutation = api.promptVariants.create.useMutation();
-  const utils = api.useContext();
-
-  const [onClick, loading] = useHandledAsyncCallback(async () => {
-    if (!experiment.data) return;
-    await mutation.mutateAsync({
-      experimentId: experiment.data.id,
-    });
-    await utils.promptVariants.list.invalidate();
-  }, [mutation]);
-
-  return (
-    <Button
-      w="100%"
-      alignItems="center"
-      justifyContent="center"
-      fontWeight="normal"
-      bgColor="transparent"
-      _hover={{ bgColor: "gray.100" }}
-      px={cellPadding.x}
-      onClick={onClick}
-      height="unset"
-      minH={headerMinHeight}
-    >
-      <Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />
-      Add Variant
-    </Button>
-  );
-}

src/components/OutputsTable/OutputCell/ErrorHandler.tsx

@@ -1,91 +0,0 @@
-import { type ModelOutput } from "@prisma/client";
-import { HStack, VStack, Text, Button, Icon } from "@chakra-ui/react";
-import { useEffect, useState } from "react";
-import { BsArrowClockwise } from "react-icons/bs";
-import { rateLimitErrorMessage } from "~/sharedStrings";
-import pluralize from "pluralize";
-
-const MAX_AUTO_RETRIES = 3;
-
-export const ErrorHandler = ({
-  output,
-  refetchOutput,
-  numPreviousTries,
-}: {
-  output: ModelOutput;
-  refetchOutput: () => void;
-  numPreviousTries: number;
-}) => {
-  const [msToWait, setMsToWait] = useState(0);
-  const shouldAutoRetry =
-    output.errorMessage === rateLimitErrorMessage && numPreviousTries < MAX_AUTO_RETRIES;
-
-  useEffect(() => {
-    if (!shouldAutoRetry) return;
-
-    const initialWaitTime = calculateDelay(numPreviousTries);
-    const msModuloOneSecond = initialWaitTime % 1000;
-    let remainingTime = initialWaitTime - msModuloOneSecond;
-    setMsToWait(remainingTime);
-
-    let interval: NodeJS.Timeout;
-    const timeout = setTimeout(() => {
-      interval = setInterval(() => {
-        remainingTime -= 1000;
-        setMsToWait(remainingTime);
-
-        if (remainingTime <= 0) {
-          refetchOutput();
-          clearInterval(interval);
-        }
-      }, 1000);
-    }, msModuloOneSecond);
-
-    return () => {
-      clearInterval(interval);
-      clearTimeout(timeout);
-    };
-  }, [shouldAutoRetry, setMsToWait, refetchOutput, numPreviousTries]);
-
-  return (
-    <VStack w="full">
-      <HStack w="full" alignItems="flex-start" justifyContent="space-between">
-        <Text color="red.600" fontWeight="bold">
-          Error
-        </Text>
-        <Button
-          size="xs"
-          w={4}
-          h={4}
-          px={4}
-          py={4}
-          minW={0}
-          borderRadius={8}
-          variant="ghost"
-          cursor="pointer"
-          onClick={refetchOutput}
-          aria-label="refetch output"
-        >
-          <Icon as={BsArrowClockwise} boxSize={6} />
-        </Button>
-      </HStack>
-      <Text color="red.600" wordBreak="break-word">
-        {output.errorMessage}
-      </Text>
-      {msToWait > 0 && (
-        <Text color="red.600" fontSize="sm">
-          Retrying in {pluralize("second", Math.ceil(msToWait / 1000), true)}...
-        </Text>
-      )}
-    </VStack>
-  );
-};
-
-const MIN_DELAY = 500; // milliseconds
-const MAX_DELAY = 5000; // milliseconds
-
-function calculateDelay(numPreviousTries: number): number {
-  const baseDelay = Math.min(MAX_DELAY, MIN_DELAY * Math.pow(2, numPreviousTries));
-  const jitter = Math.random() * baseDelay;
-  return baseDelay + jitter;
-}

src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,17 +1,19 @@
-import { type RouterOutputs, api } from "~/utils/api";
+import { api } from "~/utils/api";
 import { type PromptVariant, type Scenario } from "../types";
-import { Spinner, Text, Box, Center, Flex } from "@chakra-ui/react";
+import { type StackProps, Text, VStack } from "@chakra-ui/react";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
 import stringify from "json-stringify-pretty-compact";
-import { type ReactElement, useState, useEffect, useRef, useCallback } from "react";
+import { type ReactElement, useState, useEffect, Fragment, useCallback } from "react";
-import { type ChatCompletion } from "openai/resources/chat";
-import { generateChannel } from "~/utils/generateChannel";
-import { isObject } from "lodash";
 import useSocket from "~/utils/useSocket";
 import { OutputStats } from "./OutputStats";
-import { ErrorHandler } from "./ErrorHandler";
+import { RetryCountdown } from "./RetryCountdown";
+import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { ResponseLog } from "./ResponseLog";
+import { CellOptions } from "./TopActions";
+
+const WAITING_MESSAGE_INTERVAL = 20000;
 
 export default function OutputCell({
   scenario,
@@ -34,97 +36,144 @@ export default function OutputCell({
 
   if (!templateHasVariables) disabledReason = "Add a value to the scenario variables to see output";
 
-  // if (variant.config === null || Object.keys(variant.config).length === 0)
+  const [refetchInterval, setRefetchInterval] = useState(0);
-  //   disabledReason = "Save your prompt variant to see output";
+  const { data: cell, isLoading: queryLoading } = api.scenarioVariantCells.get.useQuery(
-
+    { scenarioId: scenario.id, variantId: variant.id },
-  // const model = getModelName(variant.config as JSONSerializable);
+    { refetchInterval },
-  // TODO: Temporarily hardcoding this while we get other stuff working
-  const model = "gpt-3.5-turbo";
-
-  const outputMutation = api.outputs.get.useMutation();
-
-  const [output, setOutput] = useState<RouterOutputs["outputs"]["get"]>(null);
-  const [channel, setChannel] = useState<string | undefined>(undefined);
-  const [numPreviousTries, setNumPreviousTries] = useState(0);
-
-  const fetchMutex = useRef(false);
-  const [fetchOutput, fetchingOutput] = useHandledAsyncCallback(
-    async (forceRefetch?: boolean) => {
-      if (fetchMutex.current) return;
-      setNumPreviousTries((prev) => prev + 1);
-
-      fetchMutex.current = true;
-      setOutput(null);
-
-      const shouldStream =
-        isObject(variant) &&
-        "config" in variant &&
-        isObject(variant.config) &&
-        "stream" in variant.config &&
-        variant.config.stream === true;
-
-      const channel = shouldStream ? generateChannel() : undefined;
-      setChannel(channel);
-
-      const output = await outputMutation.mutateAsync({
-        scenarioId: scenario.id,
-        variantId: variant.id,
-        channel,
-        forceRefetch,
-      });
-      setOutput(output);
-      await utils.promptVariants.stats.invalidate();
-      fetchMutex.current = false;
-    },
-    [outputMutation, scenario.id, variant.id],
   );
-  const hardRefetch = useCallback(() => fetchOutput(true), [fetchOutput]);
 
-  useEffect(fetchOutput, [scenario.id, variant.id]);
+  const provider =
+    frontendModelProviders[variant.modelProvider as keyof typeof frontendModelProviders];
 
-  // Disconnect from socket if we're not streaming anymore
+  type OutputSchema = Parameters<typeof provider.normalizeOutput>[0];
-  const streamedMessage = useSocket(fetchingOutput ? channel : undefined);
+
-  const streamedContent = streamedMessage?.choices?.[0]?.message?.content;
+  const { mutateAsync: hardRefetchMutate } = api.scenarioVariantCells.forceRefetch.useMutation();
+  const [hardRefetch, hardRefetching] = useHandledAsyncCallback(async () => {
+    await hardRefetchMutate({ scenarioId: scenario.id, variantId: variant.id });
+    await utils.scenarioVariantCells.get.invalidate({
+      scenarioId: scenario.id,
+      variantId: variant.id,
+    });
+    await utils.promptVariants.stats.invalidate({
+      variantId: variant.id,
+    });
+  }, [hardRefetchMutate, scenario.id, variant.id]);
+
+  const fetchingOutput = queryLoading || hardRefetching;
+
+  const awaitingOutput =
+    !cell ||
+    !cell.evalsComplete ||
+    cell.retrievalStatus === "PENDING" ||
+    cell.retrievalStatus === "IN_PROGRESS" ||
+    hardRefetching;
+  useEffect(() => setRefetchInterval(awaitingOutput ? 1000 : 0), [awaitingOutput]);
+
+  // TODO: disconnect from socket if we're not streaming anymore
+  const streamedMessage = useSocket<OutputSchema>(cell?.id);
+
+  const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
+
+  const CellWrapper = useCallback(
+    ({ children, ...props }: StackProps) => (
+      <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
+        {cell && (
+          <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
+        )}
+        <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
+          {children}
+        </VStack>
+        {mostRecentResponse && (
+          <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
+        )}
+      </VStack>
+    ),
+    [hardRefetching, hardRefetch, mostRecentResponse, scenario, cell],
+  );
 
   if (!vars) return null;
 
-  if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
+  if (!cell && !fetchingOutput)
-
-  if (fetchingOutput && !streamedMessage)
     return (
-      <Center h="100%" w="100%">
+      <CellWrapper>
-        <Spinner />
+        <Text color="gray.500">Error retrieving output</Text>
-      </Center>
+      </CellWrapper>
     );
 
-  if (!output && !fetchingOutput) return <Text color="gray.500">Error retrieving output</Text>;
+  if (cell && cell.errorMessage) {
-
-  if (output && output.errorMessage) {
     return (
-      <ErrorHandler
+      <CellWrapper>
-        output={output}
+        <Text color="red.500">{cell.errorMessage}</Text>
-        refetchOutput={hardRefetch}
+      </CellWrapper>
-        numPreviousTries={numPreviousTries}
-      />
     );
   }
 
-  const response = output?.output as unknown as ChatCompletion;
+  if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
-  const message = response?.choices?.[0]?.message;
 
-  if (output && message?.function_call) {
+  const showLogs = !streamedMessage && !mostRecentResponse?.output;
-    const rawArgs = message.function_call.arguments ?? "null";
-    let parsedArgs: string;
-    try {
-      parsedArgs = JSON.parse(rawArgs);
-    } catch (e: any) {
-      parsedArgs = `Failed to parse arguments as JSON: '${rawArgs}' ERROR: ${e.message as string}`;
-    }
 
+  if (showLogs)
     return (
-      <Box fontSize="xs" width="100%" flexWrap="wrap" overflowX="auto">
+      <CellWrapper alignItems="flex-start" fontFamily="inconsolata, monospace" spacing={0}>
+        {cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
+        {cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
+        {cell?.modelResponses?.map((response) => {
+          let numWaitingMessages = 0;
+          const relativeWaitingTime = response.receivedAt
+            ? response.receivedAt.getTime()
+            : Date.now();
+          if (response.requestedAt) {
+            numWaitingMessages = Math.floor(
+              (relativeWaitingTime - response.requestedAt.getTime()) / WAITING_MESSAGE_INTERVAL,
+            );
+          }
+          return (
+            <Fragment key={response.id}>
+              {response.requestedAt && (
+                <ResponseLog time={response.requestedAt} title="Request sent to API" />
+              )}
+              {response.requestedAt &&
+                Array.from({ length: numWaitingMessages }, (_, i) => (
+                  <ResponseLog
+                    key={`waiting-${i}`}
+                    time={
+                      new Date(
+                        (response.requestedAt?.getTime?.() ?? 0) +
+                          (i + 1) * WAITING_MESSAGE_INTERVAL,
+                      )
+                    }
+                    title="Waiting for response..."
+                  />
+                ))}
+              {response.receivedAt && (
+                <ResponseLog
+                  time={response.receivedAt}
+                  title="Response received from API"
+                  message={`statusCode: ${response.statusCode ?? ""}\n ${
+                    response.errorMessage ?? ""
+                  }`}
+                />
+              )}
+            </Fragment>
+          );
+        }) ?? null}
+        {mostRecentResponse?.retryTime && (
+          <RetryCountdown retryTime={mostRecentResponse.retryTime} />
+        )}
+      </CellWrapper>
+    );
+
+  const normalizedOutput = mostRecentResponse?.output
+    ? provider.normalizeOutput(mostRecentResponse?.output)
+    : streamedMessage
+    ? provider.normalizeOutput(streamedMessage)
+    : null;
+
+  if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
+    return (
+      <CellWrapper>
         <SyntaxHighlighter
-          customStyle={{ overflowX: "unset" }}
+          customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
           language="json"
           style={docco}
           lineProps={{
@@ -132,25 +181,17 @@ export default function OutputCell({
           }}
           wrapLines
         >
-          {stringify(
+          {stringify(normalizedOutput.value, { maxLength: 40 })}
-            {
-              function: message.function_call.name,
-              args: parsedArgs,
-            },
-            { maxLength: 40 },
-          )}
         </SyntaxHighlighter>
-        <OutputStats model={model} modelOutput={output} scenario={scenario} />
+      </CellWrapper>
-      </Box>
     );
   }
 
-  const contentToDisplay = message?.content ?? streamedContent ?? JSON.stringify(output?.output);
+  const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
 
   return (
-    <Flex w="100%" h="100%" direction="column" justifyContent="space-between" whiteSpace="pre-wrap">
+    <CellWrapper>
-      {contentToDisplay}
+      <Text>{contentToDisplay}</Text>
-      {output && <OutputStats model={model} modelOutput={output} scenario={scenario} />}
+    </CellWrapper>
-    </Flex>
   );
 }

src/components/OutputsTable/OutputCell/OutputStats.tsx

@@ -1,64 +1,67 @@
-import { type ModelOutput } from "@prisma/client";
-import { type SupportedModel } from "~/server/types";
 import { type Scenario } from "../types";
-import { useExperiment } from "~/utils/hooks";
+import { type RouterOutputs } from "~/utils/api";
-import { api } from "~/utils/api";
+import { HStack, Icon, Text, Tooltip } from "@chakra-ui/react";
-import { calculateTokenCost } from "~/utils/calculateTokenCost";
-import { evaluateOutput } from "~/server/utils/evaluateOutput";
-import { HStack, Icon, Text } from "@chakra-ui/react";
 import { BsCheck, BsClock, BsCurrencyDollar, BsX } from "react-icons/bs";
 import { CostTooltip } from "~/components/tooltip/CostTooltip";
 
-const SHOW_COST = false;
+const SHOW_TIME = true;
-const SHOW_TIME = false;
 
 export const OutputStats = ({
-  model,
+  modelResponse,
-  modelOutput,
-  scenario,
 }: {
-  model: SupportedModel | null;
+  modelResponse: NonNullable<
-  modelOutput: ModelOutput;
+    NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>["modelResponses"][0]
+  >;
   scenario: Scenario;
 }) => {
-  const timeToComplete = modelOutput.timeToComplete;
+  const timeToComplete =
-  const experiment = useExperiment();
+    modelResponse.receivedAt && modelResponse.requestedAt
-  const evals =
+      ? modelResponse.receivedAt.getTime() - modelResponse.requestedAt.getTime()
-    api.evaluations.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+      : 0;
 
-  const promptTokens = modelOutput.promptTokens;
+  const promptTokens = modelResponse.promptTokens;
-  const completionTokens = modelOutput.completionTokens;
+  const completionTokens = modelResponse.completionTokens;
-
-  const promptCost = promptTokens && model ? calculateTokenCost(model, promptTokens) : 0;
-  const completionCost =
-    completionTokens && model ? calculateTokenCost(model, completionTokens, true) : 0;
-
-  const cost = promptCost + completionCost;
-
-  if (!evals.length) return null;
 
   return (
-    <HStack align="center" color="gray.500" fontSize="2xs" mt={{ base: 0, md: 1 }}>
+    <HStack
-      <HStack flex={1}>
+      w="full"
-        {evals.map((evaluation) => {
+      align="center"
-          const passed = evaluateOutput(modelOutput, scenario, evaluation);
+      color="gray.500"
+      fontSize="2xs"
+      mt={{ base: 0, md: 1 }}
+      alignItems="flex-end"
+    >
+      <HStack flex={1} flexWrap="wrap">
+        {modelResponse.outputEvaluations.map((evaluation) => {
+          const passed = evaluation.result > 0.5;
           return (
-            <HStack spacing={0} key={evaluation.id}>
+            <Tooltip
-              <Text>{evaluation.name}</Text>
+              isDisabled={!evaluation.details}
-              <Icon
+              label={evaluation.details}
-                as={passed ? BsCheck : BsX}
+              key={evaluation.id}
-                color={passed ? "green.500" : "red.500"}
+              shouldWrapChildren
-                boxSize={6}
+            >
-              />
+              <HStack spacing={0}>
-            </HStack>
+                <Text>{evaluation.evaluation.label}</Text>
+                <Icon
+                  as={passed ? BsCheck : BsX}
+                  color={passed ? "green.500" : "red.500"}
+                  boxSize={6}
+                />
+              </HStack>
+            </Tooltip>
           );
         })}
       </HStack>
-      {SHOW_COST && (
+      {modelResponse.cost && (
-        <CostTooltip promptTokens={promptTokens} completionTokens={completionTokens} cost={cost}>
+        <CostTooltip
+          promptTokens={promptTokens}
+          completionTokens={completionTokens}
+          cost={modelResponse.cost}
+        >
           <HStack spacing={0}>
             <Icon as={BsCurrencyDollar} />
-            <Text mr={1}>{cost.toFixed(3)}</Text>
+            <Text mr={1}>{modelResponse.cost.toFixed(3)}</Text>
           </HStack>
         </CostTooltip>
       )}

src/components/OutputsTable/OutputCell/PromptModal.tsx

@@ -0,0 +1,36 @@
+import {
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalHeader,
+  ModalOverlay,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import { type RouterOutputs } from "~/utils/api";
+import { JSONTree } from "react-json-tree";
+
+export default function ExpandedModal(props: {
+  cell: NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>;
+  disclosure: UseDisclosureReturn;
+}) {
+  return (
+    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="2xl">
+      <ModalOverlay />
+      <ModalContent>
+        <ModalHeader>Prompt</ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          <JSONTree
+            data={props.cell.prompt}
+            invertTheme={true}
+            theme="chalk"
+            shouldExpandNodeInitially={() => true}
+            getItemString={() => ""}
+            hideRoot
+          />
+        </ModalBody>
+      </ModalContent>
+    </Modal>
+  );
+}

src/components/OutputsTable/OutputCell/ResponseLog.tsx

@@ -0,0 +1,22 @@
+import { HStack, VStack, Text } from "@chakra-ui/react";
+import dayjs from "dayjs";
+
+export const ResponseLog = ({
+  time,
+  title,
+  message,
+}: {
+  time: Date;
+  title: string;
+  message?: string;
+}) => {
+  return (
+    <VStack spacing={0} alignItems="flex-start">
+      <HStack>
+        <Text>{dayjs(time).format("HH:mm:ss")}</Text>
+        <Text>{title}</Text>
+      </HStack>
+      {message && <Text pl={4}>{message}</Text>}
+    </VStack>
+  );
+};

src/components/OutputsTable/OutputCell/RetryCountdown.tsx

@@ -0,0 +1,39 @@
+import { Text } from "@chakra-ui/react";
+import { useEffect, useState } from "react";
+import pluralize from "pluralize";
+
+export const RetryCountdown = ({ retryTime }: { retryTime: Date }) => {
+  const [msToWait, setMsToWait] = useState(0);
+
+  useEffect(() => {
+    const initialWaitTime = retryTime.getTime() - Date.now();
+    const msModuloOneSecond = initialWaitTime % 1000;
+    let remainingTime = initialWaitTime - msModuloOneSecond;
+    setMsToWait(remainingTime);
+
+    let interval: NodeJS.Timeout;
+    const timeout = setTimeout(() => {
+      interval = setInterval(() => {
+        remainingTime -= 1000;
+        setMsToWait(remainingTime);
+
+        if (remainingTime <= 0) {
+          clearInterval(interval);
+        }
+      }, 1000);
+    }, msModuloOneSecond);
+
+    return () => {
+      clearInterval(interval);
+      clearTimeout(timeout);
+    };
+  }, [retryTime]);
+
+  if (msToWait <= 0) return null;
+
+  return (
+    <Text color="red.600" fontSize="sm">
+      Retrying in {pluralize("second", Math.ceil(msToWait / 1000), true)}...
+    </Text>
+  );
+};

src/components/OutputsTable/OutputCell/TopActions.tsx

@@ -0,0 +1,53 @@
+import { HStack, Icon, IconButton, Spinner, Tooltip, useDisclosure } from "@chakra-ui/react";
+import { BsArrowClockwise, BsInfoCircle } from "react-icons/bs";
+import { useExperimentAccess } from "~/utils/hooks";
+import ExpandedModal from "./PromptModal";
+import { type RouterOutputs } from "~/utils/api";
+
+export const CellOptions = ({
+  cell,
+  refetchingOutput,
+  refetchOutput,
+}: {
+  cell: RouterOutputs["scenarioVariantCells"]["get"];
+  refetchingOutput: boolean;
+  refetchOutput: () => void;
+}) => {
+  const { canModify } = useExperimentAccess();
+
+  const modalDisclosure = useDisclosure();
+
+  return (
+    <HStack justifyContent="flex-end" w="full">
+      {cell && (
+        <>
+          <Tooltip label="See Prompt">
+            <IconButton
+              aria-label="See Prompt"
+              icon={<Icon as={BsInfoCircle} boxSize={4} />}
+              onClick={modalDisclosure.onOpen}
+              size="xs"
+              colorScheme="gray"
+              color="gray.500"
+              variant="ghost"
+            />
+          </Tooltip>
+          <ExpandedModal cell={cell} disclosure={modalDisclosure} />
+        </>
+      )}
+      {canModify && (
+        <Tooltip label="Refetch output">
+          <IconButton
+            size="xs"
+            color="gray.500"
+            variant="ghost"
+            cursor="pointer"
+            onClick={refetchOutput}
+            aria-label="refetch output"
+            icon={<Icon as={refetchingOutput ? Spinner : BsArrowClockwise} boxSize={4} />}
+          />
+        </Tooltip>
+      )}
+    </HStack>
+  );
+};

src/components/OutputsTable/ScenarioEditor.tsx

@@ -1,23 +1,35 @@
-import { type DragEvent } from "react";
+import { isEqual } from "lodash-es";
+import { useEffect, useState, type DragEvent } from "react";
 import { api } from "~/utils/api";
-import { isEqual } from "lodash";
+import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
 import { type Scenario } from "./types";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-import { useState } from "react";
 
-import { Box, Button, Flex, HStack, Icon, Spinner, Stack, Tooltip, VStack } from "@chakra-ui/react";
+import {
+  Box,
+  Button,
+  HStack,
+  Icon,
+  IconButton,
+  Spinner,
+  Text,
+  Tooltip,
+  VStack,
+} from "@chakra-ui/react";
+import { BsArrowsAngleExpand, BsX } from "react-icons/bs";
 import { cellPadding } from "../constants";
-import { BsX } from "react-icons/bs";
+import { FloatingLabelInput } from "./FloatingLabelInput";
-import { RiDraggable } from "react-icons/ri";
+import { ScenarioEditorModal } from "./ScenarioEditorModal";
-import AutoResizeTextArea from "../AutoResizeTextArea";
 
 export default function ScenarioEditor({
   scenario,
-  hovered,
+  ...props
 }: {
   scenario: Scenario;
   hovered: boolean;
+  canHide: boolean;
 }) {
+  const { canModify } = useExperimentAccess();
+
   const savedValues = scenario.variableValues as Record<string, string>;
   const utils = api.useContext();
   const [isDragTarget, setIsDragTarget] = useState(false);
@@ -25,6 +37,10 @@ export default function ScenarioEditor({
 
   const [values, setValues] = useState<Record<string, string>>(savedValues);
 
+  useEffect(() => {
+    if (savedValues) setValues(savedValues);
+  }, [savedValues]);
+
   const experiment = useExperiment();
   const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
 
@@ -68,93 +84,86 @@ export default function ScenarioEditor({
     [reorderMutation, scenario.id],
   );
 
+  const [scenarioEditorModalOpen, setScenarioEditorModalOpen] = useState(false);
+
   return (
-    <HStack
+    <>
-      alignItems="flex-start"
+      <HStack
-      pr={cellPadding.x}
+        alignItems="flex-start"
-      py={cellPadding.y}
+        px={cellPadding.x}
-      height="100%"
+        py={cellPadding.y}
-      draggable={!variableInputHovered}
+        spacing={0}
-      onDragStart={(e) => {
+        height="100%"
-        e.dataTransfer.setData("text/plain", scenario.id);
+        draggable={!variableInputHovered}
-        e.currentTarget.style.opacity = "0.4";
+        onDragStart={(e) => {
-      }}
+          e.dataTransfer.setData("text/plain", scenario.id);
-      onDragEnd={(e) => {
+          e.currentTarget.style.opacity = "0.4";
-        e.currentTarget.style.opacity = "1";
+        }}
-      }}
+        onDragEnd={(e) => {
-      onDragOver={(e) => {
+          e.currentTarget.style.opacity = "1";
-        e.preventDefault();
+        }}
-        setIsDragTarget(true);
+        onDragOver={(e) => {
-      }}
+          e.preventDefault();
-      onDragLeave={() => {
+          setIsDragTarget(true);
-        setIsDragTarget(false);
+        }}
-      }}
+        onDragLeave={() => {
-      onDrop={onReorder}
+          setIsDragTarget(false);
-      backgroundColor={isDragTarget ? "gray.100" : "transparent"}
+        }}
-    >
+        onDrop={onReorder}
-      <Stack alignSelf="flex-start" opacity={hovered ? 1 : 0} spacing={0}>
+        backgroundColor={isDragTarget ? "gray.100" : "transparent"}
-        <Tooltip label="Hide scenario" hasArrow>
+      >
-          {/* for some reason the tooltip can't position itself properly relative to the icon without the wrapping box */}
+        {variableLabels.length === 0 ? (
-          <Button
+          <Box color="gray.500">
-            variant="unstyled"
+            {vars.data ? "No scenario variables configured" : "Loading..."}
-            color="gray.400"
+          </Box>
-            height="unset"
+        ) : (
-            width="unset"
+          <VStack spacing={4} flex={1} py={2}>
-            minW="unset"
+            <HStack justifyContent="space-between" w="100%" align="center" spacing={0}>
-            onClick={onHide}
+              <Text flex={1}>Scenario</Text>
-            _hover={{
+              <Tooltip label="Expand" hasArrow>
-              color: "gray.800",
+                <IconButton
-              cursor: "pointer",
+                  aria-label="Expand"
-            }}
+                  icon={<Icon as={BsArrowsAngleExpand} boxSize={3} />}
-          >
+                  onClick={() => setScenarioEditorModalOpen(true)}
-            <Icon as={hidingInProgress ? Spinner : BsX} boxSize={6} />
+                  size="xs"
-          </Button>
+                  colorScheme="gray"
-        </Tooltip>
+                  color="gray.500"
-        <Icon
+                  variant="ghost"
-          as={RiDraggable}
+                />
-          boxSize={6}
+              </Tooltip>
-          color="gray.400"
+              {canModify && props.canHide && (
-          _hover={{ color: "gray.800", cursor: "pointer" }}
+                <Tooltip label="Delete" hasArrow>
-        />
+                  <IconButton
-      </Stack>
+                    aria-label="Delete"
-      {variableLabels.length === 0 ? (
+                    icon={
-        <Box color="gray.500">{vars.data ? "No scenario variables configured" : "Loading..."}</Box>
+                      <Icon
-      ) : (
+                        as={hidingInProgress ? Spinner : BsX}
-        <VStack spacing={1}>
+                        boxSize={hidingInProgress ? 4 : 6}
-          {variableLabels.map((key) => {
+                      />
-            const value = values[key] ?? "";
+                    }
-            const layoutDirection = value.length > 20 ? "column" : "row";
+                    onClick={onHide}
-            return (
+                    size="xs"
-              <Flex
+                    display="flex"
-                key={key}
+                    colorScheme="gray"
-                direction={layoutDirection}
+                    color="gray.500"
-                alignItems={layoutDirection === "column" ? "flex-start" : "center"}
+                    variant="ghost"
-                flexWrap="wrap"
+                  />
-                width="full"
+                </Tooltip>
-              >
+              )}
-                <Box
+            </HStack>
-                  bgColor="blue.100"
+            {variableLabels.map((key) => {
-                  color="blue.600"
+              const value = values[key] ?? "";
-                  px={1}
+              return (
-                  my="3px"
+                <FloatingLabelInput
-                  fontSize="xs"
+                  key={key}
-                  fontWeight="bold"
+                  label={key}
-                >
+                  isDisabled={!canModify}
-                  {key}
+                  style={{ width: "100%" }}
-                </Box>
+                  maxHeight={32}
-                <AutoResizeTextArea
-                  px={2}
-                  py={1}
-                  placeholder="empty"
-                  borderRadius="sm"
-                  fontSize="sm"
-                  lineHeight={1.2}
                   value={value}
                   onChange={(e) => {
                     setValues((prev) => ({ ...prev, [key]: e.target.value }));
                   }}
-                  maxH="32"
-                  overflowY="auto"
                   onKeyDown={(e) => {
                     if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
                       e.preventDefault();
@@ -162,36 +171,37 @@ export default function ScenarioEditor({
                       onSave();
                     }
                   }}
-                  resize="none"
-                  overflow="hidden"
-                  flex={layoutDirection === "row" ? 1 : undefined}
-                  borderColor={hasChanged ? "blue.300" : "transparent"}
-                  _hover={{ borderColor: "gray.300" }}
-                  _focus={{ borderColor: "blue.500", outline: "none", bg: "white" }}
                   onMouseEnter={() => setVariableInputHovered(true)}
                   onMouseLeave={() => setVariableInputHovered(false)}
                 />
-              </Flex>
+              );
-            );
+            })}
-          })}
+            {hasChanged && (
-          {hasChanged && (
+              <HStack justify="right">
-            <HStack justify="right">
+                <Button
-              <Button
+                  size="sm"
-                size="sm"
+                  onMouseDown={() => {
-                onMouseDown={() => {
+                    setValues(savedValues);
-                  setValues(savedValues);
+                  }}
-                }}
+                  colorScheme="gray"
-                colorScheme="gray"
+                >
-              >
+                  Reset
-                Reset
+                </Button>
-              </Button>
+                <Button size="sm" onMouseDown={onSave} colorScheme="blue">
-              <Button size="sm" onMouseDown={onSave} colorScheme="blue">
+                  Save
-                Save
+                </Button>
-              </Button>
+              </HStack>
-            </HStack>
+            )}
-          )}
+          </VStack>
-        </VStack>
+        )}
+      </HStack>
+      {scenarioEditorModalOpen && (
+        <ScenarioEditorModal
+          scenarioId={scenario.id}
+          initialValues={savedValues}
+          onClose={() => setScenarioEditorModalOpen(false)}
+        />
       )}
-    </HStack>
+    </>
   );
 }

src/components/OutputsTable/ScenarioEditorModal.tsx

@@ -0,0 +1,123 @@
+import {
+  Button,
+  HStack,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  Spinner,
+  Text,
+  VStack,
+} from "@chakra-ui/react";
+import { useEffect, useState } from "react";
+import { isEqual } from "lodash-es";
+
+import { api } from "~/utils/api";
+import {
+  useScenario,
+  useHandledAsyncCallback,
+  useExperiment,
+  useExperimentAccess,
+} from "~/utils/hooks";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+
+export const ScenarioEditorModal = ({
+  scenarioId,
+  initialValues,
+  onClose,
+}: {
+  scenarioId: string;
+  initialValues: Record<string, string>;
+  onClose: () => void;
+}) => {
+  const utils = api.useContext();
+  const experiment = useExperiment();
+  const { canModify } = useExperimentAccess();
+  const scenario = useScenario(scenarioId);
+
+  const savedValues = scenario.data?.variableValues as Record<string, string>;
+
+  const [values, setValues] = useState<Record<string, string>>(initialValues);
+
+  useEffect(() => {
+    if (savedValues) setValues(savedValues);
+  }, [savedValues]);
+
+  const hasChanged = !isEqual(savedValues, values);
+
+  const mutation = api.scenarios.replaceWithValues.useMutation();
+
+  const [onSave, saving] = useHandledAsyncCallback(async () => {
+    await mutation.mutateAsync({
+      id: scenarioId,
+      values,
+    });
+    await utils.scenarios.list.invalidate();
+  }, [mutation, values]);
+
+  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const variableLabels = vars.data?.map((v) => v.label) ?? [];
+
+  return (
+    <Modal
+      isOpen
+      onClose={onClose}
+      size={{ base: "xl", sm: "2xl", md: "3xl", lg: "5xl", xl: "7xl" }}
+    >
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader />
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack spacing={8}>
+            {values &&
+              variableLabels.map((key) => {
+                const value = values[key] ?? "";
+                return (
+                  <FloatingLabelInput
+                    key={key}
+                    label={key}
+                    isDisabled={!canModify}
+                    _disabled={{ opacity: 1 }}
+                    style={{ width: "100%" }}
+                    value={value}
+                    onChange={(e) => {
+                      setValues((prev) => ({ ...prev, [key]: e.target.value }));
+                    }}
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
+                        e.preventDefault();
+                        e.currentTarget.blur();
+                        onSave();
+                      }
+                    }}
+                  />
+                );
+              })}
+          </VStack>
+        </ModalBody>
+
+        <ModalFooter>
+          {canModify && (
+            <HStack>
+              <Button
+                colorScheme="gray"
+                onClick={() => setValues(savedValues)}
+                minW={24}
+                isDisabled={!hasChanged}
+              >
+                <Text>Reset</Text>
+              </Button>
+              <Button colorScheme="blue" onClick={onSave} minW={24} isDisabled={!hasChanged}>
+                {saving ? <Spinner boxSize={4} /> : <Text>Save</Text>}
+              </Button>
+            </HStack>
+          )}
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

src/components/OutputsTable/ScenarioPaginator.tsx

@@ -0,0 +1,21 @@
+import { useScenarios } from "~/utils/hooks";
+import Paginator from "../Paginator";
+
+const ScenarioPaginator = () => {
+  const { data } = useScenarios();
+
+  if (!data) return null;
+
+  const { scenarios, startIndex, lastPage, count } = data;
+
+  return (
+    <Paginator
+      numItemsLoaded={scenarios.length}
+      startIndex={startIndex}
+      lastPage={lastPage}
+      count={count}
+    />
+  );
+};
+
+export default ScenarioPaginator;

src/components/OutputsTable/ScenarioRow.tsx

@@ -1,11 +1,16 @@
-import { Box, GridItem } from "@chakra-ui/react";
+import { GridItem } from "@chakra-ui/react";
 import React, { useState } from "react";
-import { cellPadding } from "../constants";
 import OutputCell from "./OutputCell/OutputCell";
 import ScenarioEditor from "./ScenarioEditor";
 import type { PromptVariant, Scenario } from "./types";
+import { borders } from "./styles";
 
-const ScenarioRow = (props: { scenario: Scenario; variants: PromptVariant[] }) => {
+const ScenarioRow = (props: {
+  scenario: Scenario;
+  variants: PromptVariant[];
+  canHide: boolean;
+  rowStart: number;
+}) => {
   const [isHovered, setIsHovered] = useState(false);
 
   const highlightStyle = { backgroundColor: "gray.50" };
@@ -17,19 +22,23 @@ const ScenarioRow = (props: { scenario: Scenario; variants: PromptVariant[] }) =
         onMouseLeave={() => setIsHovered(false)}
         sx={isHovered ? highlightStyle : undefined}
         borderLeftWidth={1}
+        {...borders}
+        rowStart={props.rowStart}
+        colStart={1}
       >
-        <ScenarioEditor scenario={props.scenario} hovered={isHovered} />
+        <ScenarioEditor scenario={props.scenario} hovered={isHovered} canHide={props.canHide} />
       </GridItem>
-      {props.variants.map((variant) => (
+      {props.variants.map((variant, i) => (
         <GridItem
           key={variant.id}
           onMouseEnter={() => setIsHovered(true)}
           onMouseLeave={() => setIsHovered(false)}
           sx={isHovered ? highlightStyle : undefined}
+          rowStart={props.rowStart}
+          colStart={i + 2}
+          {...borders}
         >
-          <Box h="100%" w="100%" px={cellPadding.x} py={cellPadding.y}>
+          <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />
-            <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />
-          </Box>
         </GridItem>
       ))}
     </>

src/components/OutputsTable/ScenariosHeader.tsx

@@ -0,0 +1,82 @@
+import {
+  Button,
+  type ButtonProps,
+  HStack,
+  Text,
+  Icon,
+  Menu,
+  MenuButton,
+  MenuList,
+  MenuItem,
+  IconButton,
+  Spinner,
+} from "@chakra-ui/react";
+import { cellPadding } from "../constants";
+import {
+  useExperiment,
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useScenarios,
+} from "~/utils/hooks";
+import { BsGear, BsPencil, BsPlus, BsStars } from "react-icons/bs";
+import { useAppStore } from "~/state/store";
+import { api } from "~/utils/api";
+
+export const ActionButton = (props: ButtonProps) => (
+  <Button size="sm" variant="ghost" color="gray.600" {...props} />
+);
+
+export const ScenariosHeader = () => {
+  const openDrawer = useAppStore((s) => s.openDrawer);
+  const { canModify } = useExperimentAccess();
+  const scenarios = useScenarios();
+
+  const experiment = useExperiment();
+  const createScenarioMutation = api.scenarios.create.useMutation();
+  const utils = api.useContext();
+
+  const [onAddScenario, loading] = useHandledAsyncCallback(
+    async (autogenerate: boolean) => {
+      if (!experiment.data) return;
+      await createScenarioMutation.mutateAsync({
+        experimentId: experiment.data.id,
+        autogenerate,
+      });
+      await utils.scenarios.list.invalidate();
+    },
+    [createScenarioMutation],
+  );
+
+  return (
+    <HStack w="100%" pb={cellPadding.y} pt={0} align="center" spacing={0}>
+      <Text fontSize={16} fontWeight="bold">
+        Scenarios ({scenarios.data?.count})
+      </Text>
+      {canModify && (
+        <Menu>
+          <MenuButton
+            as={IconButton}
+            mt={1}
+            variant="ghost"
+            aria-label="Edit Scenarios"
+            icon={<Icon as={loading ? Spinner : BsGear} />}
+          />
+          <MenuList fontSize="md" zIndex="dropdown" mt={-3}>
+            <MenuItem
+              icon={<Icon as={BsPlus} boxSize={6} mx="-5px" />}
+              onClick={() => onAddScenario(false)}
+            >
+              Add Scenario
+            </MenuItem>
+            <MenuItem icon={<BsStars />} onClick={() => onAddScenario(true)}>
+              Autogenerate Scenario
+            </MenuItem>
+            <MenuItem icon={<BsPencil />} onClick={openDrawer}>
+              Edit Vars
+            </MenuItem>
+          </MenuList>
+        </Menu>
+      )}
+    </HStack>
+  );
+};

src/components/OutputsTable/VariantEditor.tsx

@@ -1,32 +1,76 @@
-import { Box, Button, HStack, Tooltip, useToast } from "@chakra-ui/react";
+import {
-import { useRef, useEffect, useState, useCallback } from "react";
+  Box,
-import { useHandledAsyncCallback, useModifierKeyLabel } from "~/utils/hooks";
+  Button,
-import { type PromptVariant } from "./types";
+  HStack,
-import { api } from "~/utils/api";
+  IconButton,
+  Spinner,
+  Text,
+  Tooltip,
+  useToast,
+} from "@chakra-ui/react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { FiMaximize, FiMinimize } from "react-icons/fi";
+import { editorBackground } from "~/state/sharedVariantEditor.slice";
 import { useAppStore } from "~/state/store";
-// import openAITypes from "~/codegen/openai.types.ts.txt";
+import { api } from "~/utils/api";
+import {
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useModifierKeyLabel,
+  useVisibleScenarioIds,
+} from "~/utils/hooks";
+import { type PromptVariant } from "./types";
 
-export default function VariantConfigEditor(props: { variant: PromptVariant }) {
+export default function VariantEditor(props: { variant: PromptVariant }) {
+  const { canModify } = useExperimentAccess();
   const monaco = useAppStore.use.sharedVariantEditor.monaco();
   const editorRef = useRef<ReturnType<NonNullable<typeof monaco>["editor"]["create"]> | null>(null);
+  const containerRef = useRef<HTMLDivElement | null>(null);
   const [editorId] = useState(() => `editor_${Math.random().toString(36).substring(7)}`);
   const [isChanged, setIsChanged] = useState(false);
 
-  const lastSavedFn = props.variant.constructFn;
+  const [isFullscreen, setIsFullscreen] = useState(false);
+
+  const toggleFullscreen = useCallback(() => {
+    setIsFullscreen((prev) => !prev);
+    editorRef.current?.focus();
+  }, [setIsFullscreen]);
+
+  useEffect(() => {
+    const handleEsc = (event: KeyboardEvent) => {
+      if (event.key === "Escape" && isFullscreen) {
+        toggleFullscreen();
+      }
+    };
+
+    window.addEventListener("keydown", handleEsc);
+    return () => window.removeEventListener("keydown", handleEsc);
+  }, [isFullscreen, toggleFullscreen]);
+
+  const lastSavedFn = props.variant.promptConstructor;
 
   const modifierKey = useModifierKeyLabel();
 
   const checkForChanges = useCallback(() => {
     if (!editorRef.current) return;
-    const currentConfig = editorRef.current.getValue();
+    const currentFn = editorRef.current.getValue();
-    setIsChanged(currentConfig !== lastSavedFn);
+    setIsChanged(currentFn.length > 0 && currentFn !== lastSavedFn);
   }, [lastSavedFn]);
 
+  const matchUpdatedSavedFn = useCallback(() => {
+    if (!editorRef.current) return;
+    editorRef.current.setValue(lastSavedFn);
+    setIsChanged(false);
+  }, [lastSavedFn]);
+
+  useEffect(matchUpdatedSavedFn, [matchUpdatedSavedFn, lastSavedFn]);
+
   const replaceVariant = api.promptVariants.replaceVariant.useMutation();
   const utils = api.useContext();
   const toast = useToast();
+  const visibleScenarios = useVisibleScenarioIds();
 
-  const [onSave] = useHandledAsyncCallback(async () => {
+  const [onSave, saveInProgress] = useHandledAsyncCallback(async () => {
     if (!editorRef.current) return;
 
     await editorRef.current.getAction("editor.action.formatDocument")?.run();
@@ -39,39 +83,33 @@ export default function VariantConfigEditor(props: { variant: PromptVariant }) {
     const model = editorRef.current.getModel();
     if (!model) return;
 
-    const markers = monaco?.editor.getModelMarkers({ resource: model.uri });
-    const hasErrors = markers?.some((m) => m.severity === monaco?.MarkerSeverity.Error);
-
-    if (hasErrors) {
-      toast({
-        title: "Invalid TypeScript",
-        description: "Please fix the TypeScript errors before saving.",
-        status: "error",
-      });
-      return;
-    }
-
     // Make sure the user defined the prompt with the string "prompt\w*=" somewhere
-    const promptRegex = /prompt\s*=/;
+    const promptRegex = /definePrompt\(/;
     if (!promptRegex.test(currentFn)) {
-      console.log("no prompt");
-      console.log(currentFn);
       toast({
         title: "Missing prompt",
-        description: "Please define the prompt (eg. `prompt = { ...`).",
+        description: "Please define the prompt (eg. `definePrompt(...`",
         status: "error",
       });
       return;
     }
 
-    await replaceVariant.mutateAsync({
+    const resp = await replaceVariant.mutateAsync({
       id: props.variant.id,
-      constructFn: currentFn,
+      promptConstructor: currentFn,
+      streamScenarios: visibleScenarios,
     });
+    if (resp.status === "error") {
+      return toast({
+        title: "Error saving variant",
+        description: resp.message,
+        status: "error",
+      });
+    }
+
+    setIsChanged(false);
 
     await utils.promptVariants.list.invalidate();
-
-    checkForChanges();
   }, [checkForChanges]);
 
   useEffect(() => {
@@ -95,13 +133,26 @@ export default function VariantConfigEditor(props: { variant: PromptVariant }) {
         wordWrapBreakAfterCharacters: "",
         wordWrapBreakBeforeCharacters: "",
         quickSuggestions: true,
+        readOnly: !canModify,
       });
 
+      // Workaround because otherwise the commands only work on whatever
+      // editor was loaded on the page last.
+      // https://github.com/microsoft/monaco-editor/issues/2947#issuecomment-1422265201
       editorRef.current.onDidFocusEditorText(() => {
-        // Workaround because otherwise the command only works on whatever
+        editorRef.current?.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, onSave);
-        // editor was loaded on the page last.
+
-        // https://github.com/microsoft/monaco-editor/issues/2947#issuecomment-1422265201
+        editorRef.current?.addCommand(
-        editorRef.current?.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter, onSave);
+          monaco.KeyMod.CtrlCmd | monaco.KeyMod.Shift | monaco.KeyCode.KeyF,
+          toggleFullscreen,
+        );
+
+        // Exit fullscreen with escape
+        editorRef.current?.addCommand(monaco.KeyCode.Escape, () => {
+          if (isFullscreen) {
+            toggleFullscreen();
+          }
+        });
       });
 
       editorRef.current.onDidChangeModelContent(checkForChanges);
@@ -122,21 +173,48 @@ export default function VariantConfigEditor(props: { variant: PromptVariant }) {
     /* eslint-disable-next-line react-hooks/exhaustive-deps */
   }, [monaco, editorId]);
 
-  // useEffect(() => {
+  useEffect(() => {
-  //   const savedConfigChanged = lastSavedFn !== savedConfig;
+    if (!editorRef.current) return;
-
+    editorRef.current.updateOptions({
-  //   lastSavedFn = savedConfig;
+      readOnly: !canModify,
-
+    });
-  //   if (savedConfigChanged && editorRef.current?.getValue() !== savedConfig) {
+  }, [canModify]);
-  //     editorRef.current?.setValue(savedConfig);
-  //   }
-
-  //   checkForChanges();
-  // }, [savedConfig, checkForChanges]);
 
   return (
-    <Box w="100%" pos="relative">
+    <Box
-      <div id={editorId} style={{ height: "300px", width: "100%" }}></div>
+      w="100%"
+      ref={containerRef}
+      sx={
+        isFullscreen
+          ? {
+              position: "fixed",
+              top: 0,
+              left: 0,
+              right: 0,
+              bottom: 0,
+            }
+          : { h: "400px", w: "100%" }
+      }
+      bgColor={editorBackground}
+      zIndex={isFullscreen ? 1000 : "unset"}
+      pos="relative"
+      _hover={{ ".fullscreen-toggle": { opacity: 1 } }}
+    >
+      <Box id={editorId} w="100%" h="100%" />
+      <Tooltip label={`${modifierKey} + ⇧ + F`}>
+        <IconButton
+          className="fullscreen-toggle"
+          aria-label="Minimize"
+          icon={isFullscreen ? <FiMinimize /> : <FiMaximize />}
+          position="absolute"
+          top={2}
+          right={2}
+          onClick={toggleFullscreen}
+          opacity={0}
+          transition="opacity 0.2s"
+        />
+      </Tooltip>
+
       {isChanged && (
         <HStack pos="absolute" bottom={2} right={2}>
           <Button
@@ -149,9 +227,9 @@ export default function VariantConfigEditor(props: { variant: PromptVariant }) {
           >
             Reset
           </Button>
-          <Tooltip label={`${modifierKey} + Enter`}>
+          <Tooltip label={`${modifierKey} + S`}>
-            <Button size="sm" onClick={onSave} colorScheme="blue">
+            <Button size="sm" onClick={onSave} colorScheme="blue" w={16} disabled={saveInProgress}>
-              Save
+              {saveInProgress ? <Spinner boxSize={4} /> : <Text>Save</Text>}
             </Button>
           </Tooltip>
         </HStack>

src/components/OutputsTable/VariantHeader.tsx

@@ -1,105 +0,0 @@
-import { useState, type DragEvent } from "react";
-import { type PromptVariant } from "./types";
-import { api } from "~/utils/api";
-import { useHandledAsyncCallback } from "~/utils/hooks";
-import { Button, HStack, Icon, Tooltip } from "@chakra-ui/react"; // Changed here
-import { BsX } from "react-icons/bs";
-import { RiDraggable } from "react-icons/ri";
-import { cellPadding, headerMinHeight } from "../constants";
-import AutoResizeTextArea from "../AutoResizeTextArea";
-
-export default function VariantHeader(props: { variant: PromptVariant }) {
-  const utils = api.useContext();
-  const [isDragTarget, setIsDragTarget] = useState(false);
-  const [isInputHovered, setIsInputHovered] = useState(false);
-  const [label, setLabel] = useState(props.variant.label);
-
-  const updateMutation = api.promptVariants.update.useMutation();
-  const [onSaveLabel] = useHandledAsyncCallback(async () => {
-    if (label && label !== props.variant.label) {
-      await updateMutation.mutateAsync({
-        id: props.variant.id,
-        updates: { label: label },
-      });
-    }
-  }, [updateMutation, props.variant.id, props.variant.label, label]);
-
-  const hideMutation = api.promptVariants.hide.useMutation();
-  const [onHide] = useHandledAsyncCallback(async () => {
-    await hideMutation.mutateAsync({
-      id: props.variant.id,
-    });
-    await utils.promptVariants.list.invalidate();
-  }, [hideMutation, props.variant.id]);
-
-  const reorderMutation = api.promptVariants.reorder.useMutation();
-  const [onReorder] = useHandledAsyncCallback(
-    async (e: DragEvent<HTMLDivElement>) => {
-      e.preventDefault();
-      setIsDragTarget(false);
-      const draggedId = e.dataTransfer.getData("text/plain");
-      const droppedId = props.variant.id;
-      if (!draggedId || !droppedId || draggedId === droppedId) return;
-      await reorderMutation.mutateAsync({
-        draggedId,
-        droppedId,
-      });
-      await utils.promptVariants.list.invalidate();
-    },
-    [reorderMutation, props.variant.id],
-  );
-
-  return (
-    <HStack
-      spacing={4}
-      alignItems="center"
-      minH={headerMinHeight}
-      draggable={!isInputHovered}
-      onDragStart={(e) => {
-        e.dataTransfer.setData("text/plain", props.variant.id);
-        e.currentTarget.style.opacity = "0.4";
-      }}
-      onDragEnd={(e) => {
-        e.currentTarget.style.opacity = "1";
-      }}
-      onDragOver={(e) => {
-        e.preventDefault();
-        setIsDragTarget(true);
-      }}
-      onDragLeave={() => {
-        setIsDragTarget(false);
-      }}
-      onDrop={onReorder}
-      backgroundColor={isDragTarget ? "gray.100" : "transparent"}
-    >
-      <Icon
-        as={RiDraggable}
-        boxSize={6}
-        color="gray.400"
-        _hover={{ color: "gray.800", cursor: "pointer" }}
-      />
-      <AutoResizeTextArea // Changed to Input
-        size="sm"
-        value={label}
-        onChange={(e) => setLabel(e.target.value)}
-        onBlur={onSaveLabel}
-        placeholder="Variant Name"
-        borderWidth={1}
-        borderColor="transparent"
-        fontWeight="bold"
-        fontSize={16}
-        _hover={{ borderColor: "gray.300" }}
-        _focus={{ borderColor: "blue.500", outline: "none" }}
-        flex={1}
-        px={cellPadding.x}
-        onMouseEnter={() => setIsInputHovered(true)}
-        onMouseLeave={() => setIsInputHovered(false)}
-      />
-      <Tooltip label="Hide Variant" hasArrow>
-        <Button variant="ghost" colorScheme="gray" size="sm" onClick={onHide}>
-          <Icon as={BsX} boxSize={6} />
-        </Button>
-      </Tooltip>
-    </HStack>
-  );
-}

src/components/OutputsTable/VariantStats.tsx

@@ -5,8 +5,10 @@ import { api } from "~/utils/api";
 import chroma from "chroma-js";
 import { BsCurrencyDollar } from "react-icons/bs";
 import { CostTooltip } from "../tooltip/CostTooltip";
+import { useEffect, useState } from "react";
 
 export default function VariantStats(props: { variant: PromptVariant }) {
+  const [refetchInterval, setRefetchInterval] = useState(0);
   const { data } = api.promptVariants.stats.useQuery(
     {
       variantId: props.variant.id,
@@ -19,10 +21,15 @@ export default function VariantStats(props: { variant: PromptVariant }) {
         completionTokens: 0,
         scenarioCount: 0,
         outputCount: 0,
+        awaitingEvals: false,
       },
+      refetchInterval,
     },
   );
 
+  // Poll every two seconds while we are waiting for LLM retrievals to finish
+  useEffect(() => setRefetchInterval(data.awaitingEvals ? 5000 : 0), [data.awaitingEvals]);
+
   const [passColor, neutralColor, failColor] = useToken("colors", [
     "green.500",
     "gray.500",
@@ -33,21 +40,25 @@ export default function VariantStats(props: { variant: PromptVariant }) {
 
   const showNumFinished = data.scenarioCount > 0 && data.scenarioCount !== data.outputCount;
 
-  if (!(data.evalResults.length > 0) && !data.overallCost) return null;
-
   return (
-    <HStack justifyContent="space-between" alignItems="center" mx="2" fontSize="xs">
+    <HStack
-      {showNumFinished && (
+      justifyContent="space-between"
-        <Text>
+      alignItems="flex-end"
-          {data.outputCount} / {data.scenarioCount}
+      mx="2"
-        </Text>
+      fontSize="xs"
-      )}
+      py={cellPadding.y}
-      <HStack px={cellPadding.x} py={cellPadding.y}>
+    >
+      <HStack px={cellPadding.x} flexWrap="wrap">
+        {showNumFinished && (
+          <Text>
+            {data.outputCount} / {data.scenarioCount}
+          </Text>
+        )}
         {data.evalResults.map((result) => {
-          const passedFrac = result.passCount / (result.passCount + result.failCount);
+          const passedFrac = result.passCount / result.totalCount;
           return (
             <HStack key={result.id}>
-              <Text>{result.evaluation.name}</Text>
+              <Text>{result.label}</Text>
               <Text color={scale(passedFrac).hex()} fontWeight="bold">
                 {(passedFrac * 100).toFixed(1)}%
               </Text>
@@ -61,7 +72,7 @@ export default function VariantStats(props: { variant: PromptVariant }) {
           completionTokens={data.completionTokens}
           cost={data.overallCost}
         >
-          <HStack spacing={0} align="center" color="gray.500" my="2">
+          <HStack spacing={0} align="center" color="gray.500">
             <Icon as={BsCurrencyDollar} />
             <Text mr={1}>{data.overallCost.toFixed(3)}</Text>
           </HStack>

src/components/OutputsTable/index.tsx

@@ -1,113 +1,107 @@
-import { Button, Grid, GridItem, HStack, Heading, type SystemStyleObject } from "@chakra-ui/react";
+import { Grid, GridItem, type GridItemProps } from "@chakra-ui/react";
 import { api } from "~/utils/api";
-import NewScenarioButton from "./NewScenarioButton";
+import AddVariantButton from "./AddVariantButton";
-import NewVariantButton from "./NewVariantButton";
 import ScenarioRow from "./ScenarioRow";
-import VariantConfigEditor from "./VariantEditor";
+import VariantEditor from "./VariantEditor";
-import VariantHeader from "./VariantHeader";
+import VariantHeader from "../VariantHeader/VariantHeader";
-import { cellPadding } from "../constants";
-import { BsPencil } from "react-icons/bs";
 import VariantStats from "./VariantStats";
-import { useAppStore } from "~/state/store";
+import { ScenariosHeader } from "./ScenariosHeader";
-
+import { borders } from "./styles";
-const stickyHeaderStyle: SystemStyleObject = {
+import { useScenarios } from "~/utils/hooks";
-  position: "sticky",
+import ScenarioPaginator from "./ScenarioPaginator";
-  top: "-1px",
+import { Fragment } from "react";
-  backgroundColor: "#fff",
-  zIndex: 1,
-};
 
 export default function OutputsTable({ experimentId }: { experimentId: string | undefined }) {
   const variants = api.promptVariants.list.useQuery(
     { experimentId: experimentId as string },
     { enabled: !!experimentId },
   );
-  const openDrawer = useAppStore((s) => s.openDrawer);
 
-  const scenarios = api.scenarios.list.useQuery(
+  const scenarios = useScenarios();
-    { experimentId: experimentId as string },
-    { enabled: !!experimentId },
-  );
 
   if (!variants.data || !scenarios.data) return null;
 
-  const allCols = variants.data.length + 1;
+  const allCols = variants.data.length + 2;
-  const headerRows = 3;
+  const variantHeaderRows = 3;
+  const scenarioHeaderRows = 1;
+  const scenarioFooterRows = 1;
+  const visibleScenariosCount = scenarios.data.scenarios.length;
+  const allRows =
+    variantHeaderRows + scenarioHeaderRows + visibleScenariosCount + scenarioFooterRows;
 
   return (
     <Grid
-      p={4}
+      pt={4}
       pb={24}
+      pl={8}
       display="grid"
-      gridTemplateColumns={`250px repeat(${variants.data.length}, minmax(300px, 1fr)) auto`}
+      gridTemplateColumns={`250px repeat(${variants.data.length}, minmax(320px, 1fr)) auto`}
       sx={{
         "> *": {
           borderColor: "gray.300",
-          borderBottomWidth: 1,
-          borderRightWidth: 1,
         },
       }}
       fontSize="sm"
     >
-      <GridItem
+      <GridItem rowSpan={variantHeaderRows}>
-        display="flex"
+        <AddVariantButton />
-        alignItems="flex-end"
-        rowSpan={headerRows}
-        px={cellPadding.x}
-        py={cellPadding.y}
-        // TODO: This is a hack to get the sticky header to work. It's not ideal because it's not responsive to the height of the header,
-        // so if the header height changes, this will need to be updated.
-        sx={{ ...stickyHeaderStyle, top: "-337px" }}
-      >
-        <HStack w="100%">
-          <Heading size="xs" fontWeight="bold" flex={1}>
-            Scenarios ({scenarios.data.length})
-          </Heading>
-          <Button
-            size="xs"
-            variant="ghost"
-            color="gray.500"
-            aria-label="Edit"
-            leftIcon={<BsPencil />}
-            onClick={openDrawer}
-          >
-            Edit Vars
-          </Button>
-        </HStack>
       </GridItem>
 
-      {variants.data.map((variant) => (
+      {variants.data.map((variant, i) => {
-        <GridItem key={variant.uiId} padding={0} sx={stickyHeaderStyle} borderTopWidth={1}>
+        const sharedProps: GridItemProps = {
-          <VariantHeader variant={variant} />
+          ...borders,
-        </GridItem>
+          colStart: i + 2,
-      ))}
+          borderLeftWidth: i === 0 ? 1 : 0,
+          marginLeft: i === 0 ? "-1px" : 0,
+          backgroundColor: "gray.100",
+        };
+        return (
+          <Fragment key={variant.uiId}>
+            <VariantHeader
+              variant={variant}
+              canHide={variants.data.length > 1}
+              rowStart={1}
+              {...sharedProps}
+            />
+            <GridItem rowStart={2} {...sharedProps}>
+              <VariantEditor variant={variant} />
+            </GridItem>
+            <GridItem rowStart={3} {...sharedProps}>
+              <VariantStats variant={variant} />
+            </GridItem>
+          </Fragment>
+        );
+      })}
+
       <GridItem
-        rowSpan={scenarios.data.length + headerRows}
+        colSpan={allCols - 1}
-        padding={0}
+        rowStart={variantHeaderRows + 1}
-        // Have to use `style` instead of emotion style props to work around css specificity issues conflicting with the "> *" selector on Grid
+        colStart={1}
-        style={{ borderRightWidth: 0, borderBottomWidth: 0 }}
+        {...borders}
-        h={8}
+        borderRightWidth={0}
-        sx={stickyHeaderStyle}
       >
-        <NewVariantButton />
+        <ScenariosHeader />
       </GridItem>
 
-      {variants.data.map((variant) => (
+      {scenarios.data.scenarios.map((scenario, i) => (
-        <GridItem key={variant.uiId}>
+        <ScenarioRow
-          <VariantConfigEditor variant={variant} />
+          rowStart={i + variantHeaderRows + scenarioHeaderRows + 2}
-        </GridItem>
+          key={scenario.uiId}
+          scenario={scenario}
+          variants={variants.data}
+          canHide={visibleScenariosCount > 1}
+        />
       ))}
-      {variants.data.map((variant) => (
+      <GridItem
-        <GridItem key={variant.uiId}>
+        rowStart={variantHeaderRows + scenarioHeaderRows + visibleScenariosCount + 2}
-          <VariantStats variant={variant} />
+        colStart={1}
-        </GridItem>
+        colSpan={allCols}
-      ))}
+      >
-      {scenarios.data.map((scenario) => (
+        <ScenarioPaginator />
-        <ScenarioRow key={scenario.uiId} scenario={scenario} variants={variants.data} />
-      ))}
-      <GridItem borderBottomWidth={0} borderRightWidth={0} w="100%" colSpan={allCols} padding={0}>
-        <NewScenarioButton />
       </GridItem>
+
+      {/* Add some extra padding on the right, because when the table is too wide to fit in the viewport `pr` on the Grid isn't respected. */}
+      <GridItem rowStart={1} colStart={allCols} rowSpan={allRows} w={4} borderBottomWidth={0} />
     </Grid>
   );
 }

src/components/OutputsTable/styles.ts

@@ -0,0 +1,6 @@
+import { type GridItemProps } from "@chakra-ui/react";
+
+export const borders: GridItemProps = {
+  borderRightWidth: 1,
+  borderBottomWidth: 1,
+};

src/components/OutputsTable/types.ts

@@ -2,4 +2,4 @@ import { type RouterOutputs } from "~/utils/api";
 
 export type PromptVariant = NonNullable<RouterOutputs["promptVariants"]["list"]>[0];
 
-export type Scenario = NonNullable<RouterOutputs["scenarios"]["list"]>[0];
+export type Scenario = NonNullable<RouterOutputs["scenarios"]["list"]>["scenarios"][0];

src/components/Paginator.tsx

@@ -0,0 +1,79 @@
+import { Box, HStack, IconButton } from "@chakra-ui/react";
+import {
+  BsChevronDoubleLeft,
+  BsChevronDoubleRight,
+  BsChevronLeft,
+  BsChevronRight,
+} from "react-icons/bs";
+import { usePage } from "~/utils/hooks";
+
+const Paginator = ({
+  numItemsLoaded,
+  startIndex,
+  lastPage,
+  count,
+}: {
+  numItemsLoaded: number;
+  startIndex: number;
+  lastPage: number;
+  count: number;
+}) => {
+  const [page, setPage] = usePage();
+
+  const nextPage = () => {
+    if (page < lastPage) {
+      setPage(page + 1, "replace");
+    }
+  };
+
+  const prevPage = () => {
+    if (page > 1) {
+      setPage(page - 1, "replace");
+    }
+  };
+
+  const goToLastPage = () => setPage(lastPage, "replace");
+  const goToFirstPage = () => setPage(1, "replace");
+
+  return (
+    <HStack pt={4}>
+      <IconButton
+        variant="ghost"
+        size="sm"
+        onClick={goToFirstPage}
+        isDisabled={page === 1}
+        aria-label="Go to first page"
+        icon={<BsChevronDoubleLeft />}
+      />
+      <IconButton
+        variant="ghost"
+        size="sm"
+        onClick={prevPage}
+        isDisabled={page === 1}
+        aria-label="Previous page"
+        icon={<BsChevronLeft />}
+      />
+      <Box>
+        {startIndex}-{startIndex + numItemsLoaded - 1} / {count}
+      </Box>
+      <IconButton
+        variant="ghost"
+        size="sm"
+        onClick={nextPage}
+        isDisabled={page === lastPage}
+        aria-label="Next page"
+        icon={<BsChevronRight />}
+      />
+      <IconButton
+        variant="ghost"
+        size="sm"
+        onClick={goToLastPage}
+        isDisabled={page === lastPage}
+        aria-label="Go to last page"
+        icon={<BsChevronDoubleRight />}
+      />
+    </HStack>
+  );
+};
+
+export default Paginator;

src/components/PublicPlaygroundWarning.tsx

@@ -1,21 +0,0 @@
-import { Flex, Icon, Link, Text } from "@chakra-ui/react";
-import { BsExclamationTriangleFill } from "react-icons/bs";
-import { env } from "~/env.mjs";
-
-export default function PublicPlaygroundWarning() {
-  if (!env.NEXT_PUBLIC_IS_PUBLIC_PLAYGROUND) return null;
-
-  return (
-    <Flex bgColor="red.600" color="whiteAlpha.900" p={2} align="center">
-      <Icon boxSize={4} mr={2} as={BsExclamationTriangleFill} />
-      <Text>
-        Warning: this is a public playground. Anyone can see, edit or delete your experiments. For
-        private use,{" "}
-        <Link textDecor="underline" href="https://github.com/openpipe/openpipe" target="_blank">
-          run a local copy
-        </Link>
-        .
-      </Text>
-    </Flex>
-  );
-}

src/components/RefinePromptModal/CompareFunctions.tsx

@@ -0,0 +1,59 @@
+import { type StackProps, VStack, useBreakpointValue } from "@chakra-ui/react";
+import React from "react";
+import DiffViewer, { DiffMethod } from "react-diff-viewer";
+import Prism from "prismjs";
+import "prismjs/components/prism-javascript";
+import "prismjs/themes/prism.css"; // choose a theme you like
+
+const highlightSyntax = (str: string) => {
+  let highlighted;
+  try {
+    highlighted = Prism.highlight(str, Prism.languages.javascript as Prism.Grammar, "javascript");
+  } catch (e) {
+    console.error("Error highlighting:", e);
+    highlighted = str;
+  }
+  return <pre style={{ display: "inline" }} dangerouslySetInnerHTML={{ __html: highlighted }} />;
+};
+
+const CompareFunctions = ({
+  originalFunction,
+  newFunction = "",
+  leftTitle = "Original",
+  rightTitle = "Modified",
+  ...props
+}: {
+  originalFunction: string;
+  newFunction?: string;
+  leftTitle?: string;
+  rightTitle?: string;
+} & StackProps) => {
+  const showSplitView = useBreakpointValue(
+    {
+      base: false,
+      md: true,
+    },
+    {
+      fallback: "base",
+    },
+  );
+
+  return (
+    <VStack w="full" spacing={4} fontSize={12} lineHeight={1} overflowY="auto" {...props}>
+      <DiffViewer
+        oldValue={originalFunction}
+        newValue={newFunction || originalFunction}
+        splitView={showSplitView}
+        hideLineNumbers={!showSplitView}
+        leftTitle={leftTitle}
+        rightTitle={rightTitle}
+        disableWordDiff={true}
+        compareMethod={DiffMethod.CHARS}
+        renderContent={highlightSyntax}
+        showDiffOnly={false}
+      />
+    </VStack>
+  );
+};
+
+export default CompareFunctions;

src/components/RefinePromptModal/RefineAction.tsx

@@ -0,0 +1,65 @@
+import { HStack, Icon, Heading, Text, VStack, GridItem } from "@chakra-ui/react";
+import { type IconType } from "react-icons";
+import { BsStars } from "react-icons/bs";
+
+export const RefineAction = ({
+  label,
+  icon,
+  desciption,
+  activeLabel,
+  onClick,
+  loading,
+}: {
+  label: string;
+  icon?: IconType;
+  desciption: string;
+  activeLabel: string | undefined;
+  onClick: (label: string) => void;
+  loading: boolean;
+}) => {
+  const isActive = activeLabel === label;
+
+  return (
+    <GridItem w="80" h="44">
+      <VStack
+        w="full"
+        h="full"
+        onClick={() => {
+          !loading && onClick(label);
+        }}
+        borderColor={isActive ? "blue.500" : "gray.200"}
+        borderWidth={2}
+        borderRadius={16}
+        padding={6}
+        backgroundColor="gray.50"
+        _hover={
+          loading
+            ? undefined
+            : {
+                backgroundColor: "gray.100",
+              }
+        }
+        spacing={8}
+        boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
+        cursor="pointer"
+        opacity={loading ? 0.5 : 1}
+      >
+        <HStack cursor="pointer" spacing={6} fontSize="sm" fontWeight="medium" color="gray.500">
+          <Icon as={icon || BsStars} boxSize={12} />
+          <Heading size="md" fontFamily="inconsolata, monospace">
+            {label}
+          </Heading>
+        </HStack>
+        <Text
+          fontSize="sm"
+          color="gray.500"
+          flexWrap="wrap"
+          wordBreak="break-word"
+          overflowWrap="break-word"
+        >
+          {desciption}
+        </Text>
+      </VStack>
+    </GridItem>
+  );
+};

src/components/RefinePromptModal/RefinePromptModal.tsx

@@ -0,0 +1,151 @@
+import {
+  Button,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  VStack,
+  Text,
+  Spinner,
+  HStack,
+  Icon,
+  SimpleGrid,
+} from "@chakra-ui/react";
+import { BsStars } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { type PromptVariant } from "@prisma/client";
+import { useState } from "react";
+import CompareFunctions from "./CompareFunctions";
+import { CustomInstructionsInput } from "../CustomInstructionsInput";
+import { RefineAction } from "./RefineAction";
+import { isObject, isString } from "lodash-es";
+import { type RefinementAction, type SupportedProvider } from "~/modelProviders/types";
+import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+
+export const RefinePromptModal = ({
+  variant,
+  onClose,
+}: {
+  variant: PromptVariant;
+  onClose: () => void;
+}) => {
+  const utils = api.useContext();
+  const visibleScenarios = useVisibleScenarioIds();
+
+  const refinementActions =
+    frontendModelProviders[variant.modelProvider as SupportedProvider].refinementActions || {};
+
+  const { mutateAsync: getModifiedPromptMutateAsync, data: refinedPromptFn } =
+    api.promptVariants.getModifiedPromptFn.useMutation();
+  const [instructions, setInstructions] = useState<string>("");
+
+  const [activeRefineActionLabel, setActiveRefineActionLabel] = useState<string | undefined>(
+    undefined,
+  );
+
+  const [getModifiedPromptFn, modificationInProgress] = useHandledAsyncCallback(
+    async (label?: string) => {
+      if (!variant.experimentId) return;
+      const updatedInstructions = label
+        ? (refinementActions[label] as RefinementAction).instructions
+        : instructions;
+      setActiveRefineActionLabel(label);
+      await getModifiedPromptMutateAsync({
+        id: variant.id,
+        instructions: updatedInstructions,
+      });
+    },
+    [getModifiedPromptMutateAsync, onClose, variant, instructions, setActiveRefineActionLabel],
+  );
+
+  const replaceVariantMutation = api.promptVariants.replaceVariant.useMutation();
+
+  const [replaceVariant, replacementInProgress] = useHandledAsyncCallback(async () => {
+    if (
+      !variant.experimentId ||
+      !refinedPromptFn ||
+      (isObject(refinedPromptFn) && "status" in refinedPromptFn)
+    )
+      return;
+    await replaceVariantMutation.mutateAsync({
+      id: variant.id,
+      promptConstructor: refinedPromptFn,
+      streamScenarios: visibleScenarios,
+    });
+    await utils.promptVariants.list.invalidate();
+    onClose();
+  }, [replaceVariantMutation, variant, onClose, refinedPromptFn]);
+
+  return (
+    <Modal
+      isOpen
+      onClose={onClose}
+      size={{ base: "xl", sm: "2xl", md: "3xl", lg: "5xl", xl: "7xl" }}
+    >
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={BsStars} />
+            <Text>Refine with GPT-4</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack spacing={8}>
+            <VStack spacing={4} w="full">
+              {Object.keys(refinementActions).length && (
+                <>
+                  <SimpleGrid columns={{ base: 1, md: 2 }} spacing={8}>
+                    {Object.keys(refinementActions).map((label) => (
+                      <RefineAction
+                        key={label}
+                        label={label}
+                        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                        icon={refinementActions[label]!.icon}
+                        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                        desciption={refinementActions[label]!.description}
+                        activeLabel={activeRefineActionLabel}
+                        onClick={getModifiedPromptFn}
+                        loading={modificationInProgress}
+                      />
+                    ))}
+                  </SimpleGrid>
+                  <Text color="gray.500">or</Text>
+                </>
+              )}
+              <CustomInstructionsInput
+                instructions={instructions}
+                setInstructions={setInstructions}
+                loading={modificationInProgress}
+                onSubmit={() => getModifiedPromptFn()}
+              />
+            </VStack>
+            <CompareFunctions
+              originalFunction={variant.promptConstructor}
+              newFunction={isString(refinedPromptFn) ? refinedPromptFn : undefined}
+              maxH="40vh"
+            />
+          </VStack>
+        </ModalBody>
+
+        <ModalFooter>
+          <HStack spacing={4}>
+            <Button
+              colorScheme="blue"
+              onClick={replaceVariant}
+              minW={24}
+              isDisabled={replacementInProgress || !refinedPromptFn}
+            >
+              {replacementInProgress ? <Spinner boxSize={4} /> : <Text>Accept</Text>}
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

src/components/VariantHeader/VariantHeader.tsx

@@ -0,0 +1,141 @@
+import { useState, type DragEvent } from "react";
+import { type PromptVariant } from "../OutputsTable/types";
+import { api } from "~/utils/api";
+import { RiDraggable } from "react-icons/ri";
+import { useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
+import { HStack, Icon, Text, GridItem, type GridItemProps } from "@chakra-ui/react"; // Changed here
+import { cellPadding, headerMinHeight } from "../constants";
+import AutoResizeTextArea from "../AutoResizeTextArea";
+import VariantHeaderMenuButton from "./VariantHeaderMenuButton";
+
+export default function VariantHeader(
+  allProps: {
+    variant: PromptVariant;
+    canHide: boolean;
+  } & GridItemProps,
+) {
+  const { variant, canHide, ...gridItemProps } = allProps;
+  const { canModify } = useExperimentAccess();
+  const utils = api.useContext();
+  const [isDragTarget, setIsDragTarget] = useState(false);
+  const [isInputHovered, setIsInputHovered] = useState(false);
+  const [label, setLabel] = useState(variant.label);
+
+  const updateMutation = api.promptVariants.update.useMutation();
+  const [onSaveLabel] = useHandledAsyncCallback(async () => {
+    if (label && label !== variant.label) {
+      await updateMutation.mutateAsync({
+        id: variant.id,
+        updates: { label: label },
+      });
+    }
+  }, [updateMutation, variant.id, variant.label, label]);
+
+  const reorderMutation = api.promptVariants.reorder.useMutation();
+  const [onReorder] = useHandledAsyncCallback(
+    async (e: DragEvent<HTMLDivElement>) => {
+      e.preventDefault();
+      setIsDragTarget(false);
+      const draggedId = e.dataTransfer.getData("text/plain");
+      const droppedId = variant.id;
+      if (!draggedId || !droppedId || draggedId === droppedId) return;
+      await reorderMutation.mutateAsync({
+        draggedId,
+        droppedId,
+      });
+      await utils.promptVariants.list.invalidate();
+    },
+    [reorderMutation, variant.id],
+  );
+
+  const [menuOpen, setMenuOpen] = useState(false);
+
+  if (!canModify) {
+    return (
+      <GridItem
+        padding={0}
+        sx={{
+          position: "sticky",
+          top: "0",
+          // Ensure that the menu always appears above the sticky header of other variants
+          zIndex: menuOpen ? "dropdown" : 10,
+        }}
+        borderTopWidth={1}
+        {...gridItemProps}
+      >
+        <Text fontSize={16} fontWeight="bold" px={cellPadding.x} py={cellPadding.y}>
+          {variant.label}
+        </Text>
+      </GridItem>
+    );
+  }
+
+  return (
+    <GridItem
+      padding={0}
+      sx={{
+        position: "sticky",
+        top: "0",
+        // Ensure that the menu always appears above the sticky header of other variants
+        zIndex: menuOpen ? "dropdown" : 10,
+      }}
+      borderTopWidth={1}
+      {...gridItemProps}
+    >
+      <HStack
+        spacing={2}
+        alignItems="flex-start"
+        minH={headerMinHeight}
+        draggable={!isInputHovered}
+        onDragStart={(e) => {
+          e.dataTransfer.setData("text/plain", variant.id);
+          e.currentTarget.style.opacity = "0.4";
+        }}
+        onDragEnd={(e) => {
+          e.currentTarget.style.opacity = "1";
+        }}
+        onDragOver={(e) => {
+          e.preventDefault();
+          setIsDragTarget(true);
+        }}
+        onDragLeave={() => {
+          setIsDragTarget(false);
+        }}
+        onDrop={onReorder}
+        backgroundColor={isDragTarget ? "gray.200" : "gray.100"}
+        h="full"
+      >
+        <Icon
+          as={RiDraggable}
+          boxSize={6}
+          mt={2}
+          color="gray.400"
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+        />
+        <AutoResizeTextArea
+          size="sm"
+          value={label}
+          onChange={(e) => setLabel(e.target.value)}
+          onBlur={onSaveLabel}
+          placeholder="Variant Name"
+          borderWidth={1}
+          borderColor="transparent"
+          fontWeight="bold"
+          fontSize={16}
+          _hover={{ borderColor: "gray.300" }}
+          _focus={{ borderColor: "blue.500", outline: "none" }}
+          flex={1}
+          px={cellPadding.x}
+          onMouseEnter={() => setIsInputHovered(true)}
+          onMouseLeave={() => setIsInputHovered(false)}
+        />
+        <VariantHeaderMenuButton
+          variant={variant}
+          canHide={canHide}
+          menuOpen={menuOpen}
+          setMenuOpen={setMenuOpen}
+        />
+      </HStack>
+    </GridItem>
+  );
+}

src/components/VariantHeader/VariantHeaderMenuButton.tsx

@@ -0,0 +1,107 @@
+import { type PromptVariant } from "../OutputsTable/types";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import {
+  Icon,
+  Menu,
+  MenuButton,
+  MenuItem,
+  MenuList,
+  MenuDivider,
+  Text,
+  Spinner,
+  IconButton,
+} from "@chakra-ui/react";
+import { BsFillTrashFill, BsGear, BsStars } from "react-icons/bs";
+import { FaRegClone } from "react-icons/fa";
+import { useState } from "react";
+import { RefinePromptModal } from "../RefinePromptModal/RefinePromptModal";
+import { RiExchangeFundsFill } from "react-icons/ri";
+import { ChangeModelModal } from "../ChangeModelModal/ChangeModelModal";
+
+export default function VariantHeaderMenuButton({
+  variant,
+  canHide,
+  menuOpen,
+  setMenuOpen,
+}: {
+  variant: PromptVariant;
+  canHide: boolean;
+  menuOpen: boolean;
+  setMenuOpen: (open: boolean) => void;
+}) {
+  const utils = api.useContext();
+
+  const duplicateMutation = api.promptVariants.create.useMutation();
+  const visibleScenarios = useVisibleScenarioIds();
+
+  const [duplicateVariant, duplicationInProgress] = useHandledAsyncCallback(async () => {
+    await duplicateMutation.mutateAsync({
+      experimentId: variant.experimentId,
+      variantId: variant.id,
+      streamScenarios: visibleScenarios,
+    });
+    await utils.promptVariants.list.invalidate();
+  }, [duplicateMutation, variant.experimentId, variant.id]);
+
+  const hideMutation = api.promptVariants.hide.useMutation();
+  const [onHide] = useHandledAsyncCallback(async () => {
+    await hideMutation.mutateAsync({
+      id: variant.id,
+    });
+    await utils.promptVariants.list.invalidate();
+  }, [hideMutation, variant.id]);
+
+  const [changeModelModalOpen, setChangeModelModalOpen] = useState(false);
+  const [refinePromptModalOpen, setRefinePromptModalOpen] = useState(false);
+
+  return (
+    <>
+      <Menu isOpen={menuOpen} onOpen={() => setMenuOpen(true)} onClose={() => setMenuOpen(false)}>
+        <MenuButton
+          as={IconButton}
+          variant="ghost"
+          aria-label="Edit Scenarios"
+          icon={<Icon as={duplicationInProgress ? Spinner : BsGear} />}
+        />
+
+        <MenuList mt={-3} fontSize="md">
+          <MenuItem icon={<Icon as={FaRegClone} boxSize={4} w={5} />} onClick={duplicateVariant}>
+            Duplicate
+          </MenuItem>
+          <MenuItem
+            icon={<Icon as={RiExchangeFundsFill} boxSize={5} />}
+            onClick={() => setChangeModelModalOpen(true)}
+          >
+            Change Model
+          </MenuItem>
+          <MenuItem
+            icon={<Icon as={BsStars} boxSize={5} />}
+            onClick={() => setRefinePromptModalOpen(true)}
+          >
+            Refine
+          </MenuItem>
+          {canHide && (
+            <>
+              <MenuDivider />
+              <MenuItem
+                onClick={onHide}
+                icon={<Icon as={BsFillTrashFill} boxSize={5} />}
+                color="red.600"
+                _hover={{ backgroundColor: "red.50" }}
+              >
+                <Text>Hide</Text>
+              </MenuItem>
+            </>
+          )}
+        </MenuList>
+      </Menu>
+      {changeModelModalOpen && (
+        <ChangeModelModal variant={variant} onClose={() => setChangeModelModalOpen(false)} />
+      )}
+      {refinePromptModalOpen && (
+        <RefinePromptModal variant={variant} onClose={() => setRefinePromptModalOpen(false)} />
+      )}
+    </>
+  );
+}

src/components/datasets/DatasetCard.tsx

@@ -0,0 +1,110 @@
+import {
+  HStack,
+  Icon,
+  VStack,
+  Text,
+  Divider,
+  Spinner,
+  AspectRatio,
+  SkeletonText,
+} from "@chakra-ui/react";
+import { RiDatabase2Line } from "react-icons/ri";
+import { formatTimePast } from "~/utils/dayjs";
+import Link from "next/link";
+import { useRouter } from "next/router";
+import { BsPlusSquare } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback } from "~/utils/hooks";
+
+type DatasetData = {
+  name: string;
+  numEntries: number;
+  id: string;
+  createdAt: Date;
+  updatedAt: Date;
+};
+
+export const DatasetCard = ({ dataset }: { dataset: DatasetData }) => {
+  return (
+    <AspectRatio ratio={1.2} w="full">
+      <VStack
+        as={Link}
+        href={{ pathname: "/data/[id]", query: { id: dataset.id } }}
+        bg="gray.50"
+        _hover={{ bg: "gray.100" }}
+        transition="background 0.2s"
+        cursor="pointer"
+        borderColor="gray.200"
+        borderWidth={1}
+        p={4}
+        justify="space-between"
+      >
+        <HStack w="full" color="gray.700" justify="center">
+          <Icon as={RiDatabase2Line} boxSize={4} />
+          <Text fontWeight="bold">{dataset.name}</Text>
+        </HStack>
+        <HStack h="full" spacing={4} flex={1} align="center">
+          <CountLabel label="Rows" count={dataset.numEntries} />
+        </HStack>
+        <HStack w="full" color="gray.500" fontSize="xs" textAlign="center">
+          <Text flex={1}>Created {formatTimePast(dataset.createdAt)}</Text>
+          <Divider h={4} orientation="vertical" />
+          <Text flex={1}>Updated {formatTimePast(dataset.updatedAt)}</Text>
+        </HStack>
+      </VStack>
+    </AspectRatio>
+  );
+};
+
+const CountLabel = ({ label, count }: { label: string; count: number }) => {
+  return (
+    <VStack alignItems="center" flex={1}>
+      <Text color="gray.500" fontWeight="bold">
+        {label}
+      </Text>
+      <Text fontSize="sm" color="gray.500">
+        {count}
+      </Text>
+    </VStack>
+  );
+};
+
+export const NewDatasetCard = () => {
+  const router = useRouter();
+  const createMutation = api.datasets.create.useMutation();
+  const [createDataset, isLoading] = useHandledAsyncCallback(async () => {
+    const newDataset = await createMutation.mutateAsync({ label: "New Dataset" });
+    await router.push({ pathname: "/data/[id]", query: { id: newDataset.id } });
+  }, [createMutation, router]);
+
+  return (
+    <AspectRatio ratio={1.2} w="full">
+      <VStack
+        align="center"
+        justify="center"
+        _hover={{ cursor: "pointer", bg: "gray.50" }}
+        transition="background 0.2s"
+        cursor="pointer"
+        borderColor="gray.200"
+        borderWidth={1}
+        p={4}
+        onClick={createDataset}
+      >
+        <Icon as={isLoading ? Spinner : BsPlusSquare} boxSize={8} />
+        <Text display={{ base: "none", md: "block" }} ml={2}>
+          New Dataset
+        </Text>
+      </VStack>
+    </AspectRatio>
+  );
+};
+
+export const DatasetCardSkeleton = () => (
+  <AspectRatio ratio={1.2} w="full">
+    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="gray.50">
+      <SkeletonText noOfLines={1} w="80%" />
+      <SkeletonText noOfLines={2} w="60%" />
+      <SkeletonText noOfLines={1} w="80%" />
+    </VStack>
+  </AspectRatio>
+);

src/components/datasets/DatasetEntriesPaginator.tsx

@@ -0,0 +1,21 @@
+import { useDatasetEntries } from "~/utils/hooks";
+import Paginator from "../Paginator";
+
+const DatasetEntriesPaginator = () => {
+  const { data } = useDatasetEntries();
+
+  if (!data) return null;
+
+  const { entries, startIndex, lastPage, count } = data;
+
+  return (
+    <Paginator
+      numItemsLoaded={entries.length}
+      startIndex={startIndex}
+      lastPage={lastPage}
+      count={count}
+    />
+  );
+};
+
+export default DatasetEntriesPaginator;

src/components/datasets/DatasetEntriesTable.tsx

@@ -0,0 +1,31 @@
+import { type StackProps, VStack, Table, Th, Tr, Thead, Tbody, Text } from "@chakra-ui/react";
+import { useDatasetEntries } from "~/utils/hooks";
+import TableRow from "./TableRow";
+import DatasetEntriesPaginator from "./DatasetEntriesPaginator";
+
+const DatasetEntriesTable = (props: StackProps) => {
+  const { data } = useDatasetEntries();
+
+  return (
+    <VStack justifyContent="space-between" {...props}>
+      <Table variant="simple" sx={{ "table-layout": "fixed", width: "full" }}>
+        <Thead>
+          <Tr>
+            <Th>Input</Th>
+            <Th>Output</Th>
+          </Tr>
+        </Thead>
+        <Tbody>{data?.entries.map((entry) => <TableRow key={entry.id} entry={entry} />)}</Tbody>
+      </Table>
+      {(!data || data.entries.length) === 0 ? (
+        <Text alignSelf="flex-start" pl={6} color="gray.500">
+          No entries found
+        </Text>
+      ) : (
+        <DatasetEntriesPaginator />
+      )}
+    </VStack>
+  );
+};
+
+export default DatasetEntriesTable;

src/components/datasets/DatasetHeaderButtons/DatasetHeaderButtons.tsx

@@ -0,0 +1,26 @@
+import { Button, HStack, useDisclosure } from "@chakra-ui/react";
+import { BiImport } from "react-icons/bi";
+import { BsStars } from "react-icons/bs";
+
+import { GenerateDataModal } from "./GenerateDataModal";
+
+export const DatasetHeaderButtons = () => {
+  const generateModalDisclosure = useDisclosure();
+
+  return (
+    <>
+      <HStack>
+        <Button leftIcon={<BiImport />} colorScheme="blue" variant="ghost">
+          Import Data
+        </Button>
+        <Button leftIcon={<BsStars />} colorScheme="blue" onClick={generateModalDisclosure.onOpen}>
+          Generate Data
+        </Button>
+      </HStack>
+      <GenerateDataModal
+        isOpen={generateModalDisclosure.isOpen}
+        onClose={generateModalDisclosure.onClose}
+      />
+    </>
+  );
+};

src/components/datasets/DatasetHeaderButtons/GenerateDataModal.tsx

@@ -0,0 +1,128 @@
+import {
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalHeader,
+  ModalOverlay,
+  ModalFooter,
+  Text,
+  HStack,
+  VStack,
+  Icon,
+  NumberInput,
+  NumberInputField,
+  NumberInputStepper,
+  NumberIncrementStepper,
+  NumberDecrementStepper,
+  Button,
+} from "@chakra-ui/react";
+import { BsStars } from "react-icons/bs";
+import { useState } from "react";
+import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import AutoResizeTextArea from "~/components/AutoResizeTextArea";
+
+export const GenerateDataModal = ({
+  isOpen,
+  onClose,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+}) => {
+  const utils = api.useContext();
+
+  const datasetId = useDataset().data?.id;
+
+  const [numToGenerate, setNumToGenerate] = useState<number>(20);
+  const [inputDescription, setInputDescription] = useState<string>(
+    "Each input should contain an email body. Half of the emails should contain event details, and the other half should not.",
+  );
+  const [outputDescription, setOutputDescription] = useState<string>(
+    `Each output should contain "true" or "false", where "true" indicates that the email contains event details.`,
+  );
+
+  const generateEntriesMutation = api.datasetEntries.autogenerateEntries.useMutation();
+
+  const [generateEntries, generateEntriesInProgress] = useHandledAsyncCallback(async () => {
+    if (!inputDescription || !outputDescription || !numToGenerate || !datasetId) return;
+    await generateEntriesMutation.mutateAsync({
+      datasetId,
+      inputDescription,
+      outputDescription,
+      numToGenerate,
+    });
+    await utils.datasetEntries.list.invalidate();
+    onClose();
+  }, [
+    generateEntriesMutation,
+    onClose,
+    inputDescription,
+    outputDescription,
+    numToGenerate,
+    datasetId,
+  ]);
+
+  return (
+    <Modal isOpen={isOpen} onClose={onClose} size={{ base: "xl", sm: "2xl", md: "3xl" }}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={BsStars} />
+            <Text>Generate Data</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} padding={8} alignItems="flex-start">
+            <VStack alignItems="flex-start" spacing={2}>
+              <Text fontWeight="bold">Number of Rows:</Text>
+              <NumberInput
+                step={5}
+                defaultValue={15}
+                min={0}
+                max={100}
+                onChange={(valueString) => setNumToGenerate(parseInt(valueString) || 0)}
+                value={numToGenerate}
+                w="24"
+              >
+                <NumberInputField />
+                <NumberInputStepper>
+                  <NumberIncrementStepper />
+                  <NumberDecrementStepper />
+                </NumberInputStepper>
+              </NumberInput>
+            </VStack>
+            <VStack alignItems="flex-start" w="full" spacing={2}>
+              <Text fontWeight="bold">Input Description:</Text>
+              <AutoResizeTextArea
+                value={inputDescription}
+                onChange={(e) => setInputDescription(e.target.value)}
+                placeholder="Each input should contain..."
+              />
+            </VStack>
+            <VStack alignItems="flex-start" w="full" spacing={2}>
+              <Text fontWeight="bold">Output Description (optional):</Text>
+              <AutoResizeTextArea
+                value={outputDescription}
+                onChange={(e) => setOutputDescription(e.target.value)}
+                placeholder="The output should contain..."
+              />
+            </VStack>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <Button
+            colorScheme="blue"
+            isLoading={generateEntriesInProgress}
+            isDisabled={!numToGenerate || !inputDescription || !outputDescription}
+            onClick={generateEntries}
+          >
+            Generate
+          </Button>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

src/components/datasets/TableRow.tsx

@@ -0,0 +1,13 @@
+import { Td, Tr } from "@chakra-ui/react";
+import { type DatasetEntry } from "@prisma/client";
+
+const TableRow = ({ entry }: { entry: DatasetEntry }) => {
+  return (
+    <Tr key={entry.id}>
+      <Td>{entry.input}</Td>
+      <Td>{entry.output}</Td>
+    </Tr>
+  );
+};
+
+export default TableRow;

src/components/experiments/ExperimentCard.tsx

@@ -1,17 +1,20 @@
 import {
-  Card,
-  CardBody,
   HStack,
   Icon,
   VStack,
   Text,
-  CardHeader,
   Divider,
-  Box,
+  Spinner,
+  AspectRatio,
+  SkeletonText,
 } from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
 import { formatTimePast } from "~/utils/dayjs";
+import Link from "next/link";
 import { useRouter } from "next/router";
+import { BsPlusSquare } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback } from "~/utils/hooks";
 
 type ExperimentData = {
   testScenarioCount: number;
@@ -24,47 +27,42 @@ type ExperimentData = {
 };
 
 export const ExperimentCard = ({ exp }: { exp: ExperimentData }) => {
-  const router = useRouter();
   return (
-    <Box
+    <AspectRatio ratio={1.2} w="full">
-      as={Card}
+      <VStack
-      variant="elevated"
+        as={Link}
-      bg="gray.50"
+        href={{ pathname: "/experiments/[id]", query: { id: exp.id } }}
-      _hover={{ bg: "gray.100" }}
+        bg="gray.50"
-      transition="background 0.2s"
+        _hover={{ bg: "gray.100" }}
-      cursor="pointer"
+        transition="background 0.2s"
-      onClick={(e) => {
+        cursor="pointer"
-        e.preventDefault();
+        borderColor="gray.200"
-        void router.push({ pathname: "/experiments/[id]", query: { id: exp.id } }, undefined, {
+        borderWidth={1}
-          shallow: true,
+        p={4}
-        });
+        justify="space-between"
-      }}
+      >
-    >
+        <HStack w="full" color="gray.700" justify="center">
-      <CardHeader>
-        <HStack w="full" color="gray.700">
           <Icon as={RiFlaskLine} boxSize={4} />
           <Text fontWeight="bold">{exp.label}</Text>
         </HStack>
-      </CardHeader>
+        <HStack h="full" spacing={4} flex={1} align="center">
-      <CardBody>
-        <HStack w="full" mb={8} spacing={4}>
           <CountLabel label="Variants" count={exp.promptVariantCount} />
           <Divider h={12} orientation="vertical" />
           <CountLabel label="Scenarios" count={exp.testScenarioCount} />
         </HStack>
-        <HStack w="full" color="gray.500" fontSize="xs">
+        <HStack w="full" color="gray.500" fontSize="xs" textAlign="center">
-          <Text>Created {formatTimePast(exp.createdAt)}</Text>
+          <Text flex={1}>Created {formatTimePast(exp.createdAt)}</Text>
           <Divider h={4} orientation="vertical" />
-          <Text>Updated {formatTimePast(exp.updatedAt)}</Text>
+          <Text flex={1}>Updated {formatTimePast(exp.updatedAt)}</Text>
         </HStack>
-      </CardBody>
+      </VStack>
-    </Box>
+    </AspectRatio>
   );
 };
 
 const CountLabel = ({ label, count }: { label: string; count: number }) => {
   return (
-    <VStack alignItems="flex-start">
+    <VStack alignItems="center" flex={1}>
       <Text color="gray.500" fontWeight="bold">
         {label}
       </Text>
@@ -74,3 +72,43 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
     </VStack>
   );
 };
+
+export const NewExperimentCard = () => {
+  const router = useRouter();
+  const createMutation = api.experiments.create.useMutation();
+  const [createExperiment, isLoading] = useHandledAsyncCallback(async () => {
+    const newExperiment = await createMutation.mutateAsync({ label: "New Experiment" });
+    await router.push({ pathname: "/experiments/[id]", query: { id: newExperiment.id } });
+  }, [createMutation, router]);
+
+  return (
+    <AspectRatio ratio={1.2} w="full">
+      <VStack
+        align="center"
+        justify="center"
+        _hover={{ cursor: "pointer", bg: "gray.50" }}
+        transition="background 0.2s"
+        cursor="pointer"
+        borderColor="gray.200"
+        borderWidth={1}
+        p={4}
+        onClick={createExperiment}
+      >
+        <Icon as={isLoading ? Spinner : BsPlusSquare} boxSize={8} />
+        <Text display={{ base: "none", md: "block" }} ml={2}>
+          New Experiment
+        </Text>
+      </VStack>
+    </AspectRatio>
+  );
+};
+
+export const ExperimentCardSkeleton = () => (
+  <AspectRatio ratio={1.2} w="full">
+    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="gray.50">
+      <SkeletonText noOfLines={1} w="80%" />
+      <SkeletonText noOfLines={2} w="60%" />
+      <SkeletonText noOfLines={1} w="80%" />
+    </VStack>
+  </AspectRatio>
+);

src/components/experiments/ExperimentHeaderButtons/DeleteDialog.tsx

@@ -0,0 +1,57 @@
+import {
+  Button,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+} from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import { useRef } from "react";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+
+export const DeleteDialog = ({ onClose }: { onClose: () => void }) => {
+  const experiment = useExperiment();
+  const deleteMutation = api.experiments.delete.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    await deleteMutation.mutateAsync({ id: experiment.data.id });
+    await utils.experiments.list.invalidate();
+    await router.push({ pathname: "/experiments" });
+    onClose();
+  }, [deleteMutation, experiment.data?.id, router]);
+
+  return (
+    <AlertDialog isOpen leastDestructiveRef={cancelRef} onClose={onClose}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Delete Experiment
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            If you delete this experiment all the associated prompts and scenarios will be deleted
+            as well. Are you sure?
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={onClose}>
+              Cancel
+            </Button>
+            <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
+              Delete
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};

src/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons.tsx

@@ -0,0 +1,42 @@
+import { Button, HStack, Icon, Spinner, Text } from "@chakra-ui/react";
+import { useOnForkButtonPressed } from "./useOnForkButtonPressed";
+import { useExperiment } from "~/utils/hooks";
+import { BsGearFill } from "react-icons/bs";
+import { TbGitFork } from "react-icons/tb";
+import { useAppStore } from "~/state/store";
+
+export const ExperimentHeaderButtons = () => {
+  const experiment = useExperiment();
+
+  const canModify = experiment.data?.access.canModify ?? false;
+
+  const { onForkButtonPressed, isForking } = useOnForkButtonPressed();
+
+  const openDrawer = useAppStore((s) => s.openDrawer);
+
+  if (experiment.isLoading) return null;
+
+  return (
+    <HStack spacing={0} mt={{ base: 2, md: 0 }}>
+      <Button
+        onClick={onForkButtonPressed}
+        mr={4}
+        colorScheme={canModify ? undefined : "orange"}
+        bgColor={canModify ? undefined : "orange.400"}
+        minW={0}
+        variant={{ base: "solid", md: canModify ? "ghost" : "solid" }}
+      >
+        {isForking ? <Spinner boxSize={5} /> : <Icon as={TbGitFork} boxSize={5} />}
+        <Text ml={2}>Fork</Text>
+      </Button>
+      {canModify && (
+        <Button variant={{ base: "solid", md: "ghost" }} onClick={openDrawer}>
+          <HStack>
+            <Icon as={BsGearFill} />
+            <Text>Settings</Text>
+          </HStack>
+        </Button>
+      )}
+    </HStack>
+  );
+};

src/components/experiments/ExperimentHeaderButtons/useOnForkButtonPressed.tsx

@@ -0,0 +1,30 @@
+import { useCallback } from "react";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { signIn, useSession } from "next-auth/react";
+import { useRouter } from "next/router";
+
+export const useOnForkButtonPressed = () => {
+  const router = useRouter();
+
+  const user = useSession().data;
+  const experiment = useExperiment();
+
+  const forkMutation = api.experiments.fork.useMutation();
+
+  const [onFork, isForking] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    const forkedExperimentId = await forkMutation.mutateAsync({ id: experiment.data.id });
+    await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
+  }, [forkMutation, experiment.data?.id, router]);
+
+  const onForkButtonPressed = useCallback(() => {
+    if (user === null) {
+      signIn("github").catch(console.error);
+    } else {
+      onFork();
+    }
+  }, [onFork, user]);
+
+  return { onForkButtonPressed, isForking };
+};

src/components/experiments/NewExperimentButton.tsx

@@ -1,31 +0,0 @@
-import { Icon, Button, Spinner, Text, type ButtonProps } from "@chakra-ui/react";
-import { api } from "~/utils/api";
-import { useRouter } from "next/router";
-import { BsPlusSquare } from "react-icons/bs";
-import { useHandledAsyncCallback } from "~/utils/hooks";
-
-export const NewExperimentButton = (props: ButtonProps) => {
-  const router = useRouter();
-  const utils = api.useContext();
-  const createMutation = api.experiments.create.useMutation();
-  const [createExperiment, isLoading] = useHandledAsyncCallback(async () => {
-    const newExperiment = await createMutation.mutateAsync({ label: "New Experiment" });
-    await utils.experiments.list.invalidate();
-    await router.push({ pathname: "/experiments/[id]", query: { id: newExperiment.id } });
-  }, [createMutation, router]);
-
-  return (
-    <Button
-      onClick={createExperiment}
-      display="flex"
-      alignItems="center"
-      variant={{ base: "solid", md: "ghost" }}
-      {...props}
-    >
-      <Icon as={isLoading ? Spinner : BsPlusSquare} boxSize={4} />
-      <Text display={{ base: "none", md: "block" }} ml={2}>
-        New Experiment
-      </Text>
-    </Button>
-  );
-};

src/components/nav/AppShell.tsx

@@ -1,84 +1,117 @@
+import { useState, useEffect } from "react";
 import {
   Heading,
   VStack,
   Icon,
   HStack,
   Image,
-  Grid,
-  GridItem,
-  Divider,
   Text,
   Box,
   type BoxProps,
-  type LinkProps,
+  Link as ChakraLink,
-  Link,
+  Flex,
 } from "@chakra-ui/react";
 import Head from "next/head";
-import { BsGithub, BsTwitter } from "react-icons/bs";
+import Link, { type LinkProps } from "next/link";
+import { BsGithub, BsPersonCircle } from "react-icons/bs";
 import { useRouter } from "next/router";
-import PublicPlaygroundWarning from "../PublicPlaygroundWarning";
 import { type IconType } from "react-icons";
-import { RiFlaskLine } from "react-icons/ri";
+import { RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
-import { useState, useEffect } from "react";
+import { signIn, useSession } from "next-auth/react";
+import UserMenu from "./UserMenu";
+import { env } from "~/env.mjs";
 
-type IconLinkProps = BoxProps & LinkProps & { label: string; icon: IconType; href: string };
+type IconLinkProps = BoxProps & LinkProps & { label?: string; icon: IconType; href: string };
 
-const IconLink = ({ icon, label, href, target, color, ...props }: IconLinkProps) => {
+const IconLink = ({ icon, label, href, color, ...props }: IconLinkProps) => {
-  const isActive = useRouter().pathname.startsWith(href);
+  const router = useRouter();
+  const isActive = href && router.pathname.startsWith(href);
   return (
-    <Box
+    <Link href={href} style={{ width: "100%" }}>
-      as={Link}
+      <HStack
-      href={href}
+        w="full"
-      target={target}
+        p={4}
-      w="full"
+        color={color}
-      bgColor={isActive ? "gray.300" : "transparent"}
+        as={ChakraLink}
-      _hover={{ bgColor: "gray.300" }}
+        bgColor={isActive ? "gray.200" : "transparent"}
-      py={4}
+        _hover={{ bgColor: "gray.300", textDecoration: "none" }}
-      justifyContent="start"
+        justifyContent="start"
-      cursor="pointer"
+        cursor="pointer"
-      {...props}
+        {...props}
-    >
+      >
-      <HStack w="full" px={4} color={color}>
         <Icon as={icon} boxSize={6} mr={2} />
-        <Text fontWeight="bold">{label}</Text>
+        <Text fontWeight="bold" fontSize="sm">
+          {label}
+        </Text>
       </HStack>
-    </Box>
+    </Link>
   );
 };
 
+const Divider = () => <Box h="1px" bgColor="gray.200" />;
+
 const NavSidebar = () => {
+  const user = useSession().data;
+
   return (
-    <VStack align="stretch" bgColor="gray.100" py={2} pb={0} height="100%">
+    <VStack
-      <Link href="/" w="full" _hover={{ textDecoration: "none" }}>
+      align="stretch"
-        <HStack spacing={0} pl="3">
+      bgColor="gray.100"
-          <Image src="/logo.svg" alt="" w={8} h={8} />
+      py={2}
-          <Heading size="md" p={2} pl={{ base: 16, md: 2 }}>
+      pb={0}
-            OpenPipe
+      height="100%"
-          </Heading>
+      w={{ base: "56px", md: "200px" }}
-        </HStack>
+      overflow="hidden"
-      </Link>
+    >
-      <Divider />
+      <HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={4} py={2}>
+        <Image src="/logo.svg" alt="" boxSize={6} mr={4} />
+        <Heading size="md" fontFamily="inconsolata, monospace">
+          OpenPipe
+        </Heading>
+      </HStack>
       <VStack spacing={0} align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
-        <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
+        {user != null && (
+          <>
+            <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
+            {env.NEXT_PUBLIC_SHOW_DATA && (
+              <IconLink icon={RiDatabase2Line} label="Data" href="/data" />
+            )}
+          </>
+        )}
+        {user === null && (
+          <HStack
+            w="full"
+            p={4}
+            as={ChakraLink}
+            _hover={{ bgColor: "gray.300", textDecoration: "none" }}
+            justifyContent="start"
+            cursor="pointer"
+            onClick={() => {
+              signIn("github").catch(console.error);
+            }}
+          >
+            <Icon as={BsPersonCircle} boxSize={6} mr={2} />
+            <Text fontWeight="bold" fontSize="sm">
+              Sign In
+            </Text>
+          </HStack>
+        )}
       </VStack>
-      <Divider />
+      {user ? (
-      <VStack w="full" spacing={0} pb={2}>
+        <UserMenu user={user} borderColor={"gray.200"} borderTopWidth={1} borderBottomWidth={1} />
-        <IconLink
+      ) : (
-          icon={BsGithub}
+        <Divider />
-          label="GitHub"
+      )}
+      <VStack spacing={0} align="center">
+        <ChakraLink
           href="https://github.com/openpipe/openpipe"
           target="_blank"
           color="gray.500"
           _hover={{ color: "gray.800" }}
-        />
+          p={2}
-        <IconLink
+        >
-          icon={BsTwitter}
+          <Icon as={BsGithub} boxSize={6} />
-          label="Twitter"
+        </ChakraLink>
-          href="https://twitter.com/corbtt"
-          target="_blank"
-          color="gray.500"
-          _hover={{ color: "gray.800" }}
-        />
       </VStack>
     </VStack>
   );
@@ -105,25 +138,14 @@ export default function AppShell(props: { children: React.ReactNode; title?: str
   }, []);
 
   return (
-    <Grid
+    <Flex h={vh} w="100vw">
-      h={vh}
-      w="100vw"
-      templateColumns={{ base: "56px minmax(0, 1fr)", md: "200px minmax(0, 1fr)" }}
-      templateRows="max-content 1fr"
-      templateAreas={'"warning warning"\n"sidebar main"'}
-    >
       <Head>
         <title>{props.title ? `${props.title} | OpenPipe` : "OpenPipe"}</title>
       </Head>
-      <GridItem area="warning">
+      <NavSidebar />
-        <PublicPlaygroundWarning />
+      <Box h="100%" flex={1} overflowY="auto">
-      </GridItem>
-      <GridItem area="sidebar" overflow="hidden">
-        <NavSidebar />
-      </GridItem>
-      <GridItem area="main" overflowY="auto">
         {props.children}
-      </GridItem>
+      </Box>
-    </Grid>
+    </Flex>
   );
 }

src/components/nav/UserMenu.tsx

@@ -0,0 +1,76 @@
+import {
+  HStack,
+  Icon,
+  Image,
+  VStack,
+  Text,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  Link,
+  useColorMode,
+  type StackProps,
+} from "@chakra-ui/react";
+import { type Session } from "next-auth";
+import { signOut } from "next-auth/react";
+import { BsBoxArrowRight, BsChevronRight, BsPersonCircle } from "react-icons/bs";
+
+export default function UserMenu({ user, ...rest }: { user: Session } & StackProps) {
+  const { colorMode } = useColorMode();
+
+  const profileImage = user.user.image ? (
+    <Image src={user.user.image} alt="profile picture" boxSize={8} borderRadius="50%" />
+  ) : (
+    <Icon as={BsPersonCircle} boxSize={6} />
+  );
+
+  return (
+    <>
+      <Popover placement="right">
+        <PopoverTrigger>
+          <HStack
+            // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
+            px={3}
+            spacing={3}
+            py={2}
+            {...rest}
+            cursor="pointer"
+            _hover={{
+              bgColor: colorMode === "light" ? "gray.200" : "gray.700",
+            }}
+          >
+            {profileImage}
+            <VStack spacing={0} align="start" flex={1} flexShrink={1}>
+              <Text fontWeight="bold" fontSize="sm">
+                {user.user.name}
+              </Text>
+              <Text color="gray.500" fontSize="xs">
+                {user.user.email}
+              </Text>
+            </VStack>
+            <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+          </HStack>
+        </PopoverTrigger>
+        <PopoverContent _focusVisible={{ boxShadow: "unset", outline: "unset" }} maxW="200px">
+          <VStack align="stretch" spacing={0}>
+            {/* sign out */}
+            <HStack
+              as={Link}
+              onClick={() => {
+                signOut().catch(console.error);
+              }}
+              px={4}
+              py={2}
+              spacing={4}
+              color="gray.500"
+              fontSize="sm"
+            >
+              <Icon as={BsBoxArrowRight} boxSize={6} />
+              <Text>Sign out</Text>
+            </HStack>
+          </VStack>
+        </PopoverContent>
+      </Popover>
+    </>
+  );
+}

src/components/tooltip/CostTooltip.tsx

@@ -20,7 +20,6 @@ export const CostTooltip = ({
       color="gray.800"
       bgColor="gray.50"
       borderWidth={1}
-      py={2}
       hasArrow
       shouldWrapChildren
       label={

src/env.mjs

@@ -9,7 +9,17 @@ export const env = createEnv({
   server: {
     DATABASE_URL: z.string().url(),
     NODE_ENV: z.enum(["development", "test", "production"]).default("development"),
+    RESTRICT_PRISMA_LOGS: z
+      .string()
+      .optional()
+      .default("false")
+      .transform((val) => val.toLowerCase() === "true"),
+    GITHUB_CLIENT_ID: z.string().min(1),
+    GITHUB_CLIENT_SECRET: z.string().min(1),
     OPENAI_API_KEY: z.string().min(1),
+    REPLICATE_API_TOKEN: z.string().default("placeholder"),
+    ANTHROPIC_API_KEY: z.string().default("placeholder"),
+    SENTRY_AUTH_TOKEN: z.string().optional(),
   },
 
   /**
@@ -19,12 +29,10 @@ export const env = createEnv({
    */
   client: {
     NEXT_PUBLIC_POSTHOG_KEY: z.string().optional(),
-    NEXT_PUBLIC_IS_PUBLIC_PLAYGROUND: z
-      .string()
-      .optional()
-      .default("false")
-      .transform((val) => val.toLowerCase() === "true"),
     NEXT_PUBLIC_SOCKET_URL: z.string().url().default("http://localhost:3318"),
+    NEXT_PUBLIC_HOST: z.string().url().default("http://localhost:3000"),
+    NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
+    NEXT_PUBLIC_SHOW_DATA: z.string().optional(),
   },
 
   /**
@@ -35,9 +43,17 @@ export const env = createEnv({
     DATABASE_URL: process.env.DATABASE_URL,
     NODE_ENV: process.env.NODE_ENV,
     OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+    RESTRICT_PRISMA_LOGS: process.env.RESTRICT_PRISMA_LOGS,
     NEXT_PUBLIC_POSTHOG_KEY: process.env.NEXT_PUBLIC_POSTHOG_KEY,
-    NEXT_PUBLIC_IS_PUBLIC_PLAYGROUND: process.env.NEXT_PUBLIC_IS_PUBLIC_PLAYGROUND,
     NEXT_PUBLIC_SOCKET_URL: process.env.NEXT_PUBLIC_SOCKET_URL,
+    NEXT_PUBLIC_HOST: process.env.NEXT_PUBLIC_HOST,
+    NEXT_PUBLIC_SHOW_DATA: process.env.NEXT_PUBLIC_SHOW_DATA,
+    GITHUB_CLIENT_ID: process.env.GITHUB_CLIENT_ID,
+    GITHUB_CLIENT_SECRET: process.env.GITHUB_CLIENT_SECRET,
+    REPLICATE_API_TOKEN: process.env.REPLICATE_API_TOKEN,
+    ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
+    NEXT_PUBLIC_SENTRY_DSN: process.env.NEXT_PUBLIC_SENTRY_DSN,
+    SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.