Add smoketest (broken)

* Continue polling stats until all evals complete

* Return evaluation changes early, before it has run

* Add task for running new eval

* requeue rate-limited tasks

* Fix prettier

.prettierignore

@@ -1,2 +0,0 @@
-src/codegen/openai.schema.json
-pnpm-lock.yaml

.vscode/settings.json

@@ -1,6 +1,3 @@
 {
-  "eslint.format.enable": true,
-  "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
-  }
+  "eslint.format.enable": true
 }

app/.gitignore

@@ -1,9 +1,11 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
+# App files
+
 # dependencies
 /node_modules
 /.pnp
-.pnp.js
+/.pnp.js
 
 # testing
 /coverage
@@ -15,28 +17,28 @@
 # next.js
 /.next/
 /out/
-next-env.d.ts
+/next-env.d.ts
 
 # production
 /build
 
 # misc
-.DS_Store
-*.pem
+/.DS_Store
+/*.pem
 
 # debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-.pnpm-debug.log*
+/npm-debug.log*
+/yarn-debug.log*
+/yarn-error.log*
+/.pnpm-debug.log*
 
 # local env files
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
-.env
-.env*.local
+/.env
+/.env*.local
 
 # vercel
-.vercel
+/.vercel
 
 # typescript
-*.tsbuildinfo
+/*.tsbuildinfo

app/.prettierignore

@@ -0,0 +1,2 @@
+*.schema.json
+pnpm-lock.yaml

app/@types/nextjs-routes.d.ts

@@ -12,7 +12,9 @@ declare module "nextjs-routes" {
 
   export type Route =
     | StaticRoute<"/account/signin">
+    | DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
+    | StaticRoute<"/api/openapi">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
     | DynamicRoute<"/experiments/[id]", { "id": string }>
     | StaticRoute<"/experiments">

app/README.md

@@ -43,7 +43,9 @@ Natively supports [OpenAI function calls](https://openai.com/blog/function-calli
 
 ## Supported Models
 
-OpenPipe currently supports GPT-3.5 and GPT-4. Wider model support is planned.
+- All models available through the OpenAI [chat completion API](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
+- Llama2 [7b chat](https://replicate.com/a16z-infra/llama7b-v2-chat), [13b chat](https://replicate.com/a16z-infra/llama13b-v2-chat), [70b chat](https://replicate.com/replicate/llama70b-v2-chat).
+- Anthropic's [Claude 1 Instant](https://www.anthropic.com/index/introducing-claude) and [Claude 2](https://www.anthropic.com/index/claude-2)
 
 ## Running Locally
 

app/openapitools.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json",
+  "spaces": 2,
+  "generator-cli": {
+    "version": "6.6.0"
+  }
+}

app/package.json

@@ -12,15 +12,16 @@
     "dev:next": "next dev",
     "dev:wss": "pnpm tsx --watch src/wss-server.ts",
     "dev:worker": "NODE_ENV='development' pnpm tsx --watch src/server/tasks/worker.ts",
-    "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss'",
+    "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss' 'pnpm dev:worker'",
     "postinstall": "prisma generate",
     "lint": "next lint",
     "start": "next start",
-    "codegen": "tsx src/codegen/export-openai-types.ts",
+    "codegen": "tsx src/codegen/export-client-types.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'"
   },
   "dependencies": {
+    "@anthropic-ai/sdk": "^0.5.8",
     "@apidevtools/json-schema-ref-parser": "^10.1.0",
     "@babel/preset-typescript": "^7.22.5",
     "@babel/standalone": "^7.22.9",
@@ -59,6 +60,8 @@
     "lodash-es": "^4.17.21",
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
+    "next-query-params": "^4.2.3",
+    "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
     "openai": "4.0.0-beta.2",
     "pluralize": "^8.0.0",
@@ -77,8 +80,11 @@
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
     "superjson": "1.12.2",
+    "trpc-openapi": "^1.2.0",
     "tsx": "^3.12.7",
     "type-fest": "^4.0.0",
+    "use-query-params": "^2.2.1",
+    "uuid": "^9.0.0",
     "vite-tsconfig-paths": "^4.2.0",
     "zod": "^3.21.4",
     "zustand": "^4.3.9"
@@ -99,6 +105,7 @@
     "@types/react": "^18.2.6",
     "@types/react-dom": "^18.2.4",
     "@types/react-syntax-highlighter": "^15.5.7",
+    "@types/uuid": "^9.0.2",
     "@typescript-eslint/eslint-plugin": "^5.59.6",
     "@typescript-eslint/parser": "^5.59.6",
     "eslint": "^8.40.0",

app/pnpm-lock.yaml

@@ -1,10 +1,13 @@
-lockfileVersion: '6.1'
+lockfileVersion: '6.0'
 
 settings:
   autoInstallPeers: true
   excludeLinksFromLockfile: false
 
 dependencies:
+  '@anthropic-ai/sdk':
+    specifier: ^0.5.8
+    version: 0.5.8
   '@apidevtools/json-schema-ref-parser':
     specifier: ^10.1.0
     version: 10.1.0
@@ -119,6 +122,12 @@ dependencies:
   next-auth:
     specifier: ^4.22.1
     version: 4.22.1(next@13.4.2)(react-dom@18.2.0)(react@18.2.0)
+  next-query-params:
+    specifier: ^4.2.3
+    version: 4.2.3(next@13.4.2)(react@18.2.0)(use-query-params@2.2.1)
+  nextjs-cors:
+    specifier: ^2.1.2
+    version: 2.1.2(next@13.4.2)
   nextjs-routes:
     specifier: ^2.0.1
     version: 2.0.1(next@13.4.2)
@@ -173,12 +182,21 @@ dependencies:
   superjson:
     specifier: 1.12.2
     version: 1.12.2
+  trpc-openapi:
+    specifier: ^1.2.0
+    version: 1.2.0(@trpc/server@10.26.0)(zod@3.21.4)
   tsx:
     specifier: ^3.12.7
     version: 3.12.7
   type-fest:
     specifier: ^4.0.0
     version: 4.0.0
+  use-query-params:
+    specifier: ^2.2.1
+    version: 2.2.1(react-dom@18.2.0)(react@18.2.0)
+  uuid:
+    specifier: ^9.0.0
+    version: 9.0.0
   vite-tsconfig-paths:
     specifier: ^4.2.0
     version: 4.2.0(typescript@5.0.4)
@@ -235,6 +253,9 @@ devDependencies:
   '@types/react-syntax-highlighter':
     specifier: ^15.5.7
     version: 15.5.7
+  '@types/uuid':
+    specifier: ^9.0.2
+    version: 9.0.2
   '@typescript-eslint/eslint-plugin':
     specifier: ^5.59.6
     version: 5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.40.0)(typescript@5.0.4)
@@ -286,6 +307,22 @@ packages:
       '@jridgewell/gen-mapping': 0.3.3
       '@jridgewell/trace-mapping': 0.3.18
 
+  /@anthropic-ai/sdk@0.5.8:
+    resolution: {integrity: sha512-iHenjcE2Q/az6VZiP1DueOSvKNRmxsly6Rx2yjJBoy7OBYVFGVjEdgs2mPQHtTX0ibKAR7tPq6F6MQbKDPWcKg==}
+    dependencies:
+      '@types/node': 18.16.0
+      '@types/node-fetch': 2.6.4
+      abort-controller: 3.0.0
+      agentkeepalive: 4.3.0
+      digest-fetch: 1.3.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.6.12
+    transitivePeerDependencies:
+      - encoding
+      - supports-color
+    dev: false
+
   /@apidevtools/json-schema-ref-parser@10.1.0:
     resolution: {integrity: sha512-3e+viyMuXdrcK8v5pvP+SDoAQ77FH6OyRmuK48SZKmdHJRFm87RsSs8qm6kP39a/pOPURByJw+OXzQIqcfmKtA==}
     engines: {node: '>= 16'}
@@ -3018,6 +3055,10 @@ packages:
     resolution: {integrity: sha512-cputDpIbFgLUaGQn6Vqg3/YsJwxUwHLO13v3i5ouxT4lat0khip9AEWxtERujXV9wxIB1EyF97BSJFt6vpdI8g==}
     dev: false
 
+  /@types/uuid@9.0.2:
+    resolution: {integrity: sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==}
+    dev: true
+
   /@typescript-eslint/eslint-plugin@5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.40.0)(typescript@5.0.4):
     resolution: {integrity: sha512-sXtOgJNEuRU5RLwPUb1jxtToZbgvq3M6FPpY4QENxoOggK+UpTxUBpj6tD8+Qh2g46Pi9We87E+eHnUw8YcGsw==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
@@ -3826,6 +3867,15 @@ packages:
       wrap-ansi: 7.0.0
     dev: false
 
+  /co-body@6.1.0:
+    resolution: {integrity: sha512-m7pOT6CdLN7FuXUcpuz/8lfQ/L77x8SchHCF4G0RBTJO20Wzmhn5Sp4/5WsKy8OSpifBSUrmg83qEqaDHdyFuQ==}
+    dependencies:
+      inflation: 2.0.0
+      qs: 6.11.2
+      raw-body: 2.5.1
+      type-is: 1.6.18
+    dev: false
+
   /color-convert@1.9.3:
     resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==}
     dependencies:
@@ -3900,6 +3950,10 @@ packages:
   /convert-source-map@1.9.0:
     resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==}
 
+  /cookie-es@1.0.0:
+    resolution: {integrity: sha512-mWYvfOLrfEc996hlKcdABeIiPHUPC6DM2QYZdGGOvhOTbA3tjm2eBwqlJpoFdjC89NI4Qt6h0Pu06Mp+1Pj5OQ==}
+    dev: false
+
   /cookie-signature@1.0.6:
     resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==}
     dev: false
@@ -4102,11 +4156,20 @@ packages:
       has-property-descriptors: 1.0.0
       object-keys: 1.1.1
 
+  /defu@6.1.2:
+    resolution: {integrity: sha512-+uO4+qr7msjNNWKYPHqN/3+Dx3NFkmIzayk2L1MyZQlvgZb/J1A0fo410dpKrN2SnqFjt8n4JL8fDJE0wIgjFQ==}
+    dev: false
+
   /delayed-stream@1.0.0:
     resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
     engines: {node: '>=0.4.0'}
     dev: false
 
+  /depd@1.1.2:
+    resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==}
+    engines: {node: '>= 0.6'}
+    dev: false
+
   /depd@2.0.0:
     resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
     engines: {node: '>= 0.8'}
@@ -4117,6 +4180,10 @@ packages:
     engines: {node: '>=6'}
     dev: true
 
+  /destr@2.0.0:
+    resolution: {integrity: sha512-FJ9RDpf3GicEBvzI3jxc2XhHzbqD8p4ANw/1kPsFBfTvP1b7Gn/Lg1vO7R9J4IVgoMbyUmFrFGZafJ1hPZpvlg==}
+    dev: false
+
   /destroy@1.2.0:
     resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==}
     engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@@ -5238,6 +5305,18 @@ packages:
       - pg-native
     dev: false
 
+  /h3@1.7.1:
+    resolution: {integrity: sha512-A9V2NEDNHet7v1gCg7CMwerSigLi0SRbhTy7C3lGb0N4YKIpPmLDjedTUopqp4dnn7COHfqUjjaz3zbtz4QduA==}
+    dependencies:
+      cookie-es: 1.0.0
+      defu: 6.1.2
+      destr: 2.0.0
+      iron-webcrypto: 0.7.1
+      radix3: 1.0.1
+      ufo: 1.1.2
+      uncrypto: 0.1.3
+    dev: false
+
   /has-bigints@1.0.2:
     resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==}
     dev: true
@@ -5365,6 +5444,11 @@ packages:
     engines: {node: '>=0.8.19'}
     dev: true
 
+  /inflation@2.0.0:
+    resolution: {integrity: sha512-m3xv4hJYR2oXw4o4Y5l6P5P16WYmazYof+el6Al3f+YlggGj6qT9kImBAnzDelRALnP5d3h4jGBPKzYCizjZZw==}
+    engines: {node: '>= 0.8.0'}
+    dev: false
+
   /inflight@1.0.6:
     resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==}
     dependencies:
@@ -5394,6 +5478,10 @@ packages:
     engines: {node: '>= 0.10'}
     dev: false
 
+  /iron-webcrypto@0.7.1:
+    resolution: {integrity: sha512-K/UmlEhPCPXEHV5hAtH5C0tI5JnFuOrv4yO/j7ODPl3HaiiHBLbOLTde+ieUaAyfCATe4LoAnclyF+hmSCOVmQ==}
+    dev: false
+
   /is-alphabetical@1.0.4:
     resolution: {integrity: sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==}
     dev: false
@@ -6037,6 +6125,19 @@ packages:
       uuid: 8.3.2
     dev: false
 
+  /next-query-params@4.2.3(next@13.4.2)(react@18.2.0)(use-query-params@2.2.1):
+    resolution: {integrity: sha512-hGNCYRH8YyA5ItiBGSKrtMl21b2MAqfPkdI1mvwloNVqSU142IaGzqHN+OTovyeLIpQfonY01y7BAHb/UH4POg==}
+    peerDependencies:
+      next: ^10.0.0 || ^11.0.0 || ^12.0.0 || ^13.0.0
+      react: ^16.8.0 || ^17.0.0 || ^18.0.0
+      use-query-params: ^2.0.0
+    dependencies:
+      next: 13.4.2(@babel/core@7.22.9)(react-dom@18.2.0)(react@18.2.0)
+      react: 18.2.0
+      tslib: 2.6.0
+      use-query-params: 2.2.1(react-dom@18.2.0)(react@18.2.0)
+    dev: false
+
   /next-tick@1.1.0:
     resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==}
     dev: false
@@ -6086,6 +6187,15 @@ packages:
       - babel-plugin-macros
     dev: false
 
+  /nextjs-cors@2.1.2(next@13.4.2):
+    resolution: {integrity: sha512-2yOVivaaf2ILe4f/qY32hnj3oC77VCOsUQJQfhVMGsXE/YMEWUY2zy78sH9FKUCM7eG42/l3pDofIzMD781XGA==}
+    peerDependencies:
+      next: ^8.1.1-canary.54 || ^9.0.0 || ^10.0.0-0 || ^11.0.0 || ^12.0.0 || ^13.0.0
+    dependencies:
+      cors: 2.8.5
+      next: 13.4.2(@babel/core@7.22.9)(react-dom@18.2.0)(react@18.2.0)
+    dev: false
+
   /nextjs-routes@2.0.1(next@13.4.2):
     resolution: {integrity: sha512-pBGRm6uR44zwUjWWYn6+gwz08BhBbqUYlIzsbNHAh1TWohHYKWFaa2YVsj8BxEo726MZYg87OJPnHpaaY1ia0w==}
     hasBin: true
@@ -6113,6 +6223,22 @@ packages:
       whatwg-url: 5.0.0
     dev: false
 
+  /node-mocks-http@1.12.2:
+    resolution: {integrity: sha512-xhWwC0dh35R9rf0j3bRZXuISXdHxxtMx0ywZQBwjrg3yl7KpRETzogfeCamUIjltpn0Fxvs/ZhGJul1vPLrdJQ==}
+    engines: {node: '>=0.6'}
+    dependencies:
+      accepts: 1.3.8
+      content-disposition: 0.5.4
+      depd: 1.1.2
+      fresh: 0.5.2
+      merge-descriptors: 1.0.1
+      methods: 1.1.2
+      mime: 1.6.0
+      parseurl: 1.3.3
+      range-parser: 1.2.1
+      type-is: 1.6.18
+    dev: false
+
   /node-releases@2.0.13:
     resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==}
 
@@ -6274,6 +6400,10 @@ packages:
       - supports-color
     dev: false
 
+  /openapi-types@12.1.3:
+    resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==}
+    dev: false
+
   /openapi-typescript@5.4.1:
     resolution: {integrity: sha512-AGB2QiZPz4rE7zIwV3dRHtoUC/CWHhUjuzGXvtmMQN2AFV8xCTLKcZUHLcdPQmt/83i22nRE7+TxXOXkK+gf4Q==}
     engines: {node: '>= 14.0.0'}
@@ -6697,6 +6827,10 @@ packages:
     resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
     dev: true
 
+  /radix3@1.0.1:
+    resolution: {integrity: sha512-y+AcwZ3HcUIGc9zGsNVf5+BY/LxL+z+4h4J3/pp8jxSmy1STaCocPS3qrj4tA5ehUSzqtqK+0Aygvz/r/8vy4g==}
+    dev: false
+
   /randombytes@2.1.0:
     resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==}
     dependencies:
@@ -7147,6 +7281,10 @@ packages:
       randombytes: 2.1.0
     dev: true
 
+  /serialize-query-params@2.0.2:
+    resolution: {integrity: sha512-1chMo1dST4pFA9RDXAtF0Rbjaut4is7bzFbI1Z26IuMub68pNCILku85aYmeFhvnY//BXUPUhoRMjYcsT93J/Q==}
+    dev: false
+
   /serve-static@1.15.0:
     resolution: {integrity: sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==}
     engines: {node: '>= 0.8.0'}
@@ -7593,6 +7731,22 @@ packages:
     hasBin: true
     dev: false
 
+  /trpc-openapi@1.2.0(@trpc/server@10.26.0)(zod@3.21.4):
+    resolution: {integrity: sha512-pfYoCd/3KYXWXvUPZBKJw455OOwngKN/6SIcj7Yit19OMLJ+8yVZkEvGEeg5wUSwfsiTdRsKuvqkRPXVSwV7ew==}
+    peerDependencies:
+      '@trpc/server': ^10.0.0
+      zod: ^3.14.4
+    dependencies:
+      '@trpc/server': 10.26.0
+      co-body: 6.1.0
+      h3: 1.7.1
+      lodash.clonedeep: 4.5.0
+      node-mocks-http: 1.12.2
+      openapi-types: 12.1.3
+      zod: 3.21.4
+      zod-to-json-schema: 3.21.4(zod@3.21.4)
+    dev: false
+
   /tsconfck@2.1.2(typescript@5.0.4):
     resolution: {integrity: sha512-ghqN1b0puy3MhhviwO2kGF8SeMDNhEbnKxjK7h6+fvY9JAxqvXi8y5NAHSQv687OVboS2uZIByzGd45/YxrRHg==}
     engines: {node: ^14.13.1 || ^16 || >=18}
@@ -7730,7 +7884,6 @@ packages:
 
   /ufo@1.1.2:
     resolution: {integrity: sha512-TrY6DsjTQQgyS3E3dBaOXf0TpPD8u9FVrVYmKVegJuFw51n/YB9XPt+U6ydzFG5ZIN7+DIjPbNmXoBj9esYhgQ==}
-    dev: true
 
   /unbox-primitive@1.0.2:
     resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==}
@@ -7741,6 +7894,10 @@ packages:
       which-boxed-primitive: 1.0.2
     dev: true
 
+  /uncrypto@0.1.3:
+    resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==}
+    dev: false
+
   /undici@5.22.1:
     resolution: {integrity: sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==}
     engines: {node: '>=14.0'}
@@ -7824,6 +7981,24 @@ packages:
       use-isomorphic-layout-effect: 1.1.2(@types/react@18.2.6)(react@18.2.0)
     dev: false
 
+  /use-query-params@2.2.1(react-dom@18.2.0)(react@18.2.0):
+    resolution: {integrity: sha512-i6alcyLB8w9i3ZK3caNftdb+UnbfBRNPDnc89CNQWkGRmDrm/gfydHvMBfVsQJRq3NoHOM2dt/ceBWG2397v1Q==}
+    peerDependencies:
+      '@reach/router': ^1.2.1
+      react: '>=16.8.0'
+      react-dom: '>=16.8.0'
+      react-router-dom: '>=5'
+    peerDependenciesMeta:
+      '@reach/router':
+        optional: true
+      react-router-dom:
+        optional: true
+    dependencies:
+      react: 18.2.0
+      react-dom: 18.2.0(react@18.2.0)
+      serialize-query-params: 2.0.2
+    dev: false
+
   /use-sidecar@1.1.2(@types/react@18.2.6)(react@18.2.0):
     resolution: {integrity: sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==}
     engines: {node: '>=10'}
@@ -7872,6 +8047,11 @@ packages:
     hasBin: true
     dev: false
 
+  /uuid@9.0.0:
+    resolution: {integrity: sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==}
+    hasBin: true
+    dev: false
+
   /vary@1.1.2:
     resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}
     engines: {node: '>= 0.8'}
@@ -8230,6 +8410,14 @@ packages:
     engines: {node: '>=12.20'}
     dev: true
 
+  /zod-to-json-schema@3.21.4(zod@3.21.4):
+    resolution: {integrity: sha512-fjUZh4nQ1s6HMccgIeE0VP4QG/YRGPmyjO9sAh890aQKPEk3nqbfUXhMFaC+Dr5KvYBm8BCyvfpZf2jY9aGSsw==}
+    peerDependencies:
+      zod: ^3.21.4
+    dependencies:
+      zod: 3.21.4
+    dev: false
+
   /zod@3.21.4:
     resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==}
     dev: false

app/prisma/migrations/20230725005817_use_id_as_streaming_channel/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `streamingChannel` on the `ScenarioVariantCell` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "ScenarioVariantCell" DROP COLUMN "streamingChannel";

app/prisma/migrations/20230725191512_migrate_model_response/migration.sql

@@ -0,0 +1,52 @@
+-- DropForeignKey
+ALTER TABLE "ModelOutput" DROP CONSTRAINT "ModelOutput_scenarioVariantCellId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "OutputEvaluation" DROP CONSTRAINT "OutputEvaluation_modelOutputId_fkey";
+
+-- DropIndex
+DROP INDEX "OutputEvaluation_modelOutputId_evaluationId_key";
+
+-- AlterTable
+ALTER TABLE "OutputEvaluation" RENAME COLUMN "modelOutputId" TO "modelResponseId";
+
+-- AlterTable
+ALTER TABLE "ScenarioVariantCell" DROP COLUMN "retryTime",
+DROP COLUMN "statusCode",
+ADD COLUMN     "jobQueuedAt" TIMESTAMP(3),
+ADD COLUMN     "jobStartedAt" TIMESTAMP(3);
+
+ALTER TABLE "ModelOutput" RENAME TO "ModelResponse";
+
+ALTER TABLE "ModelResponse" 
+ADD COLUMN "requestedAt" TIMESTAMP(3),
+ADD COLUMN "receivedAt" TIMESTAMP(3),
+ADD COLUMN "statusCode" INTEGER,
+ADD COLUMN "errorMessage" TEXT,
+ADD COLUMN "retryTime" TIMESTAMP(3),
+ADD COLUMN "outdated" BOOLEAN NOT NULL DEFAULT false;
+
+-- 3. Remove the unnecessary column
+ALTER TABLE "ModelResponse"
+DROP COLUMN "timeToComplete";
+
+-- AlterTable
+ALTER TABLE "ModelResponse" RENAME CONSTRAINT "ModelOutput_pkey" TO "ModelResponse_pkey";
+ALTER TABLE "ModelResponse" ALTER COLUMN "output" DROP NOT NULL;
+
+-- DropIndex
+DROP INDEX "ModelOutput_scenarioVariantCellId_key";
+
+-- AddForeignKey
+ALTER TABLE "ModelResponse" ADD CONSTRAINT "ModelResponse_scenarioVariantCellId_fkey" FOREIGN KEY ("scenarioVariantCellId") REFERENCES "ScenarioVariantCell"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- RenameIndex
+ALTER INDEX "ModelOutput_inputHash_idx" RENAME TO "ModelResponse_inputHash_idx";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "OutputEvaluation_modelResponseId_evaluationId_key" ON "OutputEvaluation"("modelResponseId", "evaluationId");
+
+-- AddForeignKey
+ALTER TABLE "OutputEvaluation" ADD CONSTRAINT "OutputEvaluation_modelResponseId_fkey" FOREIGN KEY ("modelResponseId") REFERENCES "ModelResponse"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+

app/prisma/migrations/20230730061804_add_logged_calls_and_data_flows/migration.sql

@@ -0,0 +1,64 @@
+-- AlterTable
+ALTER TABLE "Experiment" ADD COLUMN     "dataFlowId" UUID;
+
+-- AlterTable
+ALTER TABLE "ModelResponse" ADD COLUMN     "loggedCallId" UUID,
+ALTER COLUMN "scenarioVariantCellId" DROP NOT NULL;
+
+-- CreateTable
+CREATE TABLE "LoggedCall" (
+    "id" UUID NOT NULL,
+    "promptFunctionHash" TEXT NOT NULL,
+    "promptFunction" TEXT NOT NULL,
+    "prompt" JSONB NOT NULL,
+    "responsePayload" JSONB,
+    "scenarioVariables" JSONB NOT NULL,
+    "model" TEXT NOT NULL,
+    "modelProvider" TEXT NOT NULL,
+    "dataFlowId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCall_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "DataFlow" (
+    "id" UUID NOT NULL,
+    "label" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "DataFlow_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "ApiKey" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "key" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "ApiKey_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ApiKey_key_key" ON "ApiKey"("key");
+
+-- AddForeignKey
+ALTER TABLE "Experiment" ADD CONSTRAINT "Experiment_dataFlowId_fkey" FOREIGN KEY ("dataFlowId") REFERENCES "DataFlow"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ModelResponse" ADD CONSTRAINT "ModelResponse_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_dataFlowId_fkey" FOREIGN KEY ("dataFlowId") REFERENCES "DataFlow"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "DataFlow" ADD CONSTRAINT "DataFlow_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/schema.prisma

@@ -19,13 +19,16 @@ model Experiment {
     organizationId String        @db.Uuid
     organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
 
+    dataFlowId String?     @db.Uuid
+    dataFlow   DataFlow?   @relation(fields: [dataFlowId], references: [id], onDelete: Cascade)
+
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    TemplateVariable TemplateVariable[]
-    PromptVariant    PromptVariant[]
-    TestScenario     TestScenario[]
-    Evaluation       Evaluation[]
+    templateVariables TemplateVariable[]
+    promptVariants    PromptVariant[]
+    testScenarios     TestScenario[]
+    evaluations       Evaluation[]
 }
 
 model PromptVariant {
@@ -90,13 +93,11 @@ enum CellRetrievalStatus {
 model ScenarioVariantCell {
     id String @id @default(uuid()) @db.Uuid
 
-    statusCode       Int?
-    errorMessage     String?
-    retryTime        DateTime?
-    streamingChannel String?
-    retrievalStatus  CellRetrievalStatus @default(COMPLETE)
-
-    modelOutput ModelOutput?
+    retrievalStatus CellRetrievalStatus @default(COMPLETE)
+    jobQueuedAt     DateTime?
+    jobStartedAt    DateTime?
+    modelResponses  ModelResponse[]
+    errorMessage    String? // Contains errors that occurred independently of model responses
 
     promptVariantId String        @db.Uuid
     promptVariant   PromptVariant @relation(fields: [promptVariantId], references: [id], onDelete: Cascade)
@@ -111,24 +112,31 @@ model ScenarioVariantCell {
     @@unique([promptVariantId, testScenarioId])
 }
 
-model ModelOutput {
+model ModelResponse {
     id String @id @default(uuid()) @db.Uuid
 
-    inputHash        String
-    output           Json
-    timeToComplete   Int    @default(0)
-    cost             Float?
-    promptTokens     Int?
-    completionTokens Int?
+    inputHash           String
+    requestedAt         DateTime?
+    receivedAt          DateTime?
+    output              Json?
+    cost                Float?
+    promptTokens        Int?
+    completionTokens    Int?
+    statusCode          Int?
+    errorMessage        String?
+    retryTime           DateTime?
+    outdated            Boolean @default(false)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    scenarioVariantCellId String              @db.Uuid
-    scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
-    outputEvaluation      OutputEvaluation[]
+    scenarioVariantCellId String?              @db.Uuid
+    scenarioVariantCell   ScenarioVariantCell? @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
+    outputEvaluations      OutputEvaluation[]
+
+    loggedCallId String? @db.Uuid
+    loggedCall   LoggedCall? @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
 
-    @@unique([scenarioVariantCellId])
     @@index([inputHash])
 }
 
@@ -150,7 +158,7 @@ model Evaluation {
 
     createdAt        DateTime           @default(now())
     updatedAt        DateTime           @updatedAt
-    OutputEvaluation OutputEvaluation[]
+    outputEvaluations OutputEvaluation[]
 }
 
 model OutputEvaluation {
@@ -160,8 +168,8 @@ model OutputEvaluation {
     result  Float
     details String?
 
-    modelOutputId String      @db.Uuid
-    modelOutput   ModelOutput @relation(fields: [modelOutputId], references: [id], onDelete: Cascade)
+    modelResponseId String      @db.Uuid
+    modelResponse   ModelResponse @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
 
     evaluationId String     @db.Uuid
     evaluation   Evaluation @relation(fields: [evaluationId], references: [id], onDelete: Cascade)
@@ -169,18 +177,65 @@ model OutputEvaluation {
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    @@unique([modelOutputId, evaluationId])
+    @@unique([modelResponseId, evaluationId])
+}
+
+model LoggedCall {
+    id String @id @default(uuid()) @db.Uuid
+
+    promptFunctionHash String
+    promptFunction     String
+    prompt     Json
+    responsePayload    Json?
+    scenarioVariables Json
+    model              String
+    modelProvider      String
+
+    dataFlowId String     @db.Uuid
+    dataFlow   DataFlow   @relation(fields: [dataFlowId], references: [id], onDelete: Cascade)
+
+    modelResponse      ModelResponse[]
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
+model DataFlow {
+    id String @id @default(uuid()) @db.Uuid
+
+    label String
+    experiments   Experiment[]
+    loggedCalls      LoggedCall[]
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
+model ApiKey {
+    id        String  @id @default(uuid()) @db.Uuid
+
+    name      String
+    key     String  @unique
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
 }
 
 model Organization {
     id                String  @id @default(uuid()) @db.Uuid
     personalOrgUserId String? @unique @db.Uuid
-    PersonalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+    personalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+    apiKeys         ApiKey[]
+    dataFlows         DataFlow[]
 
     createdAt        DateTime           @default(now())
     updatedAt        DateTime           @updatedAt
-    OrganizationUser OrganizationUser[]
-    Experiment       Experiment[]
+    organizationUsers OrganizationUser[]
+    experiments       Experiment[]
 }
 
 enum OrganizationUserRole {
@@ -234,15 +289,15 @@ model Session {
 }
 
 model User {
-    id               String             @id @default(uuid()) @db.Uuid
-    name             String?
-    email            String?            @unique
-    emailVerified    DateTime?
-    image            String?
-    accounts         Account[]
-    sessions         Session[]
-    OrganizationUser OrganizationUser[]
-    Organization     Organization[]
+    id                String             @id @default(uuid()) @db.Uuid
+    name              String?
+    email             String?            @unique
+    emailVerified     DateTime?
+    image             String?
+    accounts          Account[]
+    sessions          Session[]
+    organizationUsers OrganizationUser[]
+    organizations     Organization[]
 }
 
 model VerificationToken {

app/prisma/seed.ts

@@ -7,9 +7,13 @@ const defaultId = "11111111-1111-1111-1111-111111111111";
 await prisma.organization.deleteMany({
   where: { id: defaultId },
 });
-await prisma.organization.create({
-  data: { id: defaultId },
-});
+
+// If there's an existing org, just seed into it
+const org =
+  (await prisma.organization.findFirst({})) ??
+  (await prisma.organization.create({
+    data: { id: defaultId },
+  }));
 
 await prisma.experiment.deleteMany({
   where: {
@@ -21,7 +25,7 @@ await prisma.experiment.create({
   data: {
     id: defaultId,
     label: "Country Capitals Example",
-    organizationId: defaultId,
+    organizationId: org.id,
   },
 });
 
@@ -103,30 +107,41 @@ await prisma.testScenario.deleteMany({
   },
 });
 
+const countries = [
+  "Afghanistan",
+  "Albania",
+  "Algeria",
+  "Andorra",
+  "Angola",
+  "Antigua and Barbuda",
+  "Argentina",
+  "Armenia",
+  "Australia",
+  "Austria",
+  "Austrian Empire",
+  "Azerbaijan",
+  "Baden",
+  "Bahamas, The",
+  "Bahrain",
+  "Bangladesh",
+  "Barbados",
+  "Bavaria",
+  "Belarus",
+  "Belgium",
+  "Belize",
+  "Benin (Dahomey)",
+  "Bolivia",
+  "Bosnia and Herzegovina",
+  "Botswana",
+];
 await prisma.testScenario.createMany({
-  data: [
-    {
-      experimentId: defaultId,
-      sortIndex: 0,
-      variableValues: {
-        country: "Spain",
-      },
+  data: countries.map((country, i) => ({
+    experimentId: defaultId,
+    sortIndex: i,
+    variableValues: {
+      country: country,
     },
-    {
-      experimentId: defaultId,
-      sortIndex: 1,
-      variableValues: {
-        country: "USA",
-      },
-    },
-    {
-      experimentId: defaultId,
-      sortIndex: 2,
-      variableValues: {
-        country: "Chile",
-      },
-    },
-  ],
+  })),
 });
 
 const variants = await prisma.promptVariant.findMany({
@@ -149,5 +164,5 @@ await Promise.all(
         testScenarioId: scenario.id,
       })),
     )
-    .map((cell) => generateNewCell(cell.promptVariantId, cell.testScenarioId)),
+    .map((cell) => generateNewCell(cell.promptVariantId, cell.testScenarioId, { stream: false })),
 );

app/run-prod.sh

@@ -6,4 +6,7 @@ echo "Migrating the database"
 pnpm prisma migrate deploy
 
 echo "Starting the server"
-pnpm start
+
+pnpm concurrently --kill-others \
+  "pnpm start" \
+  "pnpm tsx src/server/tasks/worker.ts"

app/src/codegen/export-client-types.ts

@@ -0,0 +1,42 @@
+import "dotenv/config";
+import { openApiDocument } from "~/pages/api/openapi.json";
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+import { generatePromptTypes } from "~/modelProviders/generatePromptTypes";
+
+console.log("Exporting public OpenAPI schema to client-libs/schema.json");
+
+const executionDir = import.meta.url.replace("file://", "");
+const schemaPath = path.join(
+  path.dirname(executionDir),
+  "../../../client-libs/schema.json"
+);
+
+console.log("Exporting schema");
+fs.writeFileSync(schemaPath, JSON.stringify(openApiDocument, null, 2), "utf-8");
+
+console.log("Generating Typescript client");
+
+const tsClientPath = path.join(
+  path.dirname(executionDir),
+  "../../../client-libs/js/codegen"
+);
+
+fs.rmSync(tsClientPath, { recursive: true, force: true });
+
+execSync(
+  `pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
+  {
+    stdio: "inherit",
+  }
+);
+
+const promptTypes = await generatePromptTypes();
+
+const promptTypesPath = path.join(tsClientPath, "promptTypes.ts");
+
+console.log(`Writing promptTypes to ${promptTypesPath}`);
+fs.writeFileSync(promptTypesPath, promptTypes, "utf-8");
+
+console.log("Done!");

app/src/components/ChangeModelModal/ChangeModelModal.tsx

@@ -1,5 +1,7 @@
 import {
   Button,
+  HStack,
+  Icon,
   Modal,
   ModalBody,
   ModalCloseButton,
@@ -7,33 +9,37 @@ import {
   ModalFooter,
   ModalHeader,
   ModalOverlay,
-  VStack,
-  Text,
   Spinner,
-  HStack,
-  Icon,
+  Text,
+  VStack,
 } from "@chakra-ui/react";
-import { RiExchangeFundsFill } from "react-icons/ri";
-import { useState } from "react";
-import { type SupportedModel } from "~/server/types";
-import { ModelStatsCard } from "./ModelStatsCard";
-import { SelectModelSearch } from "./SelectModelSearch";
-import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-import CompareFunctions from "../RefinePromptModal/CompareFunctions";
 import { type PromptVariant } from "@prisma/client";
 import { isObject, isString } from "lodash-es";
+import { useState } from "react";
+import { RiExchangeFundsFill } from "react-icons/ri";
+import { type ProviderModel } from "~/modelProviders/types";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { lookupModel, modelLabel } from "~/utils/utils";
+import CompareFunctions from "../RefinePromptModal/CompareFunctions";
+import { ModelSearch } from "./ModelSearch";
+import { ModelStatsCard } from "./ModelStatsCard";
 
-export const SelectModelModal = ({
+export const ChangeModelModal = ({
   variant,
   onClose,
 }: {
   variant: PromptVariant;
   onClose: () => void;
 }) => {
-  const originalModel = variant.model as SupportedModel;
-  const [selectedModel, setSelectedModel] = useState<SupportedModel>(originalModel);
-  const [convertedModel, setConvertedModel] = useState<SupportedModel | undefined>(undefined);
+  const originalModel = lookupModel(variant.modelProvider, variant.model);
+  const [selectedModel, setSelectedModel] = useState({
+    provider: variant.modelProvider,
+    model: variant.model,
+  } as ProviderModel);
+  const [convertedModel, setConvertedModel] = useState<ProviderModel | undefined>();
+  const visibleScenarios = useVisibleScenarioIds();
+
   const utils = api.useContext();
 
   const experiment = useExperiment();
@@ -63,11 +69,17 @@ export const SelectModelModal = ({
     await replaceVariantMutation.mutateAsync({
       id: variant.id,
       constructFn: modifiedPromptFn,
+      streamScenarios: visibleScenarios,
     });
     await utils.promptVariants.list.invalidate();
     onClose();
   }, [replaceVariantMutation, variant, onClose, modifiedPromptFn]);
 
+  const originalLabel = modelLabel(variant.modelProvider, variant.model);
+  const selectedLabel = modelLabel(selectedModel.provider, selectedModel.model);
+  const convertedLabel =
+    convertedModel && modelLabel(convertedModel.provider, convertedModel.model);
+
   return (
     <Modal
       isOpen
@@ -86,16 +98,19 @@ export const SelectModelModal = ({
         <ModalBody maxW="unset">
           <VStack spacing={8}>
             <ModelStatsCard label="Original Model" model={originalModel} />
-            {originalModel !== selectedModel && (
-              <ModelStatsCard label="New Model" model={selectedModel} />
+            {originalLabel !== selectedLabel && (
+              <ModelStatsCard
+                label="New Model"
+                model={lookupModel(selectedModel.provider, selectedModel.model)}
+              />
             )}
-            <SelectModelSearch selectedModel={selectedModel} setSelectedModel={setSelectedModel} />
+            <ModelSearch selectedModel={selectedModel} setSelectedModel={setSelectedModel} />
             {isString(modifiedPromptFn) && (
               <CompareFunctions
                 originalFunction={variant.constructFn}
                 newFunction={modifiedPromptFn}
-                leftTitle={originalModel}
-                rightTitle={convertedModel}
+                leftTitle={originalLabel}
+                rightTitle={convertedLabel}
               />
             )}
           </VStack>
@@ -107,7 +122,7 @@ export const SelectModelModal = ({
               colorScheme="gray"
               onClick={getModifiedPromptFn}
               minW={24}
-              isDisabled={originalModel === selectedModel || modificationInProgress}
+              isDisabled={originalLabel === selectedLabel || modificationInProgress}
             >
               {modificationInProgress ? <Spinner boxSize={4} /> : <Text>Convert</Text>}
             </Button>

app/src/components/ChangeModelModal/ModelSearch.tsx

@@ -0,0 +1,36 @@
+import { Text, VStack } from "@chakra-ui/react";
+import { type LegacyRef } from "react";
+import Select from "react-select";
+import { useElementDimensions } from "~/utils/hooks";
+
+import { flatMap } from "lodash-es";
+import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { type ProviderModel } from "~/modelProviders/types";
+import { modelLabel } from "~/utils/utils";
+
+const modelOptions = flatMap(Object.entries(frontendModelProviders), ([providerId, provider]) =>
+  Object.entries(provider.models).map(([modelId]) => ({
+    provider: providerId,
+    model: modelId,
+  })),
+) as ProviderModel[];
+
+export const ModelSearch = (props: {
+  selectedModel: ProviderModel;
+  setSelectedModel: (model: ProviderModel) => void;
+}) => {
+  const [containerRef, containerDimensions] = useElementDimensions();
+
+  return (
+    <VStack ref={containerRef as LegacyRef<HTMLDivElement>} w="full">
+      <Text>Browse Models</Text>
+      <Select<ProviderModel>
+        styles={{ control: (provided) => ({ ...provided, width: containerDimensions?.width }) }}
+        getOptionLabel={(data) => modelLabel(data.provider, data.model)}
+        getOptionValue={(data) => modelLabel(data.provider, data.model)}
+        options={modelOptions}
+        onChange={(option) => option && props.setSelectedModel(option)}
+      />
+    </VStack>
+  );
+};

app/src/components/ChangeModelModal/ModelStatsCard.tsx

@@ -0,0 +1,109 @@
+import {
+  GridItem,
+  HStack,
+  Link,
+  SimpleGrid,
+  Text,
+  VStack,
+  type StackProps,
+} from "@chakra-ui/react";
+import { type lookupModel } from "~/utils/utils";
+
+export const ModelStatsCard = ({
+  label,
+  model,
+}: {
+  label: string;
+  model: ReturnType<typeof lookupModel>;
+}) => {
+  if (!model) return null;
+  return (
+    <VStack w="full" align="start">
+      <Text fontWeight="bold" fontSize="sm" textTransform="uppercase">
+        {label}
+      </Text>
+
+      <VStack w="full" spacing={6} bgColor="gray.100" p={4} borderRadius={4}>
+        <HStack w="full" align="flex-start">
+          <Text flex={1} fontSize="lg">
+            <Text as="span" color="gray.600">
+              {model.provider} /{" "}
+            </Text>
+            <Text as="span" fontWeight="bold" color="gray.900">
+              {model.name}
+            </Text>
+          </Text>
+          <Link
+            href={model.learnMoreUrl}
+            isExternal
+            color="blue.500"
+            fontWeight="bold"
+            fontSize="sm"
+            ml={2}
+          >
+            Learn More
+          </Link>
+        </HStack>
+        <SimpleGrid
+          w="full"
+          justifyContent="space-between"
+          alignItems="flex-start"
+          fontSize="sm"
+          columns={{ base: 2, md: 4 }}
+        >
+          <SelectedModelLabeledInfo label="Context Window" info={model.contextWindow} />
+          {model.promptTokenPrice && (
+            <SelectedModelLabeledInfo
+              label="Input"
+              info={
+                <Text>
+                  ${(model.promptTokenPrice * 1000).toFixed(3)}
+                  <Text color="gray.500"> / 1K tokens</Text>
+                </Text>
+              }
+            />
+          )}
+          {model.completionTokenPrice && (
+            <SelectedModelLabeledInfo
+              label="Output"
+              info={
+                <Text>
+                  ${(model.completionTokenPrice * 1000).toFixed(3)}
+                  <Text color="gray.500"> / 1K tokens</Text>
+                </Text>
+              }
+            />
+          )}
+          {model.pricePerSecond && (
+            <SelectedModelLabeledInfo
+              label="Price"
+              info={
+                <Text>
+                  ${model.pricePerSecond.toFixed(3)}
+                  <Text color="gray.500"> / second</Text>
+                </Text>
+              }
+            />
+          )}
+          <SelectedModelLabeledInfo label="Speed" info={<Text>{model.speed}</Text>} />
+        </SimpleGrid>
+      </VStack>
+    </VStack>
+  );
+};
+
+const SelectedModelLabeledInfo = ({
+  label,
+  info,
+  ...props
+}: {
+  label: string;
+  info: string | number | React.ReactElement;
+} & StackProps) => (
+  <GridItem>
+    <VStack alignItems="flex-start" {...props}>
+      <Text fontWeight="bold">{label}</Text>
+      <Text>{info}</Text>
+    </VStack>
+  </GridItem>
+);

app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -0,0 +1,69 @@
+import {
+  Button,
+  Icon,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+  useDisclosure,
+  Text,
+} from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import { useRef } from "react";
+import { BsTrash } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+
+export const DeleteButton = () => {
+  const experiment = useExperiment();
+  const mutation = api.experiments.delete.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const { isOpen, onOpen, onClose } = useDisclosure();
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    await mutation.mutateAsync({ id: experiment.data.id });
+    await utils.experiments.list.invalidate();
+    await router.push({ pathname: "/experiments" });
+    onClose();
+  }, [mutation, experiment.data?.id, router]);
+
+  return (
+    <>
+      <Button size="sm" variant="ghost" colorScheme="red" fontWeight="normal" onClick={onOpen}>
+        <Icon as={BsTrash} boxSize={4} />
+        <Text ml={2}>Delete Experiment</Text>
+      </Button>
+
+      <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+        <AlertDialogOverlay>
+          <AlertDialogContent>
+            <AlertDialogHeader fontSize="lg" fontWeight="bold">
+              Delete Experiment
+            </AlertDialogHeader>
+
+            <AlertDialogBody>
+              If you delete this experiment all the associated prompts and scenarios will be deleted
+              as well. Are you sure?
+            </AlertDialogBody>
+
+            <AlertDialogFooter>
+              <Button ref={cancelRef} onClick={onClose}>
+                Cancel
+              </Button>
+              <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
+                Delete
+              </Button>
+            </AlertDialogFooter>
+          </AlertDialogContent>
+        </AlertDialogOverlay>
+      </AlertDialog>
+    </>
+  );
+};

app/src/components/ExperimentSettingsDrawer/ExperimentSettingsDrawer.tsx

@@ -6,13 +6,14 @@ import {
   DrawerHeader,
   DrawerOverlay,
   Heading,
-  Stack,
+  VStack,
 } from "@chakra-ui/react";
-import EditScenarioVars from "./EditScenarioVars";
-import EditEvaluations from "./EditEvaluations";
+import EditScenarioVars from "../OutputsTable/EditScenarioVars";
+import EditEvaluations from "../OutputsTable/EditEvaluations";
 import { useAppStore } from "~/state/store";
+import { DeleteButton } from "./DeleteButton";
 
-export default function SettingsDrawer() {
+export default function ExperimentSettingsDrawer() {
   const isOpen = useAppStore((state) => state.drawerOpen);
   const closeDrawer = useAppStore((state) => state.closeDrawer);
 
@@ -22,13 +23,16 @@ export default function SettingsDrawer() {
       <DrawerContent>
         <DrawerCloseButton />
         <DrawerHeader>
-          <Heading size="md">Settings</Heading>
+          <Heading size="md">Experiment Settings</Heading>
         </DrawerHeader>
-        <DrawerBody>
-          <Stack spacing={6}>
-            <EditScenarioVars />
-            <EditEvaluations />
-          </Stack>
+        <DrawerBody h="full" pb={4}>
+          <VStack h="full" justifyContent="space-between">
+            <VStack spacing={6}>
+              <EditScenarioVars />
+              <EditEvaluations />
+            </VStack>
+            <DeleteButton />
+          </VStack>
         </DrawerBody>
       </DrawerContent>
     </Drawer>

app/src/components/OutputsTable/AddVariantButton.tsx

@@ -0,0 +1,57 @@
+import { Box, Flex, Icon, Spinner } from "@chakra-ui/react";
+import { BsPlus } from "react-icons/bs";
+import { Text } from "@chakra-ui/react";
+import { api } from "~/utils/api";
+import {
+  useExperiment,
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useVisibleScenarioIds,
+} from "~/utils/hooks";
+import { cellPadding } from "../constants";
+import { ActionButton } from "./ScenariosHeader";
+
+export default function AddVariantButton() {
+  const experiment = useExperiment();
+  const mutation = api.promptVariants.create.useMutation();
+  const utils = api.useContext();
+  const visibleScenarios = useVisibleScenarioIds();
+
+  const [onClick, loading] = useHandledAsyncCallback(async () => {
+    if (!experiment.data) return;
+    await mutation.mutateAsync({
+      experimentId: experiment.data.id,
+      streamScenarios: visibleScenarios,
+    });
+    await utils.promptVariants.list.invalidate();
+  }, [mutation]);
+
+  const { canModify } = useExperimentAccess();
+  if (!canModify) return <Box w={cellPadding.x} />;
+
+  return (
+    <Flex w="100%" justifyContent="flex-end">
+      <ActionButton
+        onClick={onClick}
+        py={5}
+        leftIcon={<Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />}
+      >
+        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
+      </ActionButton>
+      {/* <Button
+        alignItems="center"
+        justifyContent="center"
+        fontWeight="normal"
+        bgColor="transparent"
+        _hover={{ bgColor: "gray.100" }}
+        px={cellPadding.x}
+        onClick={onClick}
+        height="unset"
+        minH={headerMinHeight}
+      >
+        <Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />
+        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
+      </Button> */}
+    </Flex>
+  );
+}

app/src/components/OutputsTable/OutputCell/CellContent.tsx

@@ -0,0 +1,19 @@
+import { type StackProps, VStack } from "@chakra-ui/react";
+import { CellOptions } from "./CellOptions";
+
+export const CellContent = ({
+  hardRefetch,
+  hardRefetching,
+  children,
+  ...props
+}: {
+  hardRefetch: () => void;
+  hardRefetching: boolean;
+} & StackProps) => (
+  <VStack w="full" alignItems="flex-start" {...props}>
+    <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} />
+    <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto">
+      {children}
+    </VStack>
+  </VStack>
+);

app/src/components/OutputsTable/OutputCell/CellOptions.tsx

@@ -1,4 +1,4 @@
-import { Button, HStack, Icon, Tooltip } from "@chakra-ui/react";
+import { Button, HStack, Icon, Spinner, Tooltip } from "@chakra-ui/react";
 import { BsArrowClockwise } from "react-icons/bs";
 import { useExperimentAccess } from "~/utils/hooks";
 
@@ -12,7 +12,7 @@ export const CellOptions = ({
   const { canModify } = useExperimentAccess();
   return (
     <HStack justifyContent="flex-end" w="full">
-      {!refetchingOutput && canModify && (
+      {canModify && (
         <Tooltip label="Refetch output" aria-label="refetch output">
           <Button
             size="xs"
@@ -28,7 +28,7 @@ export const CellOptions = ({
             onClick={refetchOutput}
             aria-label="refetch output"
           >
-            <Icon as={BsArrowClockwise} boxSize={4} />
+            <Icon as={refetchingOutput ? Spinner : BsArrowClockwise} boxSize={4} />
           </Button>
         </Tooltip>
       )}

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -0,0 +1,204 @@
+import { api } from "~/utils/api";
+import { type PromptVariant, type Scenario } from "../types";
+import { Text, VStack } from "@chakra-ui/react";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
+import stringify from "json-stringify-pretty-compact";
+import { type ReactElement, useState, useEffect, Fragment } from "react";
+import useSocket from "~/utils/useSocket";
+import { OutputStats } from "./OutputStats";
+import { RetryCountdown } from "./RetryCountdown";
+import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { ResponseLog } from "./ResponseLog";
+import { CellContent } from "./CellContent";
+
+const WAITING_MESSAGE_INTERVAL = 20000;
+
+export default function OutputCell({
+  scenario,
+  variant,
+}: {
+  scenario: Scenario;
+  variant: PromptVariant;
+}): ReactElement | null {
+  const utils = api.useContext();
+  const experiment = useExperiment();
+  const vars = api.templateVars.list.useQuery({
+    experimentId: experiment.data?.id ?? "",
+  }).data;
+
+  const scenarioVariables = scenario.variableValues as Record<string, string>;
+  const templateHasVariables =
+    vars?.length === 0 || vars?.some((v) => scenarioVariables[v.label] !== undefined);
+
+  let disabledReason: string | null = null;
+
+  if (!templateHasVariables) disabledReason = "Add a value to the scenario variables to see output";
+
+  const [refetchInterval, setRefetchInterval] = useState(0);
+  const { data: cell, isLoading: queryLoading } = api.scenarioVariantCells.get.useQuery(
+    { scenarioId: scenario.id, variantId: variant.id },
+    { refetchInterval },
+  );
+
+  const provider =
+    frontendModelProviders[variant.modelProvider as keyof typeof frontendModelProviders];
+
+  type OutputSchema = Parameters<typeof provider.normalizeOutput>[0];
+
+  const { mutateAsync: hardRefetchMutate } = api.scenarioVariantCells.forceRefetch.useMutation();
+  const [hardRefetch, hardRefetching] = useHandledAsyncCallback(async () => {
+    await hardRefetchMutate({ scenarioId: scenario.id, variantId: variant.id });
+    await utils.scenarioVariantCells.get.invalidate({
+      scenarioId: scenario.id,
+      variantId: variant.id,
+    });
+    await utils.promptVariants.stats.invalidate({
+      variantId: variant.id,
+    });
+  }, [hardRefetchMutate, scenario.id, variant.id]);
+
+  const fetchingOutput = queryLoading || hardRefetching;
+
+  const awaitingOutput =
+    !cell ||
+    !cell.evalsComplete ||
+    cell.retrievalStatus === "PENDING" ||
+    cell.retrievalStatus === "IN_PROGRESS" ||
+    hardRefetching;
+  useEffect(() => setRefetchInterval(awaitingOutput ? 1000 : 0), [awaitingOutput]);
+
+  // TODO: disconnect from socket if we're not streaming anymore
+  const streamedMessage = useSocket<OutputSchema>(cell?.id);
+
+  if (!vars) return null;
+
+  if (!cell && !fetchingOutput)
+    return (
+      <CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
+        <Text color="gray.500">Error retrieving output</Text>
+      </CellContent>
+    );
+
+  if (cell && cell.errorMessage) {
+    return (
+      <CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
+        <Text color="red.500">{cell.errorMessage}</Text>
+      </CellContent>
+    );
+  }
+
+  if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
+
+  const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
+  const showLogs = !streamedMessage && !mostRecentResponse?.output;
+
+  if (showLogs)
+    return (
+      <CellContent
+        hardRefetching={hardRefetching}
+        hardRefetch={hardRefetch}
+        alignItems="flex-start"
+        fontFamily="inconsolata, monospace"
+        spacing={0}
+      >
+        {cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
+        {cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
+        {cell?.modelResponses?.map((response) => {
+          let numWaitingMessages = 0;
+          const relativeWaitingTime = response.receivedAt
+            ? response.receivedAt.getTime()
+            : Date.now();
+          if (response.requestedAt) {
+            numWaitingMessages = Math.floor(
+              (relativeWaitingTime - response.requestedAt.getTime()) / WAITING_MESSAGE_INTERVAL,
+            );
+          }
+          return (
+            <Fragment key={response.id}>
+              {response.requestedAt && (
+                <ResponseLog time={response.requestedAt} title="Request sent to API" />
+              )}
+              {response.requestedAt &&
+                Array.from({ length: numWaitingMessages }, (_, i) => (
+                  <ResponseLog
+                    key={`waiting-${i}`}
+                    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                    time={new Date(response.requestedAt!.getTime() + i * WAITING_MESSAGE_INTERVAL)}
+                    title="Waiting for response"
+                  />
+                ))}
+              {response.receivedAt && (
+                <ResponseLog
+                  time={response.receivedAt}
+                  title="Response received from API"
+                  message={`statusCode: ${response.statusCode ?? ""}\n ${
+                    response.errorMessage ?? ""
+                  }`}
+                />
+              )}
+            </Fragment>
+          );
+        }) ?? null}
+        {mostRecentResponse?.retryTime && (
+          <RetryCountdown retryTime={mostRecentResponse.retryTime} />
+        )}
+      </CellContent>
+    );
+
+  const normalizedOutput = mostRecentResponse?.output
+    ? provider.normalizeOutput(mostRecentResponse?.output)
+    : streamedMessage
+    ? provider.normalizeOutput(streamedMessage)
+    : null;
+
+  if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
+    return (
+      <VStack
+        w="100%"
+        h="100%"
+        fontSize="xs"
+        flexWrap="wrap"
+        overflowX="hidden"
+        justifyContent="space-between"
+      >
+        <CellContent
+          hardRefetching={hardRefetching}
+          hardRefetch={hardRefetch}
+          w="full"
+          flex={1}
+          spacing={0}
+        >
+          <SyntaxHighlighter
+            customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
+            language="json"
+            style={docco}
+            lineProps={{
+              style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
+            }}
+            wrapLines
+          >
+            {stringify(normalizedOutput.value, { maxLength: 40 })}
+          </SyntaxHighlighter>
+        </CellContent>
+        <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
+      </VStack>
+    );
+  }
+
+  const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
+
+  return (
+    <VStack w="100%" h="100%" justifyContent="space-between" whiteSpace="pre-wrap">
+      <VStack w="full" alignItems="flex-start" spacing={0}>
+        <CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
+          <Text>{contentToDisplay}</Text>
+        </CellContent>
+      </VStack>
+      {mostRecentResponse?.output && (
+        <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
+      )}
+    </VStack>
+  );
+}

app/src/components/OutputsTable/OutputCell/OutputStats.tsx

@@ -7,28 +7,32 @@ import { CostTooltip } from "~/components/tooltip/CostTooltip";
 const SHOW_TIME = true;
 
 export const OutputStats = ({
-  modelOutput,
+  modelResponse,
 }: {
-  modelOutput: NonNullable<
-    NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>["modelOutput"]
+  modelResponse: NonNullable<
+    NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>["modelResponses"][0]
   >;
   scenario: Scenario;
 }) => {
-  const timeToComplete = modelOutput.timeToComplete;
+  const timeToComplete =
+    modelResponse.receivedAt && modelResponse.requestedAt
+      ? modelResponse.receivedAt.getTime() - modelResponse.requestedAt.getTime()
+      : 0;
 
-  const promptTokens = modelOutput.promptTokens;
-  const completionTokens = modelOutput.completionTokens;
+  const promptTokens = modelResponse.promptTokens;
+  const completionTokens = modelResponse.completionTokens;
 
   return (
     <HStack w="full" align="center" color="gray.500" fontSize="2xs" mt={{ base: 0, md: 1 }}>
       <HStack flex={1}>
-        {modelOutput.outputEvaluation.map((evaluation) => {
+        {modelResponse.outputEvaluations.map((evaluation) => {
           const passed = evaluation.result > 0.5;
           return (
             <Tooltip
               isDisabled={!evaluation.details}
               label={evaluation.details}
               key={evaluation.id}
+              shouldWrapChildren
             >
               <HStack spacing={0}>
                 <Text>{evaluation.evaluation.label}</Text>
@@ -42,15 +46,15 @@ export const OutputStats = ({
           );
         })}
       </HStack>
-      {modelOutput.cost && (
+      {modelResponse.cost && (
         <CostTooltip
           promptTokens={promptTokens}
           completionTokens={completionTokens}
-          cost={modelOutput.cost}
+          cost={modelResponse.cost}
         >
           <HStack spacing={0}>
             <Icon as={BsCurrencyDollar} />
-            <Text mr={1}>{modelOutput.cost.toFixed(3)}</Text>
+            <Text mr={1}>{modelResponse.cost.toFixed(3)}</Text>
           </HStack>
         </CostTooltip>
       )}

app/src/components/OutputsTable/OutputCell/ResponseLog.tsx

@@ -0,0 +1,22 @@
+import { HStack, VStack, Text } from "@chakra-ui/react";
+import dayjs from "dayjs";
+
+export const ResponseLog = ({
+  time,
+  title,
+  message,
+}: {
+  time: Date;
+  title: string;
+  message?: string;
+}) => {
+  return (
+    <VStack spacing={0} alignItems="flex-start">
+      <HStack>
+        <Text>{dayjs(time).format("HH:mm:ss")}</Text>
+        <Text>{title}</Text>
+      </HStack>
+      {message && <Text pl={4}>{message}</Text>}
+    </VStack>
+  );
+};

app/src/components/OutputsTable/OutputCell/RetryCountdown.tsx

@@ -1,21 +1,12 @@
-import { type ScenarioVariantCell } from "@prisma/client";
-import { VStack, Text } from "@chakra-ui/react";
+import { Text } from "@chakra-ui/react";
 import { useEffect, useState } from "react";
 import pluralize from "pluralize";
 
-export const ErrorHandler = ({
-  cell,
-  refetchOutput,
-}: {
-  cell: ScenarioVariantCell;
-  refetchOutput: () => void;
-}) => {
+export const RetryCountdown = ({ retryTime }: { retryTime: Date }) => {
   const [msToWait, setMsToWait] = useState(0);
 
   useEffect(() => {
-    if (!cell.retryTime) return;
-
-    const initialWaitTime = cell.retryTime.getTime() - Date.now();
+    const initialWaitTime = retryTime.getTime() - Date.now();
     const msModuloOneSecond = initialWaitTime % 1000;
     let remainingTime = initialWaitTime - msModuloOneSecond;
     setMsToWait(remainingTime);
@@ -36,18 +27,13 @@ export const ErrorHandler = ({
       clearInterval(interval);
       clearTimeout(timeout);
     };
-  }, [cell.retryTime, cell.statusCode, setMsToWait, refetchOutput]);
+  }, [retryTime]);
+
+  if (msToWait <= 0) return null;
 
   return (
-    <VStack w="full">
-      <Text color="red.600" wordBreak="break-word">
-        {cell.errorMessage}
-      </Text>
-      {msToWait > 0 && (
-        <Text color="red.600" fontSize="sm">
-          Retrying in {pluralize("second", Math.ceil(msToWait / 1000), true)}...
-        </Text>
-      )}
-    </VStack>
+    <Text color="red.600" fontSize="sm">
+      Retrying in {pluralize("second", Math.ceil(msToWait / 1000), true)}...
+    </Text>
   );
 };

app/src/components/OutputsTable/ScenarioPaginator.tsx

@@ -0,0 +1,74 @@
+import { Box, HStack, IconButton } from "@chakra-ui/react";
+import {
+  BsChevronDoubleLeft,
+  BsChevronDoubleRight,
+  BsChevronLeft,
+  BsChevronRight,
+} from "react-icons/bs";
+import { usePage, useScenarios } from "~/utils/hooks";
+
+const ScenarioPaginator = () => {
+  const [page, setPage] = usePage();
+  const { data } = useScenarios();
+
+  if (!data) return null;
+
+  const { scenarios, startIndex, lastPage, count } = data;
+
+  const nextPage = () => {
+    if (page < lastPage) {
+      setPage(page + 1, "replace");
+    }
+  };
+
+  const prevPage = () => {
+    if (page > 1) {
+      setPage(page - 1, "replace");
+    }
+  };
+
+  const goToLastPage = () => setPage(lastPage, "replace");
+  const goToFirstPage = () => setPage(1, "replace");
+
+  return (
+    <HStack pt={4}>
+      <IconButton
+        variant="ghost"
+        size="sm"
+        onClick={goToFirstPage}
+        isDisabled={page === 1}
+        aria-label="Go to first page"
+        icon={<BsChevronDoubleLeft />}
+      />
+      <IconButton
+        variant="ghost"
+        size="sm"
+        onClick={prevPage}
+        isDisabled={page === 1}
+        aria-label="Previous page"
+        icon={<BsChevronLeft />}
+      />
+      <Box>
+        {startIndex}-{startIndex + scenarios.length - 1} / {count}
+      </Box>
+      <IconButton
+        variant="ghost"
+        size="sm"
+        onClick={nextPage}
+        isDisabled={page === lastPage}
+        aria-label="Next page"
+        icon={<BsChevronRight />}
+      />
+      <IconButton
+        variant="ghost"
+        size="sm"
+        onClick={goToLastPage}
+        isDisabled={page === lastPage}
+        aria-label="Go to last page"
+        icon={<BsChevronDoubleRight />}
+      />
+    </HStack>
+  );
+};
+
+export default ScenarioPaginator;

app/src/components/OutputsTable/ScenarioRow.tsx

@@ -4,11 +4,13 @@ import { cellPadding } from "../constants";
 import OutputCell from "./OutputCell/OutputCell";
 import ScenarioEditor from "./ScenarioEditor";
 import type { PromptVariant, Scenario } from "./types";
+import { borders } from "./styles";
 
 const ScenarioRow = (props: {
   scenario: Scenario;
   variants: PromptVariant[];
   canHide: boolean;
+  rowStart: number;
 }) => {
   const [isHovered, setIsHovered] = useState(false);
 
@@ -21,15 +23,21 @@ const ScenarioRow = (props: {
         onMouseLeave={() => setIsHovered(false)}
         sx={isHovered ? highlightStyle : undefined}
         borderLeftWidth={1}
+        {...borders}
+        rowStart={props.rowStart}
+        colStart={1}
       >
         <ScenarioEditor scenario={props.scenario} hovered={isHovered} canHide={props.canHide} />
       </GridItem>
-      {props.variants.map((variant) => (
+      {props.variants.map((variant, i) => (
         <GridItem
           key={variant.id}
           onMouseEnter={() => setIsHovered(true)}
           onMouseLeave={() => setIsHovered(false)}
           sx={isHovered ? highlightStyle : undefined}
+          rowStart={props.rowStart}
+          colStart={i + 2}
+          {...borders}
         >
           <Box h="100%" w="100%" px={cellPadding.x} py={cellPadding.y}>
             <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -0,0 +1,82 @@
+import {
+  Button,
+  type ButtonProps,
+  HStack,
+  Text,
+  Icon,
+  Menu,
+  MenuButton,
+  MenuList,
+  MenuItem,
+  IconButton,
+  Spinner,
+} from "@chakra-ui/react";
+import { cellPadding } from "../constants";
+import {
+  useExperiment,
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useScenarios,
+} from "~/utils/hooks";
+import { BsGear, BsPencil, BsPlus, BsStars } from "react-icons/bs";
+import { useAppStore } from "~/state/store";
+import { api } from "~/utils/api";
+
+export const ActionButton = (props: ButtonProps) => (
+  <Button size="sm" variant="ghost" color="gray.600" {...props} />
+);
+
+export const ScenariosHeader = () => {
+  const openDrawer = useAppStore((s) => s.openDrawer);
+  const { canModify } = useExperimentAccess();
+  const scenarios = useScenarios();
+
+  const experiment = useExperiment();
+  const createScenarioMutation = api.scenarios.create.useMutation();
+  const utils = api.useContext();
+
+  const [onAddScenario, loading] = useHandledAsyncCallback(
+    async (autogenerate: boolean) => {
+      if (!experiment.data) return;
+      await createScenarioMutation.mutateAsync({
+        experimentId: experiment.data.id,
+        autogenerate,
+      });
+      await utils.scenarios.list.invalidate();
+    },
+    [createScenarioMutation],
+  );
+
+  return (
+    <HStack w="100%" pb={cellPadding.y} pt={0} align="center" spacing={0}>
+      <Text fontSize={16} fontWeight="bold">
+        Scenarios ({scenarios.data?.count})
+      </Text>
+      {canModify && (
+        <Menu>
+          <MenuButton
+            as={IconButton}
+            mt={1}
+            variant="ghost"
+            aria-label="Edit Scenarios"
+            icon={<Icon as={loading ? Spinner : BsGear} />}
+          />
+          <MenuList fontSize="md" zIndex="dropdown" mt={-3}>
+            <MenuItem
+              icon={<Icon as={BsPlus} boxSize={6} mx="-5px" />}
+              onClick={() => onAddScenario(false)}
+            >
+              Add Scenario
+            </MenuItem>
+            <MenuItem icon={<BsStars />} onClick={() => onAddScenario(true)}>
+              Autogenerate Scenario
+            </MenuItem>
+            <MenuItem icon={<BsPencil />} onClick={openDrawer}>
+              Edit Vars
+            </MenuItem>
+          </MenuList>
+        </Menu>
+      )}
+    </HStack>
+  );
+};

app/src/components/OutputsTable/VariantEditor.tsx

@@ -1,17 +1,52 @@
-import { Box, Button, HStack, Spinner, Tooltip, useToast, Text } from "@chakra-ui/react";
-import { useRef, useEffect, useState, useCallback } from "react";
-import { useExperimentAccess, useHandledAsyncCallback, useModifierKeyLabel } from "~/utils/hooks";
-import { type PromptVariant } from "./types";
-import { api } from "~/utils/api";
+import {
+  Box,
+  Button,
+  HStack,
+  IconButton,
+  Spinner,
+  Text,
+  Tooltip,
+  useToast,
+} from "@chakra-ui/react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { FiMaximize, FiMinimize } from "react-icons/fi";
+import { editorBackground } from "~/state/sharedVariantEditor.slice";
 import { useAppStore } from "~/state/store";
+import { api } from "~/utils/api";
+import {
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useModifierKeyLabel,
+  useVisibleScenarioIds,
+} from "~/utils/hooks";
+import { type PromptVariant } from "./types";
 
 export default function VariantEditor(props: { variant: PromptVariant }) {
   const { canModify } = useExperimentAccess();
   const monaco = useAppStore.use.sharedVariantEditor.monaco();
   const editorRef = useRef<ReturnType<NonNullable<typeof monaco>["editor"]["create"]> | null>(null);
+  const containerRef = useRef<HTMLDivElement | null>(null);
   const [editorId] = useState(() => `editor_${Math.random().toString(36).substring(7)}`);
   const [isChanged, setIsChanged] = useState(false);
 
+  const [isFullscreen, setIsFullscreen] = useState(false);
+
+  const toggleFullscreen = useCallback(() => {
+    setIsFullscreen((prev) => !prev);
+    editorRef.current?.focus();
+  }, [setIsFullscreen]);
+
+  useEffect(() => {
+    const handleEsc = (event: KeyboardEvent) => {
+      if (event.key === "Escape" && isFullscreen) {
+        toggleFullscreen();
+      }
+    };
+
+    window.addEventListener("keydown", handleEsc);
+    return () => window.removeEventListener("keydown", handleEsc);
+  }, [isFullscreen, toggleFullscreen]);
+
   const lastSavedFn = props.variant.constructFn;
 
   const modifierKey = useModifierKeyLabel();
@@ -33,6 +68,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
   const replaceVariant = api.promptVariants.replaceVariant.useMutation();
   const utils = api.useContext();
   const toast = useToast();
+  const visibleScenarios = useVisibleScenarioIds();
 
   const [onSave, saveInProgress] = useHandledAsyncCallback(async () => {
     if (!editorRef.current) return;
@@ -61,6 +97,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
     const resp = await replaceVariant.mutateAsync({
       id: props.variant.id,
       constructFn: currentFn,
+      streamScenarios: visibleScenarios,
     });
     if (resp.status === "error") {
       return toast({
@@ -99,11 +136,23 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
         readOnly: !canModify,
       });
 
+      // Workaround because otherwise the commands only work on whatever
+      // editor was loaded on the page last.
+      // https://github.com/microsoft/monaco-editor/issues/2947#issuecomment-1422265201
       editorRef.current.onDidFocusEditorText(() => {
-        // Workaround because otherwise the command only works on whatever
-        // editor was loaded on the page last.
-        // https://github.com/microsoft/monaco-editor/issues/2947#issuecomment-1422265201
-        editorRef.current?.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter, onSave);
+        editorRef.current?.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, onSave);
+
+        editorRef.current?.addCommand(
+          monaco.KeyMod.CtrlCmd | monaco.KeyMod.Shift | monaco.KeyCode.KeyF,
+          toggleFullscreen,
+        );
+
+        // Exit fullscreen with escape
+        editorRef.current?.addCommand(monaco.KeyCode.Escape, () => {
+          if (isFullscreen) {
+            toggleFullscreen();
+          }
+        });
       });
 
       editorRef.current.onDidChangeModelContent(checkForChanges);
@@ -132,8 +181,40 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
   }, [canModify]);
 
   return (
-    <Box w="100%" pos="relative">
-      <div id={editorId} style={{ height: "400px", width: "100%" }}></div>
+    <Box
+      w="100%"
+      ref={containerRef}
+      sx={
+        isFullscreen
+          ? {
+              position: "fixed",
+              top: 0,
+              left: 0,
+              right: 0,
+              bottom: 0,
+            }
+          : { h: "400px", w: "100%" }
+      }
+      bgColor={editorBackground}
+      zIndex={isFullscreen ? 1000 : "unset"}
+      pos="relative"
+      _hover={{ ".fullscreen-toggle": { opacity: 1 } }}
+    >
+      <Box id={editorId} w="100%" h="100%" />
+      <Tooltip label={`${modifierKey} + ⇧ + F`}>
+        <IconButton
+          className="fullscreen-toggle"
+          aria-label="Minimize"
+          icon={isFullscreen ? <FiMinimize /> : <FiMaximize />}
+          position="absolute"
+          top={2}
+          right={2}
+          onClick={toggleFullscreen}
+          opacity={0}
+          transition="opacity 0.2s"
+        />
+      </Tooltip>
+
       {isChanged && (
         <HStack pos="absolute" bottom={2} right={2}>
           <Button
@@ -146,7 +227,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
           >
             Reset
           </Button>
-          <Tooltip label={`${modifierKey} + Enter`}>
+          <Tooltip label={`${modifierKey} + S`}>
             <Button size="sm" onClick={onSave} colorScheme="blue" w={16} disabled={saveInProgress}>
               {saveInProgress ? <Spinner boxSize={4} /> : <Text>Save</Text>}
             </Button>

app/src/components/OutputsTable/VariantStats.tsx

@@ -21,17 +21,14 @@ export default function VariantStats(props: { variant: PromptVariant }) {
         completionTokens: 0,
         scenarioCount: 0,
         outputCount: 0,
-        awaitingRetrievals: false,
+        awaitingEvals: false,
       },
       refetchInterval,
     },
   );
 
   // Poll every two seconds while we are waiting for LLM retrievals to finish
-  useEffect(
-    () => setRefetchInterval(data.awaitingRetrievals ? 2000 : 0),
-    [data.awaitingRetrievals],
-  );
+  useEffect(() => setRefetchInterval(data.awaitingEvals ? 5000 : 0), [data.awaitingEvals]);
 
   const [passColor, neutralColor, failColor] = useToken("colors", [
     "green.500",
@@ -69,7 +66,7 @@ export default function VariantStats(props: { variant: PromptVariant }) {
           );
         })}
       </HStack>
-      {data.overallCost && !data.awaitingRetrievals && (
+      {data.overallCost && (
         <CostTooltip
           promptTokens={data.promptTokens}
           completionTokens={data.completionTokens}

app/src/components/OutputsTable/index.tsx

@@ -0,0 +1,107 @@
+import { Grid, GridItem, type GridItemProps } from "@chakra-ui/react";
+import { api } from "~/utils/api";
+import AddVariantButton from "./AddVariantButton";
+import ScenarioRow from "./ScenarioRow";
+import VariantEditor from "./VariantEditor";
+import VariantHeader from "../VariantHeader/VariantHeader";
+import VariantStats from "./VariantStats";
+import { ScenariosHeader } from "./ScenariosHeader";
+import { borders } from "./styles";
+import { useScenarios } from "~/utils/hooks";
+import ScenarioPaginator from "./ScenarioPaginator";
+import { Fragment } from "react";
+
+export default function OutputsTable({ experimentId }: { experimentId: string | undefined }) {
+  const variants = api.promptVariants.list.useQuery(
+    { experimentId: experimentId as string },
+    { enabled: !!experimentId },
+  );
+
+  const scenarios = useScenarios();
+
+  if (!variants.data || !scenarios.data) return null;
+
+  const allCols = variants.data.length + 2;
+  const variantHeaderRows = 3;
+  const scenarioHeaderRows = 1;
+  const scenarioFooterRows = 1;
+  const visibleScenariosCount = scenarios.data.scenarios.length;
+  const allRows =
+    variantHeaderRows + scenarioHeaderRows + visibleScenariosCount + scenarioFooterRows;
+
+  return (
+    <Grid
+      pt={4}
+      pb={24}
+      pl={8}
+      display="grid"
+      gridTemplateColumns={`250px repeat(${variants.data.length}, minmax(300px, 1fr)) auto`}
+      sx={{
+        "> *": {
+          borderColor: "gray.300",
+        },
+      }}
+      fontSize="sm"
+    >
+      <GridItem rowSpan={variantHeaderRows}>
+        <AddVariantButton />
+      </GridItem>
+
+      {variants.data.map((variant, i) => {
+        const sharedProps: GridItemProps = {
+          ...borders,
+          colStart: i + 2,
+          borderLeftWidth: i === 0 ? 1 : 0,
+          marginLeft: i === 0 ? "-1px" : 0,
+          backgroundColor: "gray.100",
+        };
+        return (
+          <Fragment key={variant.uiId}>
+            <VariantHeader
+              variant={variant}
+              canHide={variants.data.length > 1}
+              rowStart={1}
+              {...sharedProps}
+            />
+            <GridItem rowStart={2} {...sharedProps}>
+              <VariantEditor variant={variant} />
+            </GridItem>
+            <GridItem rowStart={3} {...sharedProps}>
+              <VariantStats variant={variant} />
+            </GridItem>
+          </Fragment>
+        );
+      })}
+
+      <GridItem
+        colSpan={allCols - 1}
+        rowStart={variantHeaderRows + 1}
+        colStart={1}
+        {...borders}
+        borderRightWidth={0}
+      >
+        <ScenariosHeader />
+      </GridItem>
+
+      {scenarios.data.scenarios.map((scenario, i) => (
+        <ScenarioRow
+          rowStart={i + variantHeaderRows + scenarioHeaderRows + 2}
+          key={scenario.uiId}
+          scenario={scenario}
+          variants={variants.data}
+          canHide={visibleScenariosCount > 1}
+        />
+      ))}
+      <GridItem
+        rowStart={variantHeaderRows + scenarioHeaderRows + visibleScenariosCount + 2}
+        colStart={1}
+        colSpan={allCols}
+      >
+        <ScenarioPaginator />
+      </GridItem>
+
+      {/* Add some extra padding on the right, because when the table is too wide to fit in the viewport `pr` on the Grid isn't respected. */}
+      <GridItem rowStart={1} colStart={allCols} rowSpan={allRows} w={4} borderBottomWidth={0} />
+    </Grid>
+  );
+}

app/src/components/OutputsTable/styles.ts

@@ -0,0 +1,6 @@
+import { type GridItemProps } from "@chakra-ui/react";
+
+export const borders: GridItemProps = {
+  borderRightWidth: 1,
+  borderBottomWidth: 1,
+};

app/src/components/OutputsTable/types.ts

@@ -2,4 +2,4 @@ import { type RouterOutputs } from "~/utils/api";
 
 export type PromptVariant = NonNullable<RouterOutputs["promptVariants"]["list"]>[0];
 
-export type Scenario = NonNullable<RouterOutputs["scenarios"]["list"]>[0];
+export type Scenario = NonNullable<RouterOutputs["scenarios"]["list"]>["scenarios"][0];

app/src/components/RefinePromptModal/RefineAction.tsx

@@ -1,22 +1,23 @@
 import { HStack, Icon, Heading, Text, VStack, GridItem } from "@chakra-ui/react";
 import { type IconType } from "react-icons";
-import { refineOptions, type RefineOptionLabel } from "./refineOptions";
+import { BsStars } from "react-icons/bs";
 
-export const RefineOption = ({
+export const RefineAction = ({
   label,
-  activeLabel,
   icon,
+  desciption,
+  activeLabel,
   onClick,
   loading,
 }: {
-  label: RefineOptionLabel;
-  activeLabel: RefineOptionLabel | undefined;
-  icon: IconType;
-  onClick: (label: RefineOptionLabel) => void;
+  label: string;
+  icon?: IconType;
+  desciption: string;
+  activeLabel: string | undefined;
+  onClick: (label: string) => void;
   loading: boolean;
 }) => {
   const isActive = activeLabel === label;
-  const desciption = refineOptions[label].description;
 
   return (
     <GridItem w="80" h="44">
@@ -44,7 +45,7 @@ export const RefineOption = ({
         opacity={loading ? 0.5 : 1}
       >
         <HStack cursor="pointer" spacing={6} fontSize="sm" fontWeight="medium" color="gray.500">
-          <Icon as={icon} boxSize={12} />
+          <Icon as={icon || BsStars} boxSize={12} />
           <Heading size="md" fontFamily="inconsolata, monospace">
             {label}
           </Heading>

app/src/components/RefinePromptModal/RefinePromptModal.tsx

@@ -15,17 +15,16 @@ import {
   SimpleGrid,
 } from "@chakra-ui/react";
 import { BsStars } from "react-icons/bs";
-import { VscJson } from "react-icons/vsc";
-import { TfiThought } from "react-icons/tfi";
 import { api } from "~/utils/api";
-import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
 import { type PromptVariant } from "@prisma/client";
 import { useState } from "react";
 import CompareFunctions from "./CompareFunctions";
 import { CustomInstructionsInput } from "./CustomInstructionsInput";
-import { type RefineOptionLabel, refineOptions } from "./refineOptions";
-import { RefineOption } from "./RefineOption";
+import { RefineAction } from "./RefineAction";
 import { isObject, isString } from "lodash-es";
+import { type RefinementAction, type SupportedProvider } from "~/modelProviders/types";
+import frontendModelProviders from "~/modelProviders/frontendModelProviders";
 
 export const RefinePromptModal = ({
   variant,
@@ -35,26 +34,32 @@ export const RefinePromptModal = ({
   onClose: () => void;
 }) => {
   const utils = api.useContext();
+  const visibleScenarios = useVisibleScenarioIds();
+
+  const refinementActions =
+    frontendModelProviders[variant.modelProvider as SupportedProvider].refinementActions || {};
 
   const { mutateAsync: getModifiedPromptMutateAsync, data: refinedPromptFn } =
     api.promptVariants.getModifiedPromptFn.useMutation();
   const [instructions, setInstructions] = useState<string>("");
 
-  const [activeRefineOptionLabel, setActiveRefineOptionLabel] = useState<
-    RefineOptionLabel | undefined
-  >(undefined);
+  const [activeRefineActionLabel, setActiveRefineActionLabel] = useState<string | undefined>(
+    undefined,
+  );
 
   const [getModifiedPromptFn, modificationInProgress] = useHandledAsyncCallback(
-    async (label?: RefineOptionLabel) => {
+    async (label?: string) => {
       if (!variant.experimentId) return;
-      const updatedInstructions = label ? refineOptions[label].instructions : instructions;
-      setActiveRefineOptionLabel(label);
+      const updatedInstructions = label
+        ? (refinementActions[label] as RefinementAction).instructions
+        : instructions;
+      setActiveRefineActionLabel(label);
       await getModifiedPromptMutateAsync({
         id: variant.id,
         instructions: updatedInstructions,
       });
     },
-    [getModifiedPromptMutateAsync, onClose, variant, instructions, setActiveRefineOptionLabel],
+    [getModifiedPromptMutateAsync, onClose, variant, instructions, setActiveRefineActionLabel],
   );
 
   const replaceVariantMutation = api.promptVariants.replaceVariant.useMutation();
@@ -69,6 +74,7 @@ export const RefinePromptModal = ({
     await replaceVariantMutation.mutateAsync({
       id: variant.id,
       constructFn: refinedPromptFn,
+      streamScenarios: visibleScenarios,
     });
     await utils.promptVariants.list.invalidate();
     onClose();
@@ -92,25 +98,26 @@ export const RefinePromptModal = ({
         <ModalBody maxW="unset">
           <VStack spacing={8}>
             <VStack spacing={4}>
-              <SimpleGrid columns={{ base: 1, md: 2 }} spacing={8}>
-                <RefineOption
-                  label="Convert to function call"
-                  activeLabel={activeRefineOptionLabel}
-                  icon={VscJson}
-                  onClick={getModifiedPromptFn}
-                  loading={modificationInProgress}
-                />
-                <RefineOption
-                  label="Add chain of thought"
-                  activeLabel={activeRefineOptionLabel}
-                  icon={TfiThought}
-                  onClick={getModifiedPromptFn}
-                  loading={modificationInProgress}
-                />
-              </SimpleGrid>
-              <HStack>
-                <Text color="gray.500">or</Text>
-              </HStack>
+              {Object.keys(refinementActions).length && (
+                <>
+                  <SimpleGrid columns={{ base: 1, md: 2 }} spacing={8}>
+                    {Object.keys(refinementActions).map((label) => (
+                      <RefineAction
+                        key={label}
+                        label={label}
+                        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                        icon={refinementActions[label]!.icon}
+                        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                        desciption={refinementActions[label]!.description}
+                        activeLabel={activeRefineActionLabel}
+                        onClick={getModifiedPromptFn}
+                        loading={modificationInProgress}
+                      />
+                    ))}
+                  </SimpleGrid>
+                  <Text color="gray.500">or</Text>
+                </>
+              )}
               <CustomInstructionsInput
                 instructions={instructions}
                 setInstructions={setInstructions}

app/src/components/VariantHeader/VariantHeader.tsx

@@ -3,28 +3,33 @@ import { type PromptVariant } from "../OutputsTable/types";
 import { api } from "~/utils/api";
 import { RiDraggable } from "react-icons/ri";
 import { useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
-import { HStack, Icon, Text, GridItem } from "@chakra-ui/react"; // Changed here
+import { HStack, Icon, Text, GridItem, type GridItemProps } from "@chakra-ui/react"; // Changed here
 import { cellPadding, headerMinHeight } from "../constants";
 import AutoResizeTextArea from "../AutoResizeTextArea";
-import { stickyHeaderStyle } from "../OutputsTable/styles";
 import VariantHeaderMenuButton from "./VariantHeaderMenuButton";
 
-export default function VariantHeader(props: { variant: PromptVariant; canHide: boolean }) {
+export default function VariantHeader(
+  allProps: {
+    variant: PromptVariant;
+    canHide: boolean;
+  } & GridItemProps,
+) {
+  const { variant, canHide, ...gridItemProps } = allProps;
   const { canModify } = useExperimentAccess();
   const utils = api.useContext();
   const [isDragTarget, setIsDragTarget] = useState(false);
   const [isInputHovered, setIsInputHovered] = useState(false);
-  const [label, setLabel] = useState(props.variant.label);
+  const [label, setLabel] = useState(variant.label);
 
   const updateMutation = api.promptVariants.update.useMutation();
   const [onSaveLabel] = useHandledAsyncCallback(async () => {
-    if (label && label !== props.variant.label) {
+    if (label && label !== variant.label) {
       await updateMutation.mutateAsync({
-        id: props.variant.id,
+        id: variant.id,
         updates: { label: label },
       });
     }
-  }, [updateMutation, props.variant.id, props.variant.label, label]);
+  }, [updateMutation, variant.id, variant.label, label]);
 
   const reorderMutation = api.promptVariants.reorder.useMutation();
   const [onReorder] = useHandledAsyncCallback(
@@ -32,7 +37,7 @@ export default function VariantHeader(props: { variant: PromptVariant; canHide:
       e.preventDefault();
       setIsDragTarget(false);
       const draggedId = e.dataTransfer.getData("text/plain");
-      const droppedId = props.variant.id;
+      const droppedId = variant.id;
       if (!draggedId || !droppedId || draggedId === droppedId) return;
       await reorderMutation.mutateAsync({
         draggedId,
@@ -40,16 +45,26 @@ export default function VariantHeader(props: { variant: PromptVariant; canHide:
       });
       await utils.promptVariants.list.invalidate();
     },
-    [reorderMutation, props.variant.id],
+    [reorderMutation, variant.id],
   );
 
   const [menuOpen, setMenuOpen] = useState(false);
 
   if (!canModify) {
     return (
-      <GridItem padding={0} sx={stickyHeaderStyle} borderTopWidth={1}>
+      <GridItem
+        padding={0}
+        sx={{
+          position: "sticky",
+          top: "0",
+          // Ensure that the menu always appears above the sticky header of other variants
+          zIndex: menuOpen ? "dropdown" : 10,
+        }}
+        borderTopWidth={1}
+        {...gridItemProps}
+      >
         <Text fontSize={16} fontWeight="bold" px={cellPadding.x} py={cellPadding.y}>
-          {props.variant.label}
+          {variant.label}
         </Text>
       </GridItem>
     );
@@ -59,19 +74,21 @@ export default function VariantHeader(props: { variant: PromptVariant; canHide:
     <GridItem
       padding={0}
       sx={{
-        ...stickyHeaderStyle,
+        position: "sticky",
+        top: "0",
         // Ensure that the menu always appears above the sticky header of other variants
-        zIndex: menuOpen ? "dropdown" : stickyHeaderStyle.zIndex,
+        zIndex: menuOpen ? "dropdown" : 10,
       }}
       borderTopWidth={1}
+      {...gridItemProps}
     >
       <HStack
-        spacing={4}
+        spacing={2}
         alignItems="flex-start"
         minH={headerMinHeight}
         draggable={!isInputHovered}
         onDragStart={(e) => {
-          e.dataTransfer.setData("text/plain", props.variant.id);
+          e.dataTransfer.setData("text/plain", variant.id);
           e.currentTarget.style.opacity = "0.4";
         }}
         onDragEnd={(e) => {
@@ -85,7 +102,8 @@ export default function VariantHeader(props: { variant: PromptVariant; canHide:
           setIsDragTarget(false);
         }}
         onDrop={onReorder}
-        backgroundColor={isDragTarget ? "gray.100" : "transparent"}
+        backgroundColor={isDragTarget ? "gray.200" : "gray.100"}
+        h="full"
       >
         <Icon
           as={RiDraggable}
@@ -112,8 +130,8 @@ export default function VariantHeader(props: { variant: PromptVariant; canHide:
           onMouseLeave={() => setIsInputHovered(false)}
         />
         <VariantHeaderMenuButton
-          variant={props.variant}
-          canHide={props.canHide}
+          variant={variant}
+          canHide={canHide}
           menuOpen={menuOpen}
           setMenuOpen={setMenuOpen}
         />

app/src/components/VariantHeader/VariantHeaderMenuButton.tsx

@@ -1,8 +1,7 @@
 import { type PromptVariant } from "../OutputsTable/types";
 import { api } from "~/utils/api";
-import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
 import {
-  Button,
   Icon,
   Menu,
   MenuButton,
@@ -11,13 +10,14 @@ import {
   MenuDivider,
   Text,
   Spinner,
+  IconButton,
 } from "@chakra-ui/react";
 import { BsFillTrashFill, BsGear, BsStars } from "react-icons/bs";
 import { FaRegClone } from "react-icons/fa";
 import { useState } from "react";
 import { RefinePromptModal } from "../RefinePromptModal/RefinePromptModal";
 import { RiExchangeFundsFill } from "react-icons/ri";
-import { SelectModelModal } from "../SelectModelModal/SelectModelModal";
+import { ChangeModelModal } from "../ChangeModelModal/ChangeModelModal";
 
 export default function VariantHeaderMenuButton({
   variant,
@@ -33,11 +33,13 @@ export default function VariantHeaderMenuButton({
   const utils = api.useContext();
 
   const duplicateMutation = api.promptVariants.create.useMutation();
+  const visibleScenarios = useVisibleScenarioIds();
 
   const [duplicateVariant, duplicationInProgress] = useHandledAsyncCallback(async () => {
     await duplicateMutation.mutateAsync({
       experimentId: variant.experimentId,
       variantId: variant.id,
+      streamScenarios: visibleScenarios,
     });
     await utils.promptVariants.list.invalidate();
   }, [duplicateMutation, variant.experimentId, variant.id]);
@@ -50,21 +52,18 @@ export default function VariantHeaderMenuButton({
     await utils.promptVariants.list.invalidate();
   }, [hideMutation, variant.id]);
 
-  const [selectModelModalOpen, setSelectModelModalOpen] = useState(false);
+  const [changeModelModalOpen, setChangeModelModalOpen] = useState(false);
   const [refinePromptModalOpen, setRefinePromptModalOpen] = useState(false);
 
   return (
     <>
       <Menu isOpen={menuOpen} onOpen={() => setMenuOpen(true)} onClose={() => setMenuOpen(false)}>
-        {duplicationInProgress ? (
-          <Spinner boxSize={4} mx={3} my={3} />
-        ) : (
-          <MenuButton>
-            <Button variant="ghost">
-              <Icon as={BsGear} />
-            </Button>
-          </MenuButton>
-        )}
+        <MenuButton
+          as={IconButton}
+          variant="ghost"
+          aria-label="Edit Scenarios"
+          icon={<Icon as={duplicationInProgress ? Spinner : BsGear} />}
+        />
 
         <MenuList mt={-3} fontSize="md">
           <MenuItem icon={<Icon as={FaRegClone} boxSize={4} w={5} />} onClick={duplicateVariant}>
@@ -72,7 +71,7 @@ export default function VariantHeaderMenuButton({
           </MenuItem>
           <MenuItem
             icon={<Icon as={RiExchangeFundsFill} boxSize={5} />}
-            onClick={() => setSelectModelModalOpen(true)}
+            onClick={() => setChangeModelModalOpen(true)}
           >
             Change Model
           </MenuItem>
@@ -97,8 +96,8 @@ export default function VariantHeaderMenuButton({
           )}
         </MenuList>
       </Menu>
-      {selectModelModalOpen && (
-        <SelectModelModal variant={variant} onClose={() => setSelectModelModalOpen(false)} />
+      {changeModelModalOpen && (
+        <ChangeModelModal variant={variant} onClose={() => setChangeModelModalOpen(false)} />
       )}
       {refinePromptModalOpen && (
         <RefinePromptModal variant={variant} onClose={() => setRefinePromptModalOpen(false)} />

app/src/components/experiments/ExperimentCard.tsx

@@ -1,4 +1,13 @@
-import { HStack, Icon, VStack, Text, Divider, Spinner, AspectRatio } from "@chakra-ui/react";
+import {
+  HStack,
+  Icon,
+  VStack,
+  Text,
+  Divider,
+  Spinner,
+  AspectRatio,
+  SkeletonText,
+} from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
 import { formatTimePast } from "~/utils/dayjs";
 import Link from "next/link";
@@ -93,3 +102,13 @@ export const NewExperimentCard = () => {
     </AspectRatio>
   );
 };
+
+export const ExperimentCardSkeleton = () => (
+  <AspectRatio ratio={1.2} w="full">
+    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="gray.50">
+      <SkeletonText noOfLines={1} w="80%" />
+      <SkeletonText noOfLines={2} w="60%" />
+      <SkeletonText noOfLines={1} w="80%" />
+    </VStack>
+  </AspectRatio>
+);

app/src/components/experiments/HeaderButtons/DeleteDialog.tsx

@@ -0,0 +1,57 @@
+import {
+  Button,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+} from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import { useRef } from "react";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+
+export const DeleteDialog = ({ onClose }: { onClose: () => void }) => {
+  const experiment = useExperiment();
+  const deleteMutation = api.experiments.delete.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    await deleteMutation.mutateAsync({ id: experiment.data.id });
+    await utils.experiments.list.invalidate();
+    await router.push({ pathname: "/experiments" });
+    onClose();
+  }, [deleteMutation, experiment.data?.id, router]);
+
+  return (
+    <AlertDialog isOpen leastDestructiveRef={cancelRef} onClose={onClose}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Delete Experiment
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            If you delete this experiment all the associated prompts and scenarios will be deleted
+            as well. Are you sure?
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={onClose}>
+              Cancel
+            </Button>
+            <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
+              Delete
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};

app/src/components/experiments/HeaderButtons/HeaderButtons.tsx

@@ -0,0 +1,42 @@
+import { Button, HStack, Icon, Spinner, Text } from "@chakra-ui/react";
+import { useOnForkButtonPressed } from "./useOnForkButtonPressed";
+import { useExperiment } from "~/utils/hooks";
+import { BsGearFill } from "react-icons/bs";
+import { TbGitFork } from "react-icons/tb";
+import { useAppStore } from "~/state/store";
+
+export const HeaderButtons = () => {
+  const experiment = useExperiment();
+
+  const canModify = experiment.data?.access.canModify ?? false;
+
+  const { onForkButtonPressed, isForking } = useOnForkButtonPressed();
+
+  const openDrawer = useAppStore((s) => s.openDrawer);
+
+  if (experiment.isLoading) return null;
+
+  return (
+    <HStack spacing={0} mt={{ base: 2, md: 0 }}>
+      <Button
+        onClick={onForkButtonPressed}
+        mr={4}
+        colorScheme={canModify ? undefined : "orange"}
+        bgColor={canModify ? undefined : "orange.400"}
+        minW={0}
+        variant={{ base: "solid", md: canModify ? "ghost" : "solid" }}
+      >
+        {isForking ? <Spinner boxSize={5} /> : <Icon as={TbGitFork} boxSize={5} />}
+        <Text ml={2}>Fork</Text>
+      </Button>
+      {canModify && (
+        <Button variant={{ base: "solid", md: "ghost" }} onClick={openDrawer}>
+          <HStack>
+            <Icon as={BsGearFill} />
+            <Text>Settings</Text>
+          </HStack>
+        </Button>
+      )}
+    </HStack>
+  );
+};

app/src/components/experiments/HeaderButtons/useOnForkButtonPressed.tsx

@@ -0,0 +1,30 @@
+import { useCallback } from "react";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { signIn, useSession } from "next-auth/react";
+import { useRouter } from "next/router";
+
+export const useOnForkButtonPressed = () => {
+  const router = useRouter();
+
+  const user = useSession().data;
+  const experiment = useExperiment();
+
+  const forkMutation = api.experiments.fork.useMutation();
+
+  const [onFork, isForking] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    const forkedExperimentId = await forkMutation.mutateAsync({ id: experiment.data.id });
+    await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
+  }, [forkMutation, experiment.data?.id, router]);
+
+  const onForkButtonPressed = useCallback(() => {
+    if (user === null) {
+      signIn("github").catch(console.error);
+    } else {
+      onFork();
+    }
+  }, [onFork, user]);
+
+  return { onForkButtonPressed, isForking };
+};

app/src/env.mjs

@@ -9,7 +9,6 @@ export const env = createEnv({
   server: {
     DATABASE_URL: z.string().url(),
     NODE_ENV: z.enum(["development", "test", "production"]).default("development"),
-    OPENAI_API_KEY: z.string().min(1),
     RESTRICT_PRISMA_LOGS: z
       .string()
       .optional()
@@ -17,7 +16,9 @@ export const env = createEnv({
       .transform((val) => val.toLowerCase() === "true"),
     GITHUB_CLIENT_ID: z.string().min(1),
     GITHUB_CLIENT_SECRET: z.string().min(1),
-    REPLICATE_API_TOKEN: z.string().min(1),
+    OPENAI_API_KEY: z.string().min(1),
+    REPLICATE_API_TOKEN: z.string().default("placeholder"),
+    ANTHROPIC_API_KEY: z.string().default("placeholder"),
   },
 
   /**
@@ -44,6 +45,7 @@ export const env = createEnv({
     GITHUB_CLIENT_ID: process.env.GITHUB_CLIENT_ID,
     GITHUB_CLIENT_SECRET: process.env.GITHUB_CLIENT_SECRET,
     REPLICATE_API_TOKEN: process.env.REPLICATE_API_TOKEN,
+    ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/modelProviders/anthropic/codegen/codegen.ts

@@ -0,0 +1,69 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+
+import YAML from "yaml";
+import fs from "fs";
+import path from "path";
+import { openapiSchemaToJsonSchema } from "@openapi-contrib/openapi-schema-to-json-schema";
+import $RefParser from "@apidevtools/json-schema-ref-parser";
+import { type JSONObject } from "superjson/dist/types";
+import assert from "assert";
+import { type JSONSchema4Object } from "json-schema";
+import { isObject } from "lodash-es";
+
+// @ts-expect-error for some reason missing from types
+import parserEstree from "prettier/plugins/estree";
+import parserBabel from "prettier/plugins/babel";
+import prettier from "prettier/standalone";
+
+const OPENAPI_URL =
+  "https://raw.githubusercontent.com/tryAGI/Anthropic/1c0871e861de60a4c3a843cb90e17d63e86c234a/docs/openapi.yaml";
+
+// Fetch the openapi document
+const response = await fetch(OPENAPI_URL);
+const openApiYaml = await response.text();
+
+// Parse the yaml document
+let schema = YAML.parse(openApiYaml) as JSONObject;
+schema = openapiSchemaToJsonSchema(schema);
+
+const jsonSchema = await $RefParser.dereference(schema);
+
+assert("components" in jsonSchema);
+const completionRequestSchema = jsonSchema.components.schemas
+  .CreateCompletionRequest as JSONSchema4Object;
+
+// We need to do a bit of surgery here since the Monaco editor doesn't like
+// the fact that the schema says `model` can be either a string or an enum,
+// and displays a warning in the editor. Let's stick with just an enum for
+// now and drop the string option.
+assert(
+  "properties" in completionRequestSchema &&
+    isObject(completionRequestSchema.properties) &&
+    "model" in completionRequestSchema.properties &&
+    isObject(completionRequestSchema.properties.model),
+);
+
+const modelProperty = completionRequestSchema.properties.model;
+assert(
+  "oneOf" in modelProperty &&
+    Array.isArray(modelProperty.oneOf) &&
+    modelProperty.oneOf.length === 2 &&
+    isObject(modelProperty.oneOf[1]) &&
+    "enum" in modelProperty.oneOf[1],
+  "Expected model to have oneOf length of 2",
+);
+modelProperty.type = "string";
+modelProperty.enum = modelProperty.oneOf[1].enum;
+delete modelProperty["oneOf"];
+
+// Get the directory of the current script
+const currentDirectory = path.dirname(import.meta.url).replace("file://", "");
+
+// Write the JSON schema to a file in the current directory
+fs.writeFileSync(
+  path.join(currentDirectory, "input.schema.json"),
+  await prettier.format(JSON.stringify(completionRequestSchema, null, 2), {
+    parser: "json",
+    plugins: [parserBabel, parserEstree],
+  }),
+);

app/src/modelProviders/anthropic/codegen/input.schema.json

@@ -0,0 +1,129 @@
+{
+  "type": "object",
+  "properties": {
+    "model": {
+      "description": "The model that will complete your prompt.\nAs we improve Claude, we develop new versions of it that you can query.\nThis parameter controls which version of Claude answers your request.\nRight now we are offering two model families: Claude, and Claude Instant.\nYou can use them by setting model to \"claude-2\" or \"claude-instant-1\", respectively.\nSee models for additional details.\n",
+      "x-oaiTypeLabel": "string",
+      "type": "string",
+      "enum": [
+        "claude-2",
+        "claude-2.0",
+        "claude-instant-1",
+        "claude-instant-1.1"
+      ]
+    },
+    "prompt": {
+      "description": "The prompt that you want Claude to complete.\n\nFor proper response generation you will need to format your prompt as follows:\n\\n\\nHuman: ${userQuestion}\\n\\nAssistant:\nSee our comments on prompts for more context.\n",
+      "default": "<|endoftext|>",
+      "nullable": true,
+      "oneOf": [
+        {
+          "type": "string",
+          "default": "",
+          "example": "This is a test."
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "default": "",
+            "example": "This is a test."
+          }
+        },
+        {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "integer"
+          },
+          "example": "[1212, 318, 257, 1332, 13]"
+        },
+        {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "integer"
+            }
+          },
+          "example": "[[1212, 318, 257, 1332, 13]]"
+        }
+      ]
+    },
+    "max_tokens_to_sample": {
+      "type": "integer",
+      "minimum": 1,
+      "default": 256,
+      "example": 256,
+      "nullable": true,
+      "description": "The maximum number of tokens to generate before stopping.\n\nNote that our models may stop before reaching this maximum. This parameter only specifies the absolute maximum number of tokens to generate.\n"
+    },
+    "temperature": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "default": 1,
+      "example": 1,
+      "nullable": true,
+      "description": "Amount of randomness injected into the response.\n\nDefaults to 1. Ranges from 0 to 1. Use temp closer to 0 for analytical / multiple choice, and closer to 1 for creative and generative tasks.\n"
+    },
+    "top_p": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "default": 1,
+      "example": 1,
+      "nullable": true,
+      "description": "Use nucleus sampling.\n\nIn nucleus sampling, we compute the cumulative distribution over all the options \nfor each subsequent token in decreasing probability order and cut it off once \nit reaches a particular probability specified by top_p. You should either alter temperature or top_p, but not both.\n"
+    },
+    "top_k": {
+      "type": "number",
+      "minimum": 0,
+      "default": 5,
+      "example": 5,
+      "nullable": true,
+      "description": "Only sample from the top K options for each subsequent token.\n\nUsed to remove \"long tail\" low probability responses. Learn more technical details here.\n"
+    },
+    "stream": {
+      "description": "Whether to incrementally stream the response using server-sent events.\nSee this guide to SSE events for details.type: boolean\n",
+      "nullable": true,
+      "default": false
+    },
+    "stop_sequences": {
+      "description": "Sequences that will cause the model to stop generating completion text.\nOur models stop on \"\\n\\nHuman:\", and may include additional built-in stop sequences in the future. By providing the stop_sequences parameter, you may include additional strings that will cause the model to stop generating.\n",
+      "default": null,
+      "nullable": true,
+      "oneOf": [
+        {
+          "type": "string",
+          "default": "<|endoftext|>",
+          "example": "\n",
+          "nullable": true
+        },
+        {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 4,
+          "items": {
+            "type": "string",
+            "example": "[\"\\n\"]"
+          }
+        }
+      ]
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "user_id": {
+          "type": "string",
+          "example": "13803d75-b4b5-4c3e-b2a2-6f21399b021b",
+          "description": "An external identifier for the user who is associated with the request.\n\nThis should be a uuid, hash value, or other opaque identifier. Anthropic may use this id to help detect abuse. \nDo not include any identifying information such as name, email address, or phone number.\n"
+        }
+      },
+      "description": "An object describing metadata about the request.\n"
+    }
+  },
+  "required": ["model", "prompt", "max_tokens_to_sample"]
+}

app/src/modelProviders/anthropic/frontend.ts

@@ -0,0 +1,40 @@
+import { type Completion } from "@anthropic-ai/sdk/resources";
+import { type SupportedModel } from ".";
+import { type FrontendModelProvider } from "../types";
+import { refinementActions } from "./refinementActions";
+
+const frontendModelProvider: FrontendModelProvider<SupportedModel, Completion> = {
+  name: "Replicate Llama2",
+
+  models: {
+    "claude-2.0": {
+      name: "Claude 2.0",
+      contextWindow: 100000,
+      promptTokenPrice: 11.02 / 1000000,
+      completionTokenPrice: 32.68 / 1000000,
+      speed: "medium",
+      provider: "anthropic",
+      learnMoreUrl: "https://www.anthropic.com/product",
+    },
+    "claude-instant-1.1": {
+      name: "Claude Instant 1.1",
+      contextWindow: 100000,
+      promptTokenPrice: 1.63 / 1000000,
+      completionTokenPrice: 5.51 / 1000000,
+      speed: "fast",
+      provider: "anthropic",
+      learnMoreUrl: "https://www.anthropic.com/product",
+    },
+  },
+
+  refinementActions,
+
+  normalizeOutput: (output) => {
+    return {
+      type: "text",
+      value: output.completion,
+    };
+  },
+};
+
+export default frontendModelProvider;

app/src/modelProviders/anthropic/getCompletion.ts

@@ -0,0 +1,86 @@
+import { env } from "~/env.mjs";
+import { type CompletionResponse } from "../types";
+
+import Anthropic, { APIError } from "@anthropic-ai/sdk";
+import { type Completion, type CompletionCreateParams } from "@anthropic-ai/sdk/resources";
+import { isObject, isString } from "lodash-es";
+
+const anthropic = new Anthropic({
+  apiKey: env.ANTHROPIC_API_KEY,
+});
+
+export async function getCompletion(
+  input: CompletionCreateParams,
+  onStream: ((partialOutput: Completion) => void) | null,
+): Promise<CompletionResponse<Completion>> {
+  const start = Date.now();
+  let finalCompletion: Completion | null = null;
+
+  try {
+    if (onStream) {
+      const resp = await anthropic.completions.create(
+        { ...input, stream: true },
+        {
+          maxRetries: 0,
+        },
+      );
+
+      for await (const part of resp) {
+        if (finalCompletion === null) {
+          finalCompletion = part;
+        } else {
+          finalCompletion = { ...part, completion: finalCompletion.completion + part.completion };
+        }
+        onStream(finalCompletion);
+      }
+      if (!finalCompletion) {
+        return {
+          type: "error",
+          message: "Streaming failed to return a completion",
+          autoRetry: false,
+        };
+      }
+    } else {
+      const resp = await anthropic.completions.create(
+        { ...input, stream: false },
+        {
+          maxRetries: 0,
+        },
+      );
+      finalCompletion = resp;
+    }
+    const timeToComplete = Date.now() - start;
+
+    return {
+      type: "success",
+      statusCode: 200,
+      value: finalCompletion,
+      timeToComplete,
+    };
+  } catch (error: unknown) {
+    console.log("CAUGHT ERROR", error);
+    if (error instanceof APIError) {
+      const message =
+        isObject(error.error) &&
+        "error" in error.error &&
+        isObject(error.error.error) &&
+        "message" in error.error.error &&
+        isString(error.error.error.message)
+          ? error.error.error.message
+          : error.message;
+
+      return {
+        type: "error",
+        message,
+        autoRetry: error.status === 429 || error.status === 503,
+        statusCode: error.status,
+      };
+    } else {
+      return {
+        type: "error",
+        message: (error as Error).message,
+        autoRetry: true,
+      };
+    }
+  }
+}

app/src/modelProviders/anthropic/index.ts

@@ -0,0 +1,34 @@
+import { type JSONSchema4 } from "json-schema";
+import { type ModelProvider } from "../types";
+import inputSchema from "./codegen/input.schema.json";
+import { getCompletion } from "./getCompletion";
+import frontendModelProvider from "./frontend";
+import type { Completion, CompletionCreateParams } from "@anthropic-ai/sdk/resources";
+
+const supportedModels = ["claude-2.0", "claude-instant-1.1"] as const;
+
+export type SupportedModel = (typeof supportedModels)[number];
+
+export type AnthropicProvider = ModelProvider<SupportedModel, CompletionCreateParams, Completion>;
+
+const modelProvider: AnthropicProvider = {
+  getModel: (input) => {
+    if (supportedModels.includes(input.model as SupportedModel))
+      return input.model as SupportedModel;
+
+    const modelMaps: Record<string, SupportedModel> = {
+      "claude-2": "claude-2.0",
+      "claude-instant-1": "claude-instant-1.1",
+    };
+
+    if (input.model in modelMaps) return modelMaps[input.model] as SupportedModel;
+
+    return null;
+  },
+  inputSchema: inputSchema as JSONSchema4,
+  canStream: true,
+  getCompletion,
+  ...frontendModelProvider,
+};
+
+export default modelProvider;

app/src/modelProviders/anthropic/refinementActions.ts

@@ -0,0 +1,3 @@
+import { type RefinementAction } from "../types";
+
+export const refinementActions: Record<string, RefinementAction> = {};

app/src/modelProviders/frontendModelProviders.ts

@@ -1,15 +1,15 @@
 import openaiChatCompletionFrontend from "./openai-ChatCompletion/frontend";
 import replicateLlama2Frontend from "./replicate-llama2/frontend";
+import anthropicFrontend from "./anthropic/frontend";
 import { type SupportedProvider, type FrontendModelProvider } from "./types";
 
-// TODO: make sure we get a typescript error if you forget to add a provider here
-
 // Keep attributes here that need to be accessible from the frontend. We can't
 // just include them in the default `modelProviders` object because it has some
 // transient dependencies that can only be imported on the server.
 const frontendModelProviders: Record<SupportedProvider, FrontendModelProvider<any, any>> = {
   "openai/ChatCompletion": openaiChatCompletionFrontend,
   "replicate/llama2": replicateLlama2Frontend,
+  anthropic: anthropicFrontend,
 };
 
 export default frontendModelProviders;