Add smoketest (broken)

app/.env.example

@@ -26,8 +26,6 @@ NEXT_PUBLIC_SOCKET_URL="http://localhost:3318"
 NEXTAUTH_SECRET="your_secret"
 NEXTAUTH_URL="http://localhost:3000"
 
-NEXT_PUBLIC_HOST="http://localhost:3000"
-
 # Next Auth Github Provider
 GITHUB_CLIENT_ID="your_client_id"
 GITHUB_CLIENT_SECRET="your_secret"

app/.eslintrc.cjs

@@ -37,7 +37,6 @@ const config = {
       "warn",
       { vars: "all", varsIgnorePattern: "^_", args: "after-used", argsIgnorePattern: "^_" },
     ],
-    "react/no-unescaped-entities": "off",
   },
 };
 

app/.gitignore

@@ -1,9 +1,11 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
+# App files
+
 # dependencies
 /node_modules
 /.pnp
-.pnp.js
+/.pnp.js
 
 # testing
 /coverage
@@ -15,31 +17,28 @@
 # next.js
 /.next/
 /out/
-next-env.d.ts
+/next-env.d.ts
 
 # production
 /build
 
 # misc
-.DS_Store
-*.pem
+/.DS_Store
+/*.pem
 
 # debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-.pnpm-debug.log*
+/npm-debug.log*
+/yarn-debug.log*
+/yarn-error.log*
+/.pnpm-debug.log*
 
 # local env files
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
-.env
-.env*.local
+/.env
+/.env*.local
 
 # vercel
-.vercel
+/.vercel
 
 # typescript
-*.tsbuildinfo
-
-# Sentry Auth Token
-.sentryclirc
+/*.tsbuildinfo

app/@types/nextjs-routes.d.ts

@@ -12,18 +12,13 @@ declare module "nextjs-routes" {
 
   export type Route =
     | StaticRoute<"/account/signin">
+    | DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
-    | StaticRoute<"/api/experiments/og-image">
-    | StaticRoute<"/api/sentry-example-api">
+    | StaticRoute<"/api/openapi">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
-    | DynamicRoute<"/data/[id]", { "id": string }>
-    | StaticRoute<"/data">
     | DynamicRoute<"/experiments/[id]", { "id": string }>
     | StaticRoute<"/experiments">
-    | StaticRoute<"/">
-    | StaticRoute<"/sentry-example-page">
-    | StaticRoute<"/world-champs">
-    | StaticRoute<"/world-champs/signup">;
+    | StaticRoute<"/">;
 
   interface StaticRoute<Pathname> {
     pathname: Pathname;

app/Dockerfile

@@ -20,9 +20,6 @@ FROM base as builder
 # Include all NEXT_PUBLIC_* env vars here
 ARG NEXT_PUBLIC_POSTHOG_KEY
 ARG NEXT_PUBLIC_SOCKET_URL
-ARG NEXT_PUBLIC_HOST
-ARG NEXT_PUBLIC_SENTRY_DSN
-ARG SENTRY_AUTH_TOKEN
 
 WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules

app/README.md

@@ -1,62 +1,52 @@
-<!-- <img src="https://github.com/openpipe/openpipe/assets/41524992/ca59596e-eb80-40f9-921f-6d67f6e6d8fa" width="72px" /> -->
+<img src="https://github.com/openpipe/openpipe/assets/41524992/ca59596e-eb80-40f9-921f-6d67f6e6d8fa" width="72px" />
 
 # OpenPipe
 
-OpenPipe is a flexible playground for comparing and optimizing LLM prompts. It lets you quickly generate, test and compare candidate prompts, and can automatically [translate](#-translate-between-model-apis) those prompts between models.
-
-<img src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="demo">
-
-You can use our hosted version of OpenPipe at https://openpipe.ai. You can also clone this repository and [run it locally](#running-locally).
+OpenPipe is a flexible playground for comparing and optimizing LLM prompts. It lets you quickly generate, test and compare candidate prompts with realistic sample data.
 
 ## Sample Experiments
 
-These are simple experiments users have created that show how OpenPipe works. Feel free to fork them and start experimenting yourself.
+These are simple experiments users have created that show how OpenPipe works.
 
-- [Twitter Sentiment Analysis](https://app.openpipe.ai/experiments/62c20a73-2012-4a64-973c-4b665ad46a57)
+- [Country Capitals](https://app.openpipe.ai/experiments/11111111-1111-1111-1111-111111111111)
 - [Reddit User Needs](https://app.openpipe.ai/experiments/22222222-2222-2222-2222-222222222222)
 - [OpenAI Function Calls](https://app.openpipe.ai/experiments/2ebbdcb3-ed51-456e-87dc-91f72eaf3e2b)
 - [Activity Classification](https://app.openpipe.ai/experiments/3950940f-ab6b-4b74-841d-7e9dbc4e4ff8)
 
+<img src="https://github.com/openpipe/openpipe/assets/176426/fc7624c6-5b65-4d4d-82b7-4a816f3e5678" alt="demo" height="400px">
+
+You can use our hosted version of OpenPipe at [https://openpipe.ai]. You can also clone this repository and [run it locally](#running-locally).
+
+## High-Level Features
+
+**Configure Multiple Prompts**  
+Set up multiple prompt configurations and compare their output side-by-side. Each configuration can be configured independently.
+
+**Visualize Responses**  
+Inspect prompt completions side-by-side.
+
+**Test Many Inputs**  
+OpenPipe lets you _template_ a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broader coverage of your problem space than you'd get with manual testing.
+
+**🪄 Auto-generate Test Scenarios**  
+OpenPipe includes a tool to generate new test scenarios based on your existing prompts and scenarios. Just click "Autogenerate Scenario" to try it out!
+
+**Prompt Validation and Typeahead**  
+We use OpenAI's OpenAPI spec to automatically provide typeahead and validate prompts.
+
+<img alt="typeahead" src="https://github.com/openpipe/openpipe/assets/176426/acc638f8-d851-4742-8d01-fe6f98890840" height="300px">
+
+**Function Call Support**  
+Natively supports [OpenAI function calls](https://openai.com/blog/function-calling-and-other-api-updates) on supported models.
+
+<img height="300px" alt="function calls" src="https://github.com/openpipe/openpipe/assets/176426/48ad13fe-af2f-4294-bf32-62015597fd9b">
+
 ## Supported Models
 
 - All models available through the OpenAI [chat completion API](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
 - Llama2 [7b chat](https://replicate.com/a16z-infra/llama7b-v2-chat), [13b chat](https://replicate.com/a16z-infra/llama13b-v2-chat), [70b chat](https://replicate.com/replicate/llama70b-v2-chat).
 - Anthropic's [Claude 1 Instant](https://www.anthropic.com/index/introducing-claude) and [Claude 2](https://www.anthropic.com/index/claude-2)
 
-## Features
-
-### 🔍 Visualize Responses
-
-Inspect prompt completions side-by-side.
-
-### 🧪 Bulk-Test
-
-OpenPipe lets you _template_ a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broad coverage of your problem space.
-
-### 📟 Translate between Model APIs
-
-Write your prompt in one format and automatically convert it to work with any other model.
-
-<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/1e19ccf2-96b6-4e93-a3a5-1449710d1b5b" alt="translate between models">
-
-<br><br>
-
-### 🛠️ Refine Your Prompts Automatically
-
-Use a growing database of best-practice refinements to improve your prompts automatically.
-
-<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/87a27fe7-daef-445c-a5e2-1c82b23f9f99" alt="add function call">
-
-<br><br>
-
-### 🪄 Auto-generate Test Scenarios
-
-OpenPipe includes a tool to generate new test scenarios based on your existing prompts and scenarios. Just click "Autogenerate Scenario" to try it out!
-
-<img width="600" src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="auto-generate">
-
-<br><br>
-
 ## Running Locally
 
 1. Install [Postgresql](https://www.postgresql.org/download/).

app/next.config.mjs

@@ -0,0 +1,33 @@
+import nextRoutes from "nextjs-routes/config";
+
+/**
+ * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation. This is especially useful
+ * for Docker builds.
+ */
+await import("./src/env.mjs");
+
+/** @type {import("next").NextConfig} */
+const config = {
+  reactStrictMode: true,
+
+  /**
+   * If you have `experimental: { appDir: true }` set, then you must comment the below `i18n` config
+   * out.
+   *
+   * @see https://github.com/vercel/next.js/issues/41980
+   */
+  i18n: {
+    locales: ["en"],
+    defaultLocale: "en",
+  },
+
+  webpack: (config) => {
+    config.module.rules.push({
+      test: /\.txt$/,
+      use: "raw-loader",
+    });
+    return config;
+  },
+};
+
+export default nextRoutes()(config);

app/openapitools.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json",
+  "spaces": 2,
+  "generator-cli": {
+    "version": "6.6.0"
+  }
+}

app/package.json

@@ -16,20 +16,17 @@
     "postinstall": "prisma generate",
     "lint": "next lint",
     "start": "next start",
-    "codegen": "tsx src/codegen/export-openai-types.ts",
+    "codegen": "tsx src/codegen/export-client-types.ts",
     "seed": "tsx prisma/seed.ts",
-    "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
-    "test": "pnpm vitest --no-threads"
+    "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.5.8",
     "@apidevtools/json-schema-ref-parser": "^10.1.0",
     "@babel/preset-typescript": "^7.22.5",
     "@babel/standalone": "^7.22.9",
-    "@chakra-ui/anatomy": "^2.2.0",
     "@chakra-ui/next-js": "^2.1.4",
     "@chakra-ui/react": "^2.7.1",
-    "@chakra-ui/styled-system": "^2.9.1",
     "@emotion/react": "^11.11.1",
     "@emotion/server": "^11.11.0",
     "@emotion/styled": "^11.11.0",
@@ -37,7 +34,6 @@
     "@monaco-editor/loader": "^1.3.3",
     "@next-auth/prisma-adapter": "^1.0.5",
     "@prisma/client": "^4.14.0",
-    "@sentry/nextjs": "^7.61.0",
     "@t3-oss/env-nextjs": "^0.3.1",
     "@tabler/icons-react": "^2.22.0",
     "@tanstack/react-query": "^4.29.7",
@@ -45,7 +41,6 @@
     "@trpc/next": "^10.26.0",
     "@trpc/react-query": "^10.26.0",
     "@trpc/server": "^10.26.0",
-    "@vercel/og": "^0.5.9",
     "ast-types": "^0.14.2",
     "chroma-js": "^2.4.2",
     "concurrently": "^8.2.0",
@@ -66,19 +61,17 @@
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
     "next-query-params": "^4.2.3",
+    "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
-    "openai": "4.0.0-beta.7",
+    "openai": "4.0.0-beta.2",
     "pluralize": "^8.0.0",
-    "posthog-js": "^1.75.3",
-    "posthog-node": "^3.1.1",
+    "posthog-js": "^1.68.4",
     "prettier": "^3.0.0",
     "prismjs": "^1.29.0",
     "react": "18.2.0",
     "react-diff-viewer": "^3.1.1",
     "react-dom": "18.2.0",
-    "react-github-btn": "^1.4.0",
     "react-icons": "^4.10.1",
-    "react-json-tree": "^0.18.0",
     "react-select": "^5.7.4",
     "react-syntax-highlighter": "^15.5.0",
     "react-textarea-autosize": "^8.5.0",
@@ -87,6 +80,7 @@
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
     "superjson": "1.12.2",
+    "trpc-openapi": "^1.2.0",
     "tsx": "^3.12.7",
     "type-fest": "^4.0.0",
     "use-query-params": "^2.2.1",
@@ -114,7 +108,6 @@
     "@types/uuid": "^9.0.2",
     "@typescript-eslint/eslint-plugin": "^5.59.6",
     "@typescript-eslint/parser": "^5.59.6",
-    "csv-parse": "^5.4.0",
     "eslint": "^8.40.0",
     "eslint-config-next": "^13.4.2",
     "eslint-plugin-unused-imports": "^2.0.0",

app/prisma/migrations/20230730061804_add_logged_calls_and_data_flows/migration.sql

@@ -0,0 +1,64 @@
+-- AlterTable
+ALTER TABLE "Experiment" ADD COLUMN     "dataFlowId" UUID;
+
+-- AlterTable
+ALTER TABLE "ModelResponse" ADD COLUMN     "loggedCallId" UUID,
+ALTER COLUMN "scenarioVariantCellId" DROP NOT NULL;
+
+-- CreateTable
+CREATE TABLE "LoggedCall" (
+    "id" UUID NOT NULL,
+    "promptFunctionHash" TEXT NOT NULL,
+    "promptFunction" TEXT NOT NULL,
+    "prompt" JSONB NOT NULL,
+    "responsePayload" JSONB,
+    "scenarioVariables" JSONB NOT NULL,
+    "model" TEXT NOT NULL,
+    "modelProvider" TEXT NOT NULL,
+    "dataFlowId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCall_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "DataFlow" (
+    "id" UUID NOT NULL,
+    "label" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "DataFlow_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "ApiKey" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "key" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "ApiKey_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ApiKey_key_key" ON "ApiKey"("key");
+
+-- AddForeignKey
+ALTER TABLE "Experiment" ADD CONSTRAINT "Experiment_dataFlowId_fkey" FOREIGN KEY ("dataFlowId") REFERENCES "DataFlow"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ModelResponse" ADD CONSTRAINT "ModelResponse_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_dataFlowId_fkey" FOREIGN KEY ("dataFlowId") REFERENCES "DataFlow"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "DataFlow" ADD CONSTRAINT "DataFlow_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/schema.prisma

@@ -19,6 +19,9 @@ model Experiment {
     organizationId String        @db.Uuid
     organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
 
+    dataFlowId String?     @db.Uuid
+    dataFlow   DataFlow?   @relation(fields: [dataFlowId], references: [id], onDelete: Cascade)
+
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
@@ -31,11 +34,11 @@ model Experiment {
 model PromptVariant {
     id String @id @default(uuid()) @db.Uuid
 
-    label                    String
-    promptConstructor        String
-    promptConstructorVersion Int
-    model                    String
-    modelProvider            String
+    label              String
+    constructFn        String
+    constructFnVersion Int
+    model              String
+    modelProvider      String
 
     uiId      String  @default(uuid()) @db.Uuid
     visible   Boolean @default(true)
@@ -112,24 +115,27 @@ model ScenarioVariantCell {
 model ModelResponse {
     id String @id @default(uuid()) @db.Uuid
 
-    inputHash        String
-    requestedAt      DateTime?
-    receivedAt       DateTime?
-    output           Json?
-    cost             Float?
-    promptTokens     Int?
-    completionTokens Int?
-    statusCode       Int?
-    errorMessage     String?
-    retryTime        DateTime?
-    outdated         Boolean   @default(false)
+    inputHash           String
+    requestedAt         DateTime?
+    receivedAt          DateTime?
+    output              Json?
+    cost                Float?
+    promptTokens        Int?
+    completionTokens    Int?
+    statusCode          Int?
+    errorMessage        String?
+    retryTime           DateTime?
+    outdated            Boolean @default(false)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    scenarioVariantCellId String              @db.Uuid
-    scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
-    outputEvaluations     OutputEvaluation[]
+    scenarioVariantCellId String?              @db.Uuid
+    scenarioVariantCell   ScenarioVariantCell? @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
+    outputEvaluations      OutputEvaluation[]
+
+    loggedCallId String? @db.Uuid
+    loggedCall   LoggedCall? @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
 
     @@index([inputHash])
 }
@@ -150,8 +156,8 @@ model Evaluation {
     experimentId String     @db.Uuid
     experiment   Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade)
 
-    createdAt         DateTime           @default(now())
-    updatedAt         DateTime           @updatedAt
+    createdAt        DateTime           @default(now())
+    updatedAt        DateTime           @updatedAt
     outputEvaluations OutputEvaluation[]
 }
 
@@ -162,7 +168,7 @@ model OutputEvaluation {
     result  Float
     details String?
 
-    modelResponseId String        @db.Uuid
+    modelResponseId String      @db.Uuid
     modelResponse   ModelResponse @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
 
     evaluationId String     @db.Uuid
@@ -174,27 +180,46 @@ model OutputEvaluation {
     @@unique([modelResponseId, evaluationId])
 }
 
-model Dataset {
+model LoggedCall {
     id String @id @default(uuid()) @db.Uuid
 
-    name           String
-    datasetEntries DatasetEntry[]
+    promptFunctionHash String
+    promptFunction     String
+    prompt     Json
+    responsePayload    Json?
+    scenarioVariables Json
+    model              String
+    modelProvider      String
 
-    organizationId String       @db.Uuid
-    organization   Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    dataFlowId String     @db.Uuid
+    dataFlow   DataFlow   @relation(fields: [dataFlowId], references: [id], onDelete: Cascade)
+
+    modelResponse      ModelResponse[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 }
 
-model DatasetEntry {
+model DataFlow {
     id String @id @default(uuid()) @db.Uuid
 
-    input  String
-    output String?
+    label String
+    experiments   Experiment[]
+    loggedCalls      LoggedCall[]
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
 
-    datasetId String   @db.Uuid
-    dataset   Dataset? @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
+model ApiKey {
+    id        String  @id @default(uuid()) @db.Uuid
+
+    name      String
+    key     String  @unique
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -203,13 +228,14 @@ model DatasetEntry {
 model Organization {
     id                String  @id @default(uuid()) @db.Uuid
     personalOrgUserId String? @unique @db.Uuid
-    PersonalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+    personalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+    apiKeys         ApiKey[]
+    dataFlows         DataFlow[]
 
-    createdAt         DateTime           @default(now())
-    updatedAt         DateTime           @updatedAt
+    createdAt        DateTime           @default(now())
+    updatedAt        DateTime           @updatedAt
     organizationUsers OrganizationUser[]
     experiments       Experiment[]
-    datasets          Dataset[]
 }
 
 enum OrganizationUserRole {
@@ -235,20 +261,6 @@ model OrganizationUser {
     @@unique([organizationId, userId])
 }
 
-model WorldChampEntrant {
-    id String @id @default(uuid()) @db.Uuid
-
-    userId String @db.Uuid
-    user   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
-
-    approved Boolean @default(false)
-
-    createdAt DateTime @default(now())
-    updatedAt DateTime @updatedAt
-
-    @@unique([userId])
-}
-
 model Account {
     id                       String  @id @default(uuid()) @db.Uuid
     userId                   String  @db.Uuid
@@ -276,28 +288,16 @@ model Session {
     user         User     @relation(fields: [userId], references: [id], onDelete: Cascade)
 }
 
-enum UserRole {
-    ADMIN
-    USER
-}
-
 model User {
-    id            String    @id @default(uuid()) @db.Uuid
-    name          String?
-    email         String?   @unique
-    emailVerified DateTime?
-    image         String?
-
-    role UserRole @default(USER)
-
+    id                String             @id @default(uuid()) @db.Uuid
+    name              String?
+    email             String?            @unique
+    emailVerified     DateTime?
+    image             String?
     accounts          Account[]
     sessions          Session[]
     organizationUsers OrganizationUser[]
     organizations     Organization[]
-    worldChampEntrant WorldChampEntrant?
-
-    createdAt DateTime @default(now())
-    updatedAt DateTime @default(now()) @updatedAt
 }
 
 model VerificationToken {

app/prisma/seed.ts

@@ -1,7 +1,6 @@
 import { prisma } from "~/server/db";
 import dedent from "dedent";
 import { generateNewCell } from "~/server/utils/generateNewCell";
-import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111111";
 
@@ -52,8 +51,8 @@ await prisma.promptVariant.createMany({
       sortIndex: 0,
       model: "gpt-3.5-turbo-0613",
       modelProvider: "openai/ChatCompletion",
-      promptConstructorVersion,
-      promptConstructor: dedent`
+      constructFnVersion: 1,
+      constructFn: dedent`
         definePrompt("openai/ChatCompletion", {
           model: "gpt-3.5-turbo-0613",
           messages: [
@@ -71,8 +70,8 @@ await prisma.promptVariant.createMany({
       sortIndex: 1,
       model: "gpt-3.5-turbo-0613",
       modelProvider: "openai/ChatCompletion",
-      promptConstructorVersion,
-      promptConstructor: dedent`
+      constructFnVersion: 1,
+      constructFn: dedent`
         definePrompt("openai/ChatCompletion", {
           model: "gpt-3.5-turbo-0613",
           messages: [

app/run-prod.sh

@@ -5,9 +5,6 @@ set -e
 echo "Migrating the database"
 pnpm prisma migrate deploy
 
-echo "Migrating promptConstructors"
-pnpm tsx src/promptConstructor/migrate.ts
-
 echo "Starting the server"
 
 pnpm concurrently --kill-others \

app/src/codegen/export-client-types.ts

@@ -0,0 +1,42 @@
+import "dotenv/config";
+import { openApiDocument } from "~/pages/api/openapi.json";
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+import { generatePromptTypes } from "~/modelProviders/generatePromptTypes";
+
+console.log("Exporting public OpenAPI schema to client-libs/schema.json");
+
+const executionDir = import.meta.url.replace("file://", "");
+const schemaPath = path.join(
+  path.dirname(executionDir),
+  "../../../client-libs/schema.json"
+);
+
+console.log("Exporting schema");
+fs.writeFileSync(schemaPath, JSON.stringify(openApiDocument, null, 2), "utf-8");
+
+console.log("Generating Typescript client");
+
+const tsClientPath = path.join(
+  path.dirname(executionDir),
+  "../../../client-libs/js/codegen"
+);
+
+fs.rmSync(tsClientPath, { recursive: true, force: true });
+
+execSync(
+  `pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
+  {
+    stdio: "inherit",
+  }
+);
+
+const promptTypes = await generatePromptTypes();
+
+const promptTypesPath = path.join(tsClientPath, "promptTypes.ts");
+
+console.log(`Writing promptTypes to ${promptTypesPath}`);
+fs.writeFileSync(promptTypesPath, promptTypes, "utf-8");
+
+console.log("Done!");

app/src/components/ChangeModelModal/ChangeModelModal.tsx

@@ -68,7 +68,7 @@ export const ChangeModelModal = ({
       return;
     await replaceVariantMutation.mutateAsync({
       id: variant.id,
-      promptConstructor: modifiedPromptFn,
+      constructFn: modifiedPromptFn,
       streamScenarios: visibleScenarios,
     });
     await utils.promptVariants.list.invalidate();
@@ -107,7 +107,7 @@ export const ChangeModelModal = ({
             <ModelSearch selectedModel={selectedModel} setSelectedModel={setSelectedModel} />
             {isString(modifiedPromptFn) && (
               <CompareFunctions
-                originalFunction={variant.promptConstructor}
+                originalFunction={variant.constructFn}
                 newFunction={modifiedPromptFn}
                 leftTitle={originalLabel}
                 rightTitle={convertedLabel}

app/src/components/ChangeModelModal/ModelSearch.tsx

@@ -22,8 +22,8 @@ export const ModelSearch = (props: {
   const [containerRef, containerDimensions] = useElementDimensions();
 
   return (
-    <VStack ref={containerRef as LegacyRef<HTMLDivElement>} w="full" fontFamily="inconsolata">
-      <Text fontWeight="bold">Browse Models</Text>
+    <VStack ref={containerRef as LegacyRef<HTMLDivElement>} w="full">
+      <Text>Browse Models</Text>
       <Select<ProviderModel>
         styles={{ control: (provided) => ({ ...provided, width: containerDimensions?.width }) }}
         getOptionLabel={(data) => modelLabel(data.provider, data.model)}

app/src/components/ChangeModelModal/ModelStatsCard.tsx

@@ -23,24 +23,16 @@ export const ModelStatsCard = ({
         {label}
       </Text>
 
-      <VStack
-        w="full"
-        spacing={6}
-        borderWidth={1}
-        borderColor="gray.300"
-        p={4}
-        borderRadius={8}
-        fontFamily="inconsolata"
-      >
+      <VStack w="full" spacing={6} bgColor="gray.100" p={4} borderRadius={4}>
         <HStack w="full" align="flex-start">
-          <VStack flex={1} fontSize="lg" alignItems="flex-start">
+          <Text flex={1} fontSize="lg">
+            <Text as="span" color="gray.600">
+              {model.provider} /{" "}
+            </Text>
             <Text as="span" fontWeight="bold" color="gray.900">
               {model.name}
             </Text>
-            <Text as="span" color="gray.600" fontSize="sm">
-              Provider: {model.provider}
-            </Text>
-          </VStack>
+          </Text>
           <Link
             href={model.learnMoreUrl}
             isExternal

app/src/components/OutputsTable/FloatingLabelInput.tsx

@@ -37,6 +37,7 @@ export const FloatingLabelInput = ({
         borderColor={isFocused ? "blue.500" : "gray.400"}
         autoComplete="off"
         value={value}
+        maxHeight={32}
         overflowY="auto"
         overflowX="hidden"
         {...props}

app/src/components/OutputsTable/OutputCell/CellContent.tsx

@@ -0,0 +1,19 @@
+import { type StackProps, VStack } from "@chakra-ui/react";
+import { CellOptions } from "./CellOptions";
+
+export const CellContent = ({
+  hardRefetch,
+  hardRefetching,
+  children,
+  ...props
+}: {
+  hardRefetch: () => void;
+  hardRefetching: boolean;
+} & StackProps) => (
+  <VStack w="full" alignItems="flex-start" {...props}>
+    <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} />
+    <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto">
+      {children}
+    </VStack>
+  </VStack>
+);

app/src/components/OutputsTable/OutputCell/CellOptions.tsx

@@ -0,0 +1,37 @@
+import { Button, HStack, Icon, Spinner, Tooltip } from "@chakra-ui/react";
+import { BsArrowClockwise } from "react-icons/bs";
+import { useExperimentAccess } from "~/utils/hooks";
+
+export const CellOptions = ({
+  refetchingOutput,
+  refetchOutput,
+}: {
+  refetchingOutput: boolean;
+  refetchOutput: () => void;
+}) => {
+  const { canModify } = useExperimentAccess();
+  return (
+    <HStack justifyContent="flex-end" w="full">
+      {canModify && (
+        <Tooltip label="Refetch output" aria-label="refetch output">
+          <Button
+            size="xs"
+            w={4}
+            h={4}
+            py={4}
+            px={4}
+            minW={0}
+            borderRadius={8}
+            color="gray.500"
+            variant="ghost"
+            cursor="pointer"
+            onClick={refetchOutput}
+            aria-label="refetch output"
+          >
+            <Icon as={refetchingOutput ? Spinner : BsArrowClockwise} boxSize={4} />
+          </Button>
+        </Tooltip>
+      )}
+    </HStack>
+  );
+};

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,17 +1,17 @@
 import { api } from "~/utils/api";
 import { type PromptVariant, type Scenario } from "../types";
-import { type StackProps, Text, VStack } from "@chakra-ui/react";
+import { Text, VStack } from "@chakra-ui/react";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
 import stringify from "json-stringify-pretty-compact";
-import { type ReactElement, useState, useEffect, Fragment, useCallback } from "react";
+import { type ReactElement, useState, useEffect, Fragment } from "react";
 import useSocket from "~/utils/useSocket";
 import { OutputStats } from "./OutputStats";
 import { RetryCountdown } from "./RetryCountdown";
 import frontendModelProviders from "~/modelProviders/frontendModelProviders";
 import { ResponseLog } from "./ResponseLog";
-import { CellOptions } from "./TopActions";
+import { CellContent } from "./CellContent";
 
 const WAITING_MESSAGE_INTERVAL = 20000;
 
@@ -72,49 +72,37 @@ export default function OutputCell({
   // TODO: disconnect from socket if we're not streaming anymore
   const streamedMessage = useSocket<OutputSchema>(cell?.id);
 
-  const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
-
-  const CellWrapper = useCallback(
-    ({ children, ...props }: StackProps) => (
-      <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
-        {cell && (
-          <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
-        )}
-        <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
-          {children}
-        </VStack>
-        {mostRecentResponse && (
-          <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
-        )}
-      </VStack>
-    ),
-    [hardRefetching, hardRefetch, mostRecentResponse, scenario, cell],
-  );
-
   if (!vars) return null;
 
   if (!cell && !fetchingOutput)
     return (
-      <CellWrapper>
+      <CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
         <Text color="gray.500">Error retrieving output</Text>
-      </CellWrapper>
+      </CellContent>
     );
 
   if (cell && cell.errorMessage) {
     return (
-      <CellWrapper>
+      <CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
         <Text color="red.500">{cell.errorMessage}</Text>
-      </CellWrapper>
+      </CellContent>
     );
   }
 
   if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
 
+  const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
   const showLogs = !streamedMessage && !mostRecentResponse?.output;
 
   if (showLogs)
     return (
-      <CellWrapper alignItems="flex-start" fontFamily="inconsolata, monospace" spacing={0}>
+      <CellContent
+        hardRefetching={hardRefetching}
+        hardRefetch={hardRefetch}
+        alignItems="flex-start"
+        fontFamily="inconsolata, monospace"
+        spacing={0}
+      >
         {cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
         {cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
         {cell?.modelResponses?.map((response) => {
@@ -136,13 +124,9 @@ export default function OutputCell({
                 Array.from({ length: numWaitingMessages }, (_, i) => (
                   <ResponseLog
                     key={`waiting-${i}`}
-                    time={
-                      new Date(
-                        (response.requestedAt?.getTime?.() ?? 0) +
-                          (i + 1) * WAITING_MESSAGE_INTERVAL,
-                      )
-                    }
-                    title="Waiting for response..."
+                    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                    time={new Date(response.requestedAt!.getTime() + i * WAITING_MESSAGE_INTERVAL)}
+                    title="Waiting for response"
                   />
                 ))}
               {response.receivedAt && (
@@ -160,7 +144,7 @@ export default function OutputCell({
         {mostRecentResponse?.retryTime && (
           <RetryCountdown retryTime={mostRecentResponse.retryTime} />
         )}
-      </CellWrapper>
+      </CellContent>
     );
 
   const normalizedOutput = mostRecentResponse?.output
@@ -171,27 +155,50 @@ export default function OutputCell({
 
   if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
     return (
-      <CellWrapper>
-        <SyntaxHighlighter
-          customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
-          language="json"
-          style={docco}
-          lineProps={{
-            style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
-          }}
-          wrapLines
+      <VStack
+        w="100%"
+        h="100%"
+        fontSize="xs"
+        flexWrap="wrap"
+        overflowX="hidden"
+        justifyContent="space-between"
+      >
+        <CellContent
+          hardRefetching={hardRefetching}
+          hardRefetch={hardRefetch}
+          w="full"
+          flex={1}
+          spacing={0}
         >
-          {stringify(normalizedOutput.value, { maxLength: 40 })}
-        </SyntaxHighlighter>
-      </CellWrapper>
+          <SyntaxHighlighter
+            customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
+            language="json"
+            style={docco}
+            lineProps={{
+              style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
+            }}
+            wrapLines
+          >
+            {stringify(normalizedOutput.value, { maxLength: 40 })}
+          </SyntaxHighlighter>
+        </CellContent>
+        <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
+      </VStack>
     );
   }
 
   const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
 
   return (
-    <CellWrapper>
-      <Text>{contentToDisplay}</Text>
-    </CellWrapper>
+    <VStack w="100%" h="100%" justifyContent="space-between" whiteSpace="pre-wrap">
+      <VStack w="full" alignItems="flex-start" spacing={0}>
+        <CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
+          <Text>{contentToDisplay}</Text>
+        </CellContent>
+      </VStack>
+      {mostRecentResponse?.output && (
+        <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
+      )}
+    </VStack>
   );
 }

app/src/components/OutputsTable/OutputCell/OutputStats.tsx

@@ -23,15 +23,8 @@ export const OutputStats = ({
   const completionTokens = modelResponse.completionTokens;
 
   return (
-    <HStack
-      w="full"
-      align="center"
-      color="gray.500"
-      fontSize="2xs"
-      mt={{ base: 0, md: 1 }}
-      alignItems="flex-end"
-    >
-      <HStack flex={1} flexWrap="wrap">
+    <HStack w="full" align="center" color="gray.500" fontSize="2xs" mt={{ base: 0, md: 1 }}>
+      <HStack flex={1}>
         {modelResponse.outputEvaluations.map((evaluation) => {
           const passed = evaluation.result > 0.5;
           return (

app/src/components/OutputsTable/ScenarioEditor.tsx

@@ -0,0 +1,188 @@
+import { type DragEvent } from "react";
+import { api } from "~/utils/api";
+import { isEqual } from "lodash-es";
+import { type Scenario } from "./types";
+import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
+import { useState } from "react";
+
+import { Box, Button, Flex, HStack, Icon, Spinner, Stack, Tooltip, VStack } from "@chakra-ui/react";
+import { cellPadding } from "../constants";
+import { BsX } from "react-icons/bs";
+import { RiDraggable } from "react-icons/ri";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+
+export default function ScenarioEditor({
+  scenario,
+  ...props
+}: {
+  scenario: Scenario;
+  hovered: boolean;
+  canHide: boolean;
+}) {
+  const { canModify } = useExperimentAccess();
+
+  const savedValues = scenario.variableValues as Record<string, string>;
+  const utils = api.useContext();
+  const [isDragTarget, setIsDragTarget] = useState(false);
+  const [variableInputHovered, setVariableInputHovered] = useState(false);
+
+  const [values, setValues] = useState<Record<string, string>>(savedValues);
+
+  const experiment = useExperiment();
+  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+
+  const variableLabels = vars.data?.map((v) => v.label) ?? [];
+
+  const hasChanged = !isEqual(savedValues, values);
+
+  const mutation = api.scenarios.replaceWithValues.useMutation();
+
+  const [onSave] = useHandledAsyncCallback(async () => {
+    await mutation.mutateAsync({
+      id: scenario.id,
+      values,
+    });
+    await utils.scenarios.list.invalidate();
+  }, [mutation, values]);
+
+  const hideMutation = api.scenarios.hide.useMutation();
+  const [onHide, hidingInProgress] = useHandledAsyncCallback(async () => {
+    await hideMutation.mutateAsync({
+      id: scenario.id,
+    });
+    await utils.scenarios.list.invalidate();
+    await utils.promptVariants.stats.invalidate();
+  }, [hideMutation, scenario.id]);
+
+  const reorderMutation = api.scenarios.reorder.useMutation();
+  const [onReorder] = useHandledAsyncCallback(
+    async (e: DragEvent<HTMLDivElement>) => {
+      e.preventDefault();
+      setIsDragTarget(false);
+      const draggedId = e.dataTransfer.getData("text/plain");
+      const droppedId = scenario.id;
+      if (!draggedId || !droppedId || draggedId === droppedId) return;
+      await reorderMutation.mutateAsync({
+        draggedId,
+        droppedId,
+      });
+      await utils.scenarios.list.invalidate();
+    },
+    [reorderMutation, scenario.id],
+  );
+
+  return (
+    <HStack
+      alignItems="flex-start"
+      px={cellPadding.x}
+      py={cellPadding.y}
+      spacing={0}
+      height="100%"
+      draggable={!variableInputHovered}
+      onDragStart={(e) => {
+        e.dataTransfer.setData("text/plain", scenario.id);
+        e.currentTarget.style.opacity = "0.4";
+      }}
+      onDragEnd={(e) => {
+        e.currentTarget.style.opacity = "1";
+      }}
+      onDragOver={(e) => {
+        e.preventDefault();
+        setIsDragTarget(true);
+      }}
+      onDragLeave={() => {
+        setIsDragTarget(false);
+      }}
+      onDrop={onReorder}
+      backgroundColor={isDragTarget ? "gray.100" : "transparent"}
+    >
+      {canModify && props.canHide && (
+        <Stack
+          alignSelf="flex-start"
+          opacity={props.hovered ? 1 : 0}
+          spacing={0}
+          ml={-cellPadding.x}
+        >
+          <Tooltip label="Hide scenario" hasArrow>
+            {/* for some reason the tooltip can't position itself properly relative to the icon without the wrapping box */}
+            <Button
+              variant="unstyled"
+              color="gray.400"
+              height="unset"
+              width="unset"
+              minW="unset"
+              onClick={onHide}
+              _hover={{
+                color: "gray.800",
+                cursor: "pointer",
+              }}
+            >
+              <Icon as={hidingInProgress ? Spinner : BsX} boxSize={hidingInProgress ? 4 : 6} />
+            </Button>
+          </Tooltip>
+          <Icon
+            as={RiDraggable}
+            boxSize={6}
+            color="gray.400"
+            _hover={{ color: "gray.800", cursor: "pointer" }}
+          />
+        </Stack>
+      )}
+
+      {variableLabels.length === 0 ? (
+        <Box color="gray.500">{vars.data ? "No scenario variables configured" : "Loading..."}</Box>
+      ) : (
+        <VStack spacing={4} flex={1} py={2}>
+          {variableLabels.map((key) => {
+            const value = values[key] ?? "";
+            const layoutDirection = value.length > 20 ? "column" : "row";
+            return (
+              <Flex
+                key={key}
+                direction={layoutDirection}
+                alignItems={layoutDirection === "column" ? "flex-start" : "center"}
+                flexWrap="wrap"
+                width="full"
+              >
+                <FloatingLabelInput
+                  label={key}
+                  isDisabled={!canModify}
+                  style={{ width: "100%" }}
+                  value={value}
+                  onChange={(e) => {
+                    setValues((prev) => ({ ...prev, [key]: e.target.value }));
+                  }}
+                  onKeyDown={(e) => {
+                    if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
+                      e.preventDefault();
+                      e.currentTarget.blur();
+                      onSave();
+                    }
+                  }}
+                  onMouseEnter={() => setVariableInputHovered(true)}
+                  onMouseLeave={() => setVariableInputHovered(false)}
+                />
+              </Flex>
+            );
+          })}
+          {hasChanged && (
+            <HStack justify="right">
+              <Button
+                size="sm"
+                onMouseDown={() => {
+                  setValues(savedValues);
+                }}
+                colorScheme="gray"
+              >
+                Reset
+              </Button>
+              <Button size="sm" onMouseDown={onSave} colorScheme="blue">
+                Save
+              </Button>
+            </HStack>
+          )}
+        </VStack>
+      )}
+    </HStack>
+  );
+}

app/src/components/OutputsTable/ScenarioPaginator.tsx

@@ -5,20 +5,15 @@ import {
   BsChevronLeft,
   BsChevronRight,
 } from "react-icons/bs";
-import { usePage } from "~/utils/hooks";
+import { usePage, useScenarios } from "~/utils/hooks";
 
-const Paginator = ({
-  numItemsLoaded,
-  startIndex,
-  lastPage,
-  count,
-}: {
-  numItemsLoaded: number;
-  startIndex: number;
-  lastPage: number;
-  count: number;
-}) => {
+const ScenarioPaginator = () => {
   const [page, setPage] = usePage();
+  const { data } = useScenarios();
+
+  if (!data) return null;
+
+  const { scenarios, startIndex, lastPage, count } = data;
 
   const nextPage = () => {
     if (page < lastPage) {
@@ -54,7 +49,7 @@ const Paginator = ({
         icon={<BsChevronLeft />}
       />
       <Box>
-        {startIndex}-{startIndex + numItemsLoaded - 1} / {count}
+        {startIndex}-{startIndex + scenarios.length - 1} / {count}
       </Box>
       <IconButton
         variant="ghost"
@@ -76,4 +71,4 @@ const Paginator = ({
   );
 };
 
-export default Paginator;
+export default ScenarioPaginator;

app/src/components/OutputsTable/ScenarioRow.tsx

@@ -1,5 +1,6 @@
-import { GridItem } from "@chakra-ui/react";
+import { Box, GridItem } from "@chakra-ui/react";
 import React, { useState } from "react";
+import { cellPadding } from "../constants";
 import OutputCell from "./OutputCell/OutputCell";
 import ScenarioEditor from "./ScenarioEditor";
 import type { PromptVariant, Scenario } from "./types";
@@ -38,7 +39,9 @@ const ScenarioRow = (props: {
           colStart={i + 2}
           {...borders}
         >
-          <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />
+          <Box h="100%" w="100%" px={cellPadding.x} py={cellPadding.y}>
+            <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />
+          </Box>
         </GridItem>
       ))}
     </>

app/src/components/OutputsTable/VariantEditor.tsx

@@ -47,7 +47,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
     return () => window.removeEventListener("keydown", handleEsc);
   }, [isFullscreen, toggleFullscreen]);
 
-  const lastSavedFn = props.variant.promptConstructor;
+  const lastSavedFn = props.variant.constructFn;
 
   const modifierKey = useModifierKeyLabel();
 
@@ -96,7 +96,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
 
     const resp = await replaceVariant.mutateAsync({
       id: props.variant.id,
-      promptConstructor: currentFn,
+      constructFn: currentFn,
       streamScenarios: visibleScenarios,
     });
     if (resp.status === "error") {

app/src/components/OutputsTable/VariantStats.tsx

@@ -43,17 +43,17 @@ export default function VariantStats(props: { variant: PromptVariant }) {
   return (
     <HStack
       justifyContent="space-between"
-      alignItems="flex-end"
+      alignItems="center"
       mx="2"
       fontSize="xs"
       py={cellPadding.y}
     >
-      <HStack px={cellPadding.x} flexWrap="wrap">
-        {showNumFinished && (
-          <Text>
-            {data.outputCount} / {data.scenarioCount}
-          </Text>
-        )}
+      {showNumFinished && (
+        <Text>
+          {data.outputCount} / {data.scenarioCount}
+        </Text>
+      )}
+      <HStack px={cellPadding.x}>
         {data.evalResults.map((result) => {
           const passedFrac = result.passCount / result.totalCount;
           return (

app/src/components/OutputsTable/index.tsx

@@ -35,7 +35,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
       pb={24}
       pl={8}
       display="grid"
-      gridTemplateColumns={`250px repeat(${variants.data.length}, minmax(320px, 1fr)) auto`}
+      gridTemplateColumns={`250px repeat(${variants.data.length}, minmax(300px, 1fr)) auto`}
       sx={{
         "> *": {
           borderColor: "gray.300",

app/src/components/RefinePromptModal/CustomInstructionsInput.tsx

@@ -1,29 +1,18 @@
-import {
-  Button,
-  Spinner,
-  InputGroup,
-  InputRightElement,
-  Icon,
-  HStack,
-  type InputGroupProps,
-} from "@chakra-ui/react";
+import { Button, Spinner, InputGroup, InputRightElement, Icon, HStack } from "@chakra-ui/react";
 import { IoMdSend } from "react-icons/io";
-import AutoResizeTextArea from "./AutoResizeTextArea";
+import AutoResizeTextArea from "../AutoResizeTextArea";
 
 export const CustomInstructionsInput = ({
   instructions,
   setInstructions,
   loading,
   onSubmit,
-  placeholder = "Send custom instructions",
-  ...props
 }: {
   instructions: string;
   setInstructions: (instructions: string) => void;
   loading: boolean;
   onSubmit: () => void;
-  placeholder?: string;
-} & InputGroupProps) => {
+}) => {
   return (
     <InputGroup
       size="md"
@@ -33,7 +22,6 @@ export const CustomInstructionsInput = ({
       borderRadius={8}
       alignItems="center"
       colorScheme="orange"
-      {...props}
     >
       <AutoResizeTextArea
         value={instructions}
@@ -45,7 +33,7 @@ export const CustomInstructionsInput = ({
             onSubmit();
           }
         }}
-        placeholder={placeholder}
+        placeholder="Send custom instructions"
         py={4}
         pl={4}
         pr={12}

app/src/components/RefinePromptModal/RefinePromptModal.tsx

@@ -20,7 +20,7 @@ import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
 import { type PromptVariant } from "@prisma/client";
 import { useState } from "react";
 import CompareFunctions from "./CompareFunctions";
-import { CustomInstructionsInput } from "../CustomInstructionsInput";
+import { CustomInstructionsInput } from "./CustomInstructionsInput";
 import { RefineAction } from "./RefineAction";
 import { isObject, isString } from "lodash-es";
 import { type RefinementAction, type SupportedProvider } from "~/modelProviders/types";
@@ -73,7 +73,7 @@ export const RefinePromptModal = ({
       return;
     await replaceVariantMutation.mutateAsync({
       id: variant.id,
-      promptConstructor: refinedPromptFn,
+      constructFn: refinedPromptFn,
       streamScenarios: visibleScenarios,
     });
     await utils.promptVariants.list.invalidate();
@@ -97,7 +97,7 @@ export const RefinePromptModal = ({
         <ModalCloseButton />
         <ModalBody maxW="unset">
           <VStack spacing={8}>
-            <VStack spacing={4} w="full">
+            <VStack spacing={4}>
               {Object.keys(refinementActions).length && (
                 <>
                   <SimpleGrid columns={{ base: 1, md: 2 }} spacing={8}>
@@ -122,11 +122,11 @@ export const RefinePromptModal = ({
                 instructions={instructions}
                 setInstructions={setInstructions}
                 loading={modificationInProgress}
-                onSubmit={() => getModifiedPromptFn()}
+                onSubmit={getModifiedPromptFn}
               />
             </VStack>
             <CompareFunctions
-              originalFunction={variant.promptConstructor}
+              originalFunction={variant.constructFn}
               newFunction={isString(refinedPromptFn) ? refinedPromptFn : undefined}
               maxH="40vh"
             />

app/src/components/experiments/HeaderButtons/HeaderButtons.tsx

@@ -5,7 +5,7 @@ import { BsGearFill } from "react-icons/bs";
 import { TbGitFork } from "react-icons/tb";
 import { useAppStore } from "~/state/store";
 
-export const ExperimentHeaderButtons = () => {
+export const HeaderButtons = () => {
   const experiment = useExperiment();
 
   const canModify = experiment.data?.access.canModify ?? false;

app/src/components/nav/AppShell.tsx

@@ -8,43 +8,42 @@ import {
   Text,
   Box,
   type BoxProps,
-  Link as ChakraLink,
+  type LinkProps,
+  Link,
   Flex,
 } from "@chakra-ui/react";
 import Head from "next/head";
-import Link, { type LinkProps } from "next/link";
 import { BsGithub, BsPersonCircle } from "react-icons/bs";
 import { useRouter } from "next/router";
 import { type IconType } from "react-icons";
-import { RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
+import { RiFlaskLine } from "react-icons/ri";
 import { signIn, useSession } from "next-auth/react";
 import UserMenu from "./UserMenu";
-import { env } from "~/env.mjs";
 
-type IconLinkProps = BoxProps & LinkProps & { label?: string; icon: IconType; href: string };
+type IconLinkProps = BoxProps & LinkProps & { label?: string; icon: IconType };
 
-const IconLink = ({ icon, label, href, color, ...props }: IconLinkProps) => {
+const IconLink = ({ icon, label, href, target, color, ...props }: IconLinkProps) => {
   const router = useRouter();
   const isActive = href && router.pathname.startsWith(href);
   return (
-    <Link href={href} style={{ width: "100%" }}>
-      <HStack
-        w="full"
-        p={4}
-        color={color}
-        as={ChakraLink}
-        bgColor={isActive ? "gray.200" : "transparent"}
-        _hover={{ bgColor: "gray.300", textDecoration: "none" }}
-        justifyContent="start"
-        cursor="pointer"
-        {...props}
-      >
-        <Icon as={icon} boxSize={6} mr={2} />
-        <Text fontWeight="bold" fontSize="sm">
-          {label}
-        </Text>
-      </HStack>
-    </Link>
+    <HStack
+      w="full"
+      p={4}
+      color={color}
+      as={Link}
+      href={href}
+      target={target}
+      bgColor={isActive ? "gray.200" : "transparent"}
+      _hover={{ bgColor: "gray.200", textDecoration: "none" }}
+      justifyContent="start"
+      cursor="pointer"
+      {...props}
+    >
+      <Icon as={icon} boxSize={6} mr={2} />
+      <Text fontWeight="bold" fontSize="sm">
+        {label}
+      </Text>
+    </HStack>
   );
 };
 
@@ -73,37 +72,21 @@ const NavSidebar = () => {
         {user != null && (
           <>
             <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
-            {env.NEXT_PUBLIC_SHOW_DATA && (
-              <IconLink icon={RiDatabase2Line} label="Data" href="/data" />
-            )}
           </>
         )}
         {user === null && (
-          <HStack
-            w="full"
-            p={4}
-            as={ChakraLink}
-            _hover={{ bgColor: "gray.300", textDecoration: "none" }}
-            justifyContent="start"
-            cursor="pointer"
+          <IconLink
+            icon={BsPersonCircle}
+            label="Sign In"
             onClick={() => {
               signIn("github").catch(console.error);
             }}
-          >
-            <Icon as={BsPersonCircle} boxSize={6} mr={2} />
-            <Text fontWeight="bold" fontSize="sm">
-              Sign In
-            </Text>
-          </HStack>
+          />
         )}
       </VStack>
-      {user ? (
-        <UserMenu user={user} borderColor={"gray.200"} borderTopWidth={1} borderBottomWidth={1} />
-      ) : (
-        <Divider />
-      )}
+      {user ? <UserMenu user={user} /> : <Divider />}
       <VStack spacing={0} align="center">
-        <ChakraLink
+        <Link
           href="https://github.com/openpipe/openpipe"
           target="_blank"
           color="gray.500"
@@ -111,7 +94,7 @@ const NavSidebar = () => {
           p={2}
         >
           <Icon as={BsGithub} boxSize={6} />
-        </ChakraLink>
+        </Link>
       </VStack>
     </VStack>
   );

app/src/components/nav/UserMenu.tsx

@@ -8,16 +8,12 @@ import {
   PopoverTrigger,
   PopoverContent,
   Link,
-  useColorMode,
-  type StackProps,
 } from "@chakra-ui/react";
 import { type Session } from "next-auth";
 import { signOut } from "next-auth/react";
 import { BsBoxArrowRight, BsChevronRight, BsPersonCircle } from "react-icons/bs";
 
-export default function UserMenu({ user, ...rest }: { user: Session } & StackProps) {
-  const { colorMode } = useColorMode();
-
+export default function UserMenu({ user }: { user: Session }) {
   const profileImage = user.user.image ? (
     <Image src={user.user.image} alt="profile picture" boxSize={8} borderRadius="50%" />
   ) : (
@@ -33,10 +29,12 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
             px={3}
             spacing={3}
             py={2}
-            {...rest}
+            borderColor={"gray.200"}
+            borderTopWidth={1}
+            borderBottomWidth={1}
             cursor="pointer"
             _hover={{
-              bgColor: colorMode === "light" ? "gray.200" : "gray.700",
+              bgColor: "gray.200",
             }}
           >
             {profileImage}

app/src/env.mjs

@@ -19,7 +19,6 @@ export const env = createEnv({
     OPENAI_API_KEY: z.string().min(1),
     REPLICATE_API_TOKEN: z.string().default("placeholder"),
     ANTHROPIC_API_KEY: z.string().default("placeholder"),
-    SENTRY_AUTH_TOKEN: z.string().optional(),
   },
 
   /**
@@ -30,9 +29,6 @@ export const env = createEnv({
   client: {
     NEXT_PUBLIC_POSTHOG_KEY: z.string().optional(),
     NEXT_PUBLIC_SOCKET_URL: z.string().url().default("http://localhost:3318"),
-    NEXT_PUBLIC_HOST: z.string().url().default("http://localhost:3000"),
-    NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
-    NEXT_PUBLIC_SHOW_DATA: z.string().optional(),
   },
 
   /**
@@ -46,14 +42,10 @@ export const env = createEnv({
     RESTRICT_PRISMA_LOGS: process.env.RESTRICT_PRISMA_LOGS,
     NEXT_PUBLIC_POSTHOG_KEY: process.env.NEXT_PUBLIC_POSTHOG_KEY,
     NEXT_PUBLIC_SOCKET_URL: process.env.NEXT_PUBLIC_SOCKET_URL,
-    NEXT_PUBLIC_HOST: process.env.NEXT_PUBLIC_HOST,
-    NEXT_PUBLIC_SHOW_DATA: process.env.NEXT_PUBLIC_SHOW_DATA,
     GITHUB_CLIENT_ID: process.env.GITHUB_CLIENT_ID,
     GITHUB_CLIENT_SECRET: process.env.GITHUB_CLIENT_SECRET,
     REPLICATE_API_TOKEN: process.env.REPLICATE_API_TOKEN,
     ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
-    NEXT_PUBLIC_SENTRY_DSN: process.env.NEXT_PUBLIC_SENTRY_DSN,
-    SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/modelProviders/anthropic/codegen/input.schema.json

@@ -0,0 +1,129 @@
+{
+  "type": "object",
+  "properties": {
+    "model": {
+      "description": "The model that will complete your prompt.\nAs we improve Claude, we develop new versions of it that you can query.\nThis parameter controls which version of Claude answers your request.\nRight now we are offering two model families: Claude, and Claude Instant.\nYou can use them by setting model to \"claude-2\" or \"claude-instant-1\", respectively.\nSee models for additional details.\n",
+      "x-oaiTypeLabel": "string",
+      "type": "string",
+      "enum": [
+        "claude-2",
+        "claude-2.0",
+        "claude-instant-1",
+        "claude-instant-1.1"
+      ]
+    },
+    "prompt": {
+      "description": "The prompt that you want Claude to complete.\n\nFor proper response generation you will need to format your prompt as follows:\n\\n\\nHuman: ${userQuestion}\\n\\nAssistant:\nSee our comments on prompts for more context.\n",
+      "default": "<|endoftext|>",
+      "nullable": true,
+      "oneOf": [
+        {
+          "type": "string",
+          "default": "",
+          "example": "This is a test."
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "default": "",
+            "example": "This is a test."
+          }
+        },
+        {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "integer"
+          },
+          "example": "[1212, 318, 257, 1332, 13]"
+        },
+        {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "integer"
+            }
+          },
+          "example": "[[1212, 318, 257, 1332, 13]]"
+        }
+      ]
+    },
+    "max_tokens_to_sample": {
+      "type": "integer",
+      "minimum": 1,
+      "default": 256,
+      "example": 256,
+      "nullable": true,
+      "description": "The maximum number of tokens to generate before stopping.\n\nNote that our models may stop before reaching this maximum. This parameter only specifies the absolute maximum number of tokens to generate.\n"
+    },
+    "temperature": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "default": 1,
+      "example": 1,
+      "nullable": true,
+      "description": "Amount of randomness injected into the response.\n\nDefaults to 1. Ranges from 0 to 1. Use temp closer to 0 for analytical / multiple choice, and closer to 1 for creative and generative tasks.\n"
+    },
+    "top_p": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "default": 1,
+      "example": 1,
+      "nullable": true,
+      "description": "Use nucleus sampling.\n\nIn nucleus sampling, we compute the cumulative distribution over all the options \nfor each subsequent token in decreasing probability order and cut it off once \nit reaches a particular probability specified by top_p. You should either alter temperature or top_p, but not both.\n"
+    },
+    "top_k": {
+      "type": "number",
+      "minimum": 0,
+      "default": 5,
+      "example": 5,
+      "nullable": true,
+      "description": "Only sample from the top K options for each subsequent token.\n\nUsed to remove \"long tail\" low probability responses. Learn more technical details here.\n"
+    },
+    "stream": {
+      "description": "Whether to incrementally stream the response using server-sent events.\nSee this guide to SSE events for details.type: boolean\n",
+      "nullable": true,
+      "default": false
+    },
+    "stop_sequences": {
+      "description": "Sequences that will cause the model to stop generating completion text.\nOur models stop on \"\\n\\nHuman:\", and may include additional built-in stop sequences in the future. By providing the stop_sequences parameter, you may include additional strings that will cause the model to stop generating.\n",
+      "default": null,
+      "nullable": true,
+      "oneOf": [
+        {
+          "type": "string",
+          "default": "<|endoftext|>",
+          "example": "\n",
+          "nullable": true
+        },
+        {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 4,
+          "items": {
+            "type": "string",
+            "example": "[\"\\n\"]"
+          }
+        }
+      ]
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "user_id": {
+          "type": "string",
+          "example": "13803d75-b4b5-4c3e-b2a2-6f21399b021b",
+          "description": "An external identifier for the user who is associated with the request.\n\nThis should be a uuid, hash value, or other opaque identifier. Anthropic may use this id to help detect abuse. \nDo not include any identifying information such as name, email address, or phone number.\n"
+        }
+      },
+      "description": "An object describing metadata about the request.\n"
+    }
+  },
+  "required": ["model", "prompt", "max_tokens_to_sample"]
+}

app/src/modelProviders/anthropic/frontend.ts

@@ -13,9 +13,8 @@ const frontendModelProvider: FrontendModelProvider<SupportedModel, Completion> =
       promptTokenPrice: 11.02 / 1000000,
       completionTokenPrice: 32.68 / 1000000,
       speed: "medium",
-      provider: "anthropic/completion",
+      provider: "anthropic",
       learnMoreUrl: "https://www.anthropic.com/product",
-      apiDocsUrl: "https://docs.anthropic.com/claude/reference/complete_post",
     },
     "claude-instant-1.1": {
       name: "Claude Instant 1.1",
@@ -23,9 +22,8 @@ const frontendModelProvider: FrontendModelProvider<SupportedModel, Completion> =
       promptTokenPrice: 1.63 / 1000000,
       completionTokenPrice: 5.51 / 1000000,
       speed: "fast",
-      provider: "anthropic/completion",
+      provider: "anthropic",
       learnMoreUrl: "https://www.anthropic.com/product",
-      apiDocsUrl: "https://docs.anthropic.com/claude/reference/complete_post",
     },
   },
 

app/src/modelProviders/frontendModelProviders.ts

@@ -1,6 +1,6 @@
 import openaiChatCompletionFrontend from "./openai-ChatCompletion/frontend";
 import replicateLlama2Frontend from "./replicate-llama2/frontend";
-import anthropicFrontend from "./anthropic-completion/frontend";
+import anthropicFrontend from "./anthropic/frontend";
 import { type SupportedProvider, type FrontendModelProvider } from "./types";
 
 // Keep attributes here that need to be accessible from the frontend. We can't
@@ -9,7 +9,7 @@ import { type SupportedProvider, type FrontendModelProvider } from "./types";
 const frontendModelProviders: Record<SupportedProvider, FrontendModelProvider<any, any>> = {
   "openai/ChatCompletion": openaiChatCompletionFrontend,
   "replicate/llama2": replicateLlama2Frontend,
-  "anthropic/completion": anthropicFrontend,
+  anthropic: anthropicFrontend,
 };
 
 export default frontendModelProviders;

app/src/modelProviders/generatePromptTypes.ts

@@ -0,0 +1,41 @@
+import { type JSONSchema4Object } from "json-schema";
+import modelProviders from "./modelProviders";
+import { compile } from "json-schema-to-typescript";
+import dedent from "dedent";
+
+export const declarePromptTypes = async () => {
+  const promptTypes = (await generatePromptTypes()).replace(
+    /export interface PromptTypes/g,
+    "interface PromptTypes",
+  );
+
+  return dedent`
+    ${promptTypes}
+  
+    declare function definePrompt<T extends keyof PromptTypes>(modelProvider: T, input: PromptTypes[T])
+    `;
+};
+
+export const generatePromptTypes = async () => {
+  const combinedSchema = {
+    type: "object",
+    properties: {} as Record<string, JSONSchema4Object>,
+  };
+
+  Object.entries(modelProviders).forEach(([id, provider]) => {
+    combinedSchema.properties[id] = provider.inputSchema;
+  });
+
+  Object.entries(modelProviders).forEach(([id, provider]) => {
+    combinedSchema.properties[id] = provider.inputSchema;
+  });
+
+  return await compile(combinedSchema as JSONSchema4Object, "PromptTypes", {
+    additionalProperties: false,
+    bannerComment: dedent`
+  /**
+   * This type map defines the input types for each model provider. 
+   */ 
+  `,
+  });
+};

app/src/modelProviders/modelProviders.ts

@@ -1,12 +1,12 @@
 import openaiChatCompletion from "./openai-ChatCompletion";
 import replicateLlama2 from "./replicate-llama2";
-import anthropicCompletion from "./anthropic-completion";
+import anthropic from "./anthropic";
 import { type SupportedProvider, type ModelProvider } from "./types";
 
 const modelProviders: Record<SupportedProvider, ModelProvider<any, any, any>> = {
   "openai/ChatCompletion": openaiChatCompletion,
   "replicate/llama2": replicateLlama2,
-  "anthropic/completion": anthropicCompletion,
+  anthropic,
 };
 
 export default modelProviders;