(Probably) fixes the build

This probably fixes the build that I broke in https://github.com/OpenPipe/OpenPipe/pull/149. However, there's a small chance that it fixes it enough to deploy, but not enough to actually work. That would be bad, so not merging until I have time to monitor the deploy.

.dockerignore

@@ -0,0 +1,5 @@
+**/node_modules/
+.git
+**/.venv/
+**/.env*
+**/.next/

.gitignore

@@ -0,0 +1,5 @@
+.env
+.venv/
+*.pyc
+node_modules/
+*.tsbuildinfo

README.md

@@ -65,7 +65,14 @@ OpenPipe includes a tool to generate new test scenarios based on your existing p
 4. Clone this repository: `git clone https://github.com/openpipe/openpipe`
 5. Install the dependencies: `cd openpipe && pnpm install`
 6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`.
-7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma db push` to create the database.
+7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma migrate dev` to create the database.
 8. Create a [GitHub OAuth App](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app) and update the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` values. (Note: a PR to make auth optional when running locally would be a great contribution!)
 9. Start the app: `pnpm dev`.
 10. Navigate to [http://localhost:3000](http://localhost:3000)
+
+## Testing Locally
+
+1. Copy your `.env` file to `.env.test`.
+2. Update the `DATABASE_URL` to have a different database name than your development one
+3. Run `DATABASE_URL=[your new datatase url] pnpm prisma migrate dev --skip-seed --skip-generate`
+4. Run `pnpm test`

app/.env.example

@@ -31,3 +31,6 @@ NEXT_PUBLIC_HOST="http://localhost:3000"
 # Next Auth Github Provider
 GITHUB_CLIENT_ID="your_client_id"
 GITHUB_CLIENT_SECRET="your_secret"
+
+OPENPIPE_BASE_URL="http://localhost:3000/api"
+OPENPIPE_API_KEY="your_key"

app/.eslintrc.cjs

@@ -6,7 +6,7 @@ const config = {
   overrides: [
     {
       extends: ["plugin:@typescript-eslint/recommended-requiring-type-checking"],
-      files: ["*.ts", "*.tsx"],
+      files: ["*.mts", "*.ts", "*.tsx"],
       parserOptions: {
         project: path.join(__dirname, "tsconfig.json"),
       },

app/.gitignore

@@ -34,6 +34,7 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env
 .env*.local
+.env.test
 
 # vercel
 .vercel
@@ -43,3 +44,6 @@ yarn-error.log*
 
 # Sentry Auth Token
 .sentryclirc
+
+# custom openai intialization
+src/server/utils/openaiCustomConfig.json

app/@types/nextjs-routes.d.ts

@@ -12,15 +12,20 @@ declare module "nextjs-routes" {
 
   export type Route =
     | StaticRoute<"/account/signin">
+    | DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
     | StaticRoute<"/api/experiments/og-image">
+    | StaticRoute<"/api/openapi">
     | StaticRoute<"/api/sentry-example-api">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
+    | StaticRoute<"/dashboard">
     | DynamicRoute<"/data/[id]", { "id": string }>
     | StaticRoute<"/data">
     | DynamicRoute<"/experiments/[id]", { "id": string }>
     | StaticRoute<"/experiments">
     | StaticRoute<"/">
+    | StaticRoute<"/project/settings">
+    | StaticRoute<"/request-logs">
     | StaticRoute<"/sentry-example-page">
     | StaticRoute<"/world-champs">
     | StaticRoute<"/world-champs/signup">;

app/Dockerfile

@@ -6,13 +6,13 @@ RUN yarn global add pnpm
 # DEPS
 FROM base as deps
 
-WORKDIR /app
+WORKDIR /code
 
-COPY prisma ./
+COPY app/prisma app/package.json ./app/
+COPY client-libs/typescript/package.json ./client-libs/typescript/
+COPY pnpm-lock.yaml pnpm-workspace.yaml ./
 
-COPY package.json pnpm-lock.yaml ./
-
-RUN pnpm install --frozen-lockfile
+RUN cd app && pnpm install --frozen-lockfile
 
 # BUILDER
 FROM base as builder
@@ -23,23 +23,26 @@ ARG NEXT_PUBLIC_SOCKET_URL
 ARG NEXT_PUBLIC_HOST
 ARG NEXT_PUBLIC_SENTRY_DSN
 ARG SENTRY_AUTH_TOKEN
+ARG NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS
 
-WORKDIR /app
-COPY --from=deps /app/node_modules ./node_modules
+WORKDIR /code
+COPY --from=deps /code/node_modules ./node_modules
+COPY --from=deps /code/app/node_modules ./app/node_modules
+COPY --from=deps /code/client-libs/typescript/node_modules ./client-libs/typescript/node_modules
 COPY . .
-RUN SKIP_ENV_VALIDATION=1 pnpm build
+RUN cd app && SKIP_ENV_VALIDATION=1 pnpm build
 
 # RUNNER
 FROM base as runner
-WORKDIR /app
+WORKDIR /code/app
 
 ENV NODE_ENV production
 ENV NEXT_TELEMETRY_DISABLED 1
 
-COPY --from=builder /app/ ./
+COPY --from=builder /code/ /code/
 
 EXPOSE 3000
 ENV PORT 3000
 
 # Run the "run-prod.sh" script
-CMD /app/run-prod.sh
+CMD /code/app/run-prod.sh

app/next.config.mjs

@@ -36,6 +36,8 @@ let config = {
     });
     return config;
   },
+
+  transpilePackages: ["openpipe"],
 };
 
 config = nextRoutes()(config);

app/openapitools.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json",
+  "spaces": 2,
+  "generator-cli": {
+    "version": "6.6.0"
+  }
+}

app/package.json

@@ -1,5 +1,6 @@
 {
-  "name": "openpipe",
+  "name": "openpipe-app",
+  "private": true,
   "type": "module",
   "version": "0.1.0",
   "license": "Apache-2.0",
@@ -16,15 +17,14 @@
     "postinstall": "prisma generate",
     "lint": "next lint",
     "start": "next start",
-    "codegen": "tsx src/codegen/export-openai-types.ts",
+    "codegen:clients": "tsx src/server/scripts/client-codegen.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
-    "test": "pnpm vitest --no-threads"
+    "test": "pnpm vitest"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.5.8",
     "@apidevtools/json-schema-ref-parser": "^10.1.0",
-    "@babel/preset-typescript": "^7.22.5",
     "@babel/standalone": "^7.22.9",
     "@chakra-ui/anatomy": "^2.2.0",
     "@chakra-ui/next-js": "^2.1.4",
@@ -50,6 +50,7 @@
     "chroma-js": "^2.4.2",
     "concurrently": "^8.2.0",
     "cors": "^2.8.5",
+    "crypto-random-string": "^5.0.0",
     "dayjs": "^1.11.8",
     "dedent": "^1.0.1",
     "dotenv": "^16.3.1",
@@ -62,12 +63,16 @@
     "json-schema-to-typescript": "^13.0.2",
     "json-stringify-pretty-compact": "^4.0.0",
     "jsonschema": "^1.4.1",
+    "kysely": "^0.26.1",
     "lodash-es": "^4.17.21",
+    "lucide-react": "^0.265.0",
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
     "next-query-params": "^4.2.3",
+    "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
     "openai": "4.0.0-beta.7",
+    "pg": "^8.11.2",
     "pluralize": "^8.0.0",
     "posthog-js": "^1.75.3",
     "posthog-node": "^3.1.1",
@@ -83,17 +88,20 @@
     "react-syntax-highlighter": "^15.5.0",
     "react-textarea-autosize": "^8.5.0",
     "recast": "^0.23.3",
+    "recharts": "^2.7.2",
     "replicate": "^0.12.3",
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
     "superjson": "1.12.2",
+    "trpc-openapi": "^1.2.0",
     "tsx": "^3.12.7",
     "type-fest": "^4.0.0",
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.0",
     "vite-tsconfig-paths": "^4.2.0",
     "zod": "^3.21.4",
-    "zustand": "^4.3.9"
+    "zustand": "^4.3.9",
+    "openpipe": "workspace:*"
   },
   "devDependencies": {
     "@openapi-contrib/openapi-schema-to-json-schema": "^4.0.5",
@@ -106,6 +114,7 @@
     "@types/json-schema": "^7.0.12",
     "@types/lodash-es": "^4.17.8",
     "@types/node": "^18.16.0",
+    "@types/pg": "^8.10.2",
     "@types/pluralize": "^0.0.30",
     "@types/prismjs": "^1.26.0",
     "@types/react": "^18.2.6",

app/prisma/migrations/20230806024615_add_logged_calls_and_api_keys/migration.sql

@@ -0,0 +1,90 @@
+-- CreateTable
+CREATE TABLE "LoggedCall" (
+    "id" UUID NOT NULL,
+    "startTime" TIMESTAMP(3) NOT NULL,
+    "cacheHit" BOOLEAN NOT NULL,
+    "modelResponseId" UUID NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCall_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LoggedCallModelResponse" (
+    "id" UUID NOT NULL,
+    "reqPayload" JSONB NOT NULL,
+    "respStatus" INTEGER,
+    "respPayload" JSONB,
+    "error" TEXT,
+    "startTime" TIMESTAMP(3) NOT NULL,
+    "endTime" TIMESTAMP(3) NOT NULL,
+    "cacheKey" TEXT,
+    "durationMs" INTEGER,
+    "inputTokens" INTEGER,
+    "outputTokens" INTEGER,
+    "finishReason" TEXT,
+    "completionId" TEXT,
+    "totalCost" DECIMAL(18,12),
+    "originalLoggedCallId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCallModelResponse_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LoggedCallTag" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "value" TEXT,
+    "loggedCallId" UUID NOT NULL,
+
+    CONSTRAINT "LoggedCallTag_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "ApiKey" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "apiKey" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "ApiKey_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "LoggedCall_startTime_idx" ON "LoggedCall"("startTime");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LoggedCallModelResponse_originalLoggedCallId_key" ON "LoggedCallModelResponse"("originalLoggedCallId");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallModelResponse_cacheKey_idx" ON "LoggedCallModelResponse"("cacheKey");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_name_idx" ON "LoggedCallTag"("name");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_name_value_idx" ON "LoggedCallTag"("name", "value");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ApiKey_apiKey_key" ON "ApiKey"("apiKey");
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_modelResponseId_fkey" FOREIGN KEY ("modelResponseId") REFERENCES "LoggedCallModelResponse"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCallModelResponse" ADD CONSTRAINT "LoggedCallModelResponse_originalLoggedCallId_fkey" FOREIGN KEY ("originalLoggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCallTag" ADD CONSTRAINT "LoggedCallTag_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/migrations/20230807044936_add_name_to_organization/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Organization" ADD COLUMN     "name" TEXT NOT NULL DEFAULT 'Project 1';

app/prisma/migrations/20230808034313_make_model_response_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LoggedCall" ALTER COLUMN "modelResponseId" DROP NOT NULL;

app/prisma/migrations/20230809222649_rename_organization_to_project/migration.sql

@@ -0,0 +1,37 @@
+-- Rename Enum
+ALTER TYPE "OrganizationUserRole" RENAME TO "ProjectUserRole";
+
+-- Drop and recreate foreign keys
+ALTER TABLE "ApiKey" DROP CONSTRAINT "ApiKey_organizationId_fkey";
+ALTER TABLE "Dataset" DROP CONSTRAINT "Dataset_organizationId_fkey";
+ALTER TABLE "Experiment" DROP CONSTRAINT "Experiment_organizationId_fkey";
+ALTER TABLE "LoggedCall" DROP CONSTRAINT "LoggedCall_organizationId_fkey";
+ALTER TABLE "OrganizationUser" DROP CONSTRAINT "OrganizationUser_organizationId_fkey";
+ALTER TABLE "OrganizationUser" DROP CONSTRAINT "OrganizationUser_userId_fkey";
+
+-- Rename columns
+ALTER TABLE "ApiKey" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Dataset" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Experiment" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "LoggedCall" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "OrganizationUser" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Organization" RENAME COLUMN "personalOrgUserId" TO "personalProjectUserId";
+
+-- Rename table
+ALTER TABLE "Organization" RENAME TO "Project";
+ALTER TABLE "OrganizationUser" RENAME TO "ProjectUser";
+
+-- Recreate foreign keys
+ALTER TABLE "Experiment" ADD CONSTRAINT "Experiment_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "Dataset" ADD CONSTRAINT "Dataset_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ProjectUser" ADD CONSTRAINT "ProjectUser_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ProjectUser" ADD CONSTRAINT "ProjectUser_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- Rename indexes
+ALTER TABLE "Project" RENAME CONSTRAINT "Organization_pkey" TO "Project_pkey";
+ALTER TABLE "ProjectUser" RENAME CONSTRAINT "OrganizationUser_pkey" TO "ProjectUser_pkey";
+ALTER TABLE "Project" RENAME CONSTRAINT "Organization_personalOrgUserId_fkey" TO "Project_personalProjectUserId_fkey";
+ALTER INDEX "Organization_personalOrgUserId_key" RENAME TO "Project_personalProjectUserId_key";
+ALTER INDEX "OrganizationUser_organizationId_userId_key" RENAME TO "ProjectUser_projectId_userId_key";

app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

app/prisma/migrations/20230811200114_rename_model_response_fields/migration.sql

@@ -0,0 +1,66 @@
+/*
+  Warnings:
+
+  - You are about to rename the column `completionTokens` to `outputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `promptTokens` to `inputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `startTime` on the `LoggedCall` table to `requestedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `startTime` on the `LoggedCallModelResponse` table to `requestedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `endTime` on the `LoggedCallModelResponse` table to `receivedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `error` on the `LoggedCallModelResponse` table to `errorMessage`. Ensure compatibility with application logic.
+  - You are about to rename the column `respStatus` on the `LoggedCallModelResponse` table to `statusCode`. Ensure compatibility with application logic.
+  - You are about to rename the column `totalCost` on the `LoggedCallModelResponse` table to `cost`. Ensure compatibility with application logic.
+  - You are about to rename the column `inputHash` on the `ModelResponse` table to `cacheKey`. Ensure compatibility with application logic.
+  - You are about to rename the column `output` on the `ModelResponse` table to `respPayload`. Ensure compatibility with application logic.
+
+*/
+-- DropIndex
+DROP INDEX "LoggedCall_startTime_idx";
+
+-- DropIndex
+DROP INDEX "ModelResponse_inputHash_idx";
+
+-- Rename completionTokens to outputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "completionTokens" TO "outputTokens";
+
+-- Rename promptTokens to inputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "promptTokens" TO "inputTokens";
+
+-- AlterTable
+ALTER TABLE "LoggedCall"
+RENAME COLUMN "startTime" TO "requestedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "startTime" TO "requestedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "endTime" TO "receivedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "error" TO "errorMessage";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "respStatus" TO "statusCode";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "totalCost" TO "cost";
+
+-- AlterTable
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "inputHash" TO "cacheKey";
+
+-- AlterTable
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "output" TO "respPayload";
+
+-- CreateIndex
+CREATE INDEX "LoggedCall_requestedAt_idx" ON "LoggedCall"("requestedAt");
+
+-- CreateIndex
+CREATE INDEX "ModelResponse_cacheKey_idx" ON "ModelResponse"("cacheKey");

app/prisma/migrations/20230811231429_add_model_to_logged_call/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LoggedCall" ADD COLUMN     "model" TEXT;

app/prisma/schema.prisma

@@ -16,8 +16,8 @@ model Experiment {
 
     sortIndex Int @default(0)
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -112,13 +112,13 @@ model ScenarioVariantCell {
 model ModelResponse {
     id String @id @default(uuid()) @db.Uuid
 
-    inputHash        String
+    cacheKey        String
     requestedAt      DateTime?
     receivedAt       DateTime?
-    output           Json?
+    respPayload      Json?
     cost             Float?
-    promptTokens     Int?
-    completionTokens Int?
+    inputTokens      Int?
+    outputTokens     Int?
     statusCode       Int?
     errorMessage     String?
     retryTime        DateTime?
@@ -131,7 +131,7 @@ model ModelResponse {
     scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
     outputEvaluations     OutputEvaluation[]
 
-    @@index([inputHash])
+    @@index([cacheKey])
 }
 
 enum EvalType {
@@ -180,8 +180,8 @@ model Dataset {
     name           String
     datasetEntries DatasetEntry[]
 
-    organizationId String       @db.Uuid
-    organization   Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String  @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -200,33 +200,35 @@ model DatasetEntry {
     updatedAt DateTime @updatedAt
 }
 
-model Organization {
-    id                String  @id @default(uuid()) @db.Uuid
-    personalOrgUserId String? @unique @db.Uuid
-    PersonalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+model Project {
+    id   String @id @default(uuid()) @db.Uuid
+    name String @default("Project 1")
 
-    createdAt         DateTime           @default(now())
-    updatedAt         DateTime           @updatedAt
-    organizationUsers OrganizationUser[]
-    experiments       Experiment[]
-    datasets          Dataset[]
-    loggedCalls       LoggedCall[]
-    apiKeys           ApiKey[]
+    personalProjectUserId String? @unique @db.Uuid
+    personalProjectUser   User?   @relation(fields: [personalProjectUserId], references: [id], onDelete: Cascade)
+
+    createdAt    DateTime      @default(now())
+    updatedAt    DateTime      @updatedAt
+    projectUsers ProjectUser[]
+    experiments  Experiment[]
+    datasets     Dataset[]
+    loggedCalls  LoggedCall[]
+    apiKeys      ApiKey[]
 }
 
-enum OrganizationUserRole {
+enum ProjectUserRole {
     ADMIN
     MEMBER
     VIEWER
 }
 
-model OrganizationUser {
+model ProjectUser {
     id String @id @default(uuid()) @db.Uuid
 
-    role OrganizationUserRole
+    role ProjectUserRole
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     userId String @db.Uuid
     user   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
@@ -234,7 +236,7 @@ model OrganizationUser {
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    @@unique([organizationId, userId])
+    @@unique([projectId, userId])
 }
 
 model WorldChampEntrant {
@@ -254,29 +256,30 @@ model WorldChampEntrant {
 model LoggedCall {
     id String @id @default(uuid()) @db.Uuid
 
-    startTime DateTime
+    requestedAt DateTime
 
     // True if this call was served from the cache, false otherwise
     cacheHit Boolean
 
     // A LoggedCall is always associated with a LoggedCallModelResponse. If this
-    // is a cache miss, it's a new LoggedCallModelResponse we created for this.
-    // If it's a cache hit, it's the existing LoggedCallModelResponse we served.    
-    modelResponseId String                  @db.Uuid
-    modelResponse   LoggedCallModelResponse @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
+    // is a cache miss, we create a new LoggedCallModelResponse.
+    // If it's a cache hit, it's a pre-existing LoggedCallModelResponse.    
+    modelResponseId String?                  @db.Uuid
+    modelResponse   LoggedCallModelResponse? @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
 
-    // The response created by this LoggedCall. Will be null if this LoggedCall is a cache hit.
-    createdResponse LoggedCallModelResponse[] @relation(name: "ModelResponseCreatedBy")
+    // The responses created by this LoggedCall. Will be empty if this LoggedCall was a cache hit.
+    createdResponses LoggedCallModelResponse[] @relation(name: "ModelResponseOriginalCall")
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
-    tags LoggedCallTag[]
+    model        String?
+    tags         LoggedCallTag[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    @@index([startTime])
+    @@index([requestedAt])
 }
 
 model LoggedCallModelResponse {
@@ -285,14 +288,14 @@ model LoggedCallModelResponse {
     reqPayload Json
 
     // The HTTP status returned by the model provider
-    respStatus  Int?
+    statusCode  Int?
     respPayload Json?
 
     // Should be null if the request was successful, and some string if the request failed.
-    error String?
+    errorMessage String?
 
-    startTime DateTime
-    endTime   DateTime
+    requestedAt DateTime
+    receivedAt   DateTime
 
     // Note: the function to calculate the cacheKey should include the project
     // ID so we don't share cached responses between projects, which could be an
@@ -306,11 +309,11 @@ model LoggedCallModelResponse {
     outputTokens Int?
     finishReason String?
     completionId String?
-    totalCost    Decimal? @db.Decimal(18, 12)
+    cost         Decimal? @db.Decimal(18, 12)
 
     // The LoggedCall that created this LoggedCallModelResponse
-    createdById String     @unique @db.Uuid
-    createdBy   LoggedCall @relation(name: "ModelResponseCreatedBy", fields: [createdById], references: [id], onDelete: Cascade)
+    originalLoggedCallId String     @unique @db.Uuid
+    originalLoggedCall   LoggedCall @relation(name: "ModelResponseOriginalCall", fields: [originalLoggedCallId], references: [id], onDelete: Cascade)
 
     createdAt   DateTime     @default(now())
     updatedAt   DateTime     @updatedAt
@@ -320,11 +323,11 @@ model LoggedCallModelResponse {
 }
 
 model LoggedCallTag {
-    id    String  @id @default(cuid())
+    id    String  @id @default(uuid()) @db.Uuid
     name  String
     value String?
 
-    loggedCallId String
+    loggedCallId String     @db.Uuid
     loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
 
     @@index([name])
@@ -337,8 +340,8 @@ model ApiKey {
     name   String
     apiKey String @unique
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -387,8 +390,8 @@ model User {
 
     accounts          Account[]
     sessions          Session[]
-    organizationUsers OrganizationUser[]
-    organizations     Organization[]
+    projectUsers      ProjectUser[]
+    projects          Project[]
     worldChampEntrant WorldChampEntrant?
 
     createdAt DateTime @default(now())

app/prisma/seed.ts

@@ -5,14 +5,14 @@ import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111111";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
@@ -26,7 +26,7 @@ await prisma.experiment.create({
   data: {
     id: defaultId,
     label: "Country Capitals Example",
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 

app/prisma/seedAgiEval.ts

@@ -7,14 +7,14 @@ import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111112";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
@@ -47,7 +47,7 @@ for (const dataset of datasets) {
   const oldExperiment = await prisma.experiment.findFirst({
     where: {
       label: experimentName,
-      organizationId: org.id,
+      projectId: project.id,
     },
   });
   if (oldExperiment) {
@@ -60,7 +60,7 @@ for (const dataset of datasets) {
     data: {
       id: oldExperiment?.id ?? undefined,
       label: experimentName,
-      organizationId: org.id,
+      projectId: project.id,
     },
   });
 

app/prisma/seedTwitterSentiment.ts

@@ -6,14 +6,14 @@ import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111112";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
@@ -27,7 +27,7 @@ const experimentName = `Twitter Sentiment Analysis`;
 const oldExperiment = await prisma.experiment.findFirst({
   where: {
     label: experimentName,
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 if (oldExperiment) {
@@ -40,7 +40,7 @@ const experiment = await prisma.experiment.create({
   data: {
     id: oldExperiment?.id ?? undefined,
     label: experimentName,
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 

app/public/favicons/safari-pinned-tab.svg

@@ -9,10 +9,9 @@ Created by potrace 1.14, written by Peter Selinger 2001-2017
 </metadata>
 <g transform="translate(0.000000,550.000000) scale(0.100000,-0.100000)"
 fill="#000000" stroke="none">
-<path d="M813 5478 c-18 -13 -37 -36 -43 -52 -6 -19 -10 -236 -10 -603 0 -638
--1 -626 65 -657 25 -12 67 -16 179 -16 l146 0 0 -2032 0 -2032 23 -33 c12 -18
-35 -37 51 -43 19 -7 539 -10 1528 -10 1663 0 1549 -5 1582 65 14 30 16 235 16
-2059 l0 2026 156 0 156 0 39 39 39 39 0 587 c0 651 1 638 -65 669 -30 14 -223
-16 -1932 16 l-1898 0 -32 -22z"/>
+<path d="M785 5474 l-25 -27 0 -622 0 -622 25 -27 24 -26 171 0 170 0 0 -2050
+0 -2051 25 -25 24 -24 1557 2 1556 3 19 24 c19 23 19 70 19 2072 l0 2049 169
+0 c165 0 169 1 195 25 l26 24 0 626 0 626 -26 24 -27 25 -1939 0 -1939 0 -24
+-26z"/>
 </g>
 </svg>

app/public/logo.svg

@@ -1,5 +1,28 @@
-<svg width="380" height="320" viewBox="0 0 380 320" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M72 320L122.5 231L130.5 150.5L115 73L72 0H312L265 64.5L257 158.5L265 249L312 320H72Z" fill="#FF5733"/>
-<path d="M67.027 9.5C72.9909 9.5 79.5196 12.3449 86.3672 19.2588C93.2495 26.2075 99.8845 36.7468 105.66 50.5336C117.194 78.0671 124.554 116.764 124.554 160C124.554 203.236 117.194 241.933 105.66 269.466C99.8845 283.253 93.2495 293.793 86.3672 300.741C79.5196 307.655 72.9909 310.5 67.027 310.5C61.0632 310.5 54.5345 307.655 47.6868 300.741C40.8045 293.793 34.1695 283.253 28.394 269.466C16.8596 241.933 9.5 203.236 9.5 160C9.5 116.764 16.8596 78.0671 28.394 50.5336C34.1695 36.7468 40.8045 26.2075 47.6868 19.2588C54.5345 12.3449 61.0632 9.5 67.027 9.5Z" stroke="#FF5733" stroke-width="19"/>
-<path d="M312.027 9.5C317.991 9.5 324.52 12.3449 331.367 19.2588C338.25 26.2075 344.885 36.7468 350.66 50.5336C362.194 78.0671 369.554 116.764 369.554 160C369.554 203.236 362.194 241.933 350.66 269.466C344.885 283.253 338.25 293.793 331.367 300.741C324.52 307.655 317.991 310.5 312.027 310.5C306.063 310.5 299.534 307.655 292.687 300.741C285.805 293.793 279.17 283.253 273.394 269.466C261.86 241.933 254.5 203.236 254.5 160C254.5 116.764 261.86 78.0671 273.394 50.5336C279.17 36.7468 285.805 26.2075 292.687 19.2588C299.534 12.3449 306.063 9.5 312.027 9.5Z" stroke="#FF5733" stroke-width="19"/>
+<svg width="398" height="550" viewBox="0 0 398 550" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M39 125H359V542C359 546.418 355.418 550 351 550H47C42.5817 550 39 546.418 39 542V125Z" fill="black"/>
+<path d="M0 8C0 3.58172 3.58172 0 8 0H390C394.418 0 398 3.58172 398 8V127C398 131.418 394.418 135 390 135H7.99999C3.58171 135 0 131.418 0 127V8Z" fill="black"/>
+<path d="M50 135H348V535C348 537.209 346.209 539 344 539H54C51.7909 539 50 537.209 50 535V135Z" fill="#FF5733"/>
+<path d="M11 14.0001C11 11.791 12.7909 10.0001 15 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H15C12.7909 124 11 122.209 11 120V14.0001Z" fill="#FF5733"/>
+<path d="M11 14.0001C11 11.791 12.7909 10.0001 15 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H15C12.7909 124 11 122.209 11 120V14.0001Z" fill="url(#paint0_linear_102_49)"/>
+<path d="M50 134H348V535C348 537.209 346.209 539 344 539H54C51.7909 539 50 537.209 50 535V134Z" fill="url(#paint1_linear_102_49)"/>
+<path d="M108 142H156V535H108V142Z" fill="white"/>
+<path d="M300 135H348V535C348 537.209 346.209 539 344 539H300V135Z" fill="white" fill-opacity="0.25"/>
+<path d="M96 142H108V535H96V142Z" fill="white" fill-opacity="0.5"/>
+<path d="M84 10.0001H133V120H84V10.0001Z" fill="white"/>
+<path d="M339 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H339V10.0001Z" fill="white" fill-opacity="0.25"/>
+<path d="M71.9995 10.0001H83.9995V120H71.9995V10.0001Z" fill="white" fill-opacity="0.5"/>
+<path d="M108 534.529H156V539.019H108V534.529Z" fill="#AAAAAA"/>
+<path opacity="0.5" d="M95.9927 534.529H107.982V539.019H95.9927V534.529Z" fill="#AAAAAA"/>
+<path d="M84.0029 119.887H133.007V124.027H84.0029V119.887Z" fill="#AAAAAA"/>
+<path opacity="0.5" d="M71.9883 119.887H83.978V124.027H71.9883V119.887Z" fill="#AAAAAA"/>
+<defs>
+<linearGradient id="paint0_linear_102_49" x1="335" y1="67.0001" x2="137" y2="67.0001" gradientUnits="userSpaceOnUse">
+<stop stop-color="#D62600"/>
+<stop offset="1" stop-color="#FF5733" stop-opacity="0"/>
+</linearGradient>
+<linearGradient id="paint1_linear_102_49" x1="306.106" y1="336.5" x2="149.597" y2="336.5" gradientUnits="userSpaceOnUse">
+<stop stop-color="#D62600"/>
+<stop offset="1" stop-color="#FF5733" stop-opacity="0"/>
+</linearGradient>
+</defs>
 </svg>

app/src/components/CopiableCode.tsx

@@ -0,0 +1,40 @@
+import { HStack, Icon, IconButton, Tooltip, Text } from "@chakra-ui/react";
+import { useState } from "react";
+import { MdContentCopy } from "react-icons/md";
+import { useHandledAsyncCallback } from "~/utils/hooks";
+
+const CopiableCode = ({ code }: { code: string }) => {
+  const [copied, setCopied] = useState(false);
+
+  const [copyToClipboard] = useHandledAsyncCallback(async () => {
+    await navigator.clipboard.writeText(code);
+    setCopied(true);
+  }, [code]);
+  return (
+    <HStack
+      backgroundColor="blackAlpha.800"
+      color="white"
+      borderRadius={4}
+      padding={3}
+      w="full"
+      justifyContent="space-between"
+    >
+      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5} overflowX="auto">
+        {code}
+      </Text>
+      <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
+        <IconButton
+          aria-label="Copy"
+          icon={<Icon as={MdContentCopy} boxSize={5} />}
+          size="xs"
+          colorScheme="white"
+          variant="ghost"
+          onClick={copyToClipboard}
+          onMouseLeave={() => setCopied(false)}
+        />
+      </Tooltip>
+    </HStack>
+  );
+};
+
+export default CopiableCode;

app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -14,6 +14,7 @@ import {
 import { useRouter } from "next/router";
 import { useRef } from "react";
 import { BsTrash } from "react-icons/bs";
+import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 
@@ -23,6 +24,8 @@ export const DeleteButton = () => {
   const utils = api.useContext();
   const router = useRouter();
 
+  const closeDrawer = useAppStore((s) => s.closeDrawer);
+
   const { isOpen, onOpen, onClose } = useDisclosure();
   const cancelRef = useRef<HTMLButtonElement>(null);
 
@@ -31,6 +34,8 @@ export const DeleteButton = () => {
     await mutation.mutateAsync({ id: experiment.data.id });
     await utils.experiments.list.invalidate();
     await router.push({ pathname: "/experiments" });
+    closeDrawer();
+
     onClose();
   }, [mutation, experiment.data?.id, router]);
 

app/src/components/Favicon.tsx

@@ -8,8 +8,8 @@ export default function Favicon() {
       <link rel="icon" type="image/png" sizes="16x16" href="/favicons/favicon-16x16.png" />
       <link rel="manifest" href="/favicons/site.webmanifest" />
       <link rel="shortcut icon" href="/favicons/favicon.ico" />
+      <link rel="mask-icon" href="/favicons/safari-pinned-tab.svg" color="#5bbad5" />
       <meta name="msapplication-TileColor" content="#da532c" />
-      <meta name="msapplication-config" content="/favicons/browserconfig.xml" />
       <meta name="theme-color" content="#ffffff" />
     </Head>
   );

app/src/components/OutputsTable/AddVariantButton.tsx

@@ -33,25 +33,11 @@ export default function AddVariantButton() {
     <Flex w="100%" justifyContent="flex-end">
       <ActionButton
         onClick={onClick}
-        py={5}
+        py={7}
         leftIcon={<Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />}
       >
         <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
       </ActionButton>
-      {/* <Button
-        alignItems="center"
-        justifyContent="center"
-        fontWeight="normal"
-        bgColor="transparent"
-        _hover={{ bgColor: "gray.100" }}
-        px={cellPadding.x}
-        onClick={onClick}
-        height="unset"
-        minH={headerMinHeight}
-      >
-        <Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />
-        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
-      </Button> */}
     </Flex>
   );
 }

app/src/components/OutputsTable/EditEvaluations.tsx

@@ -12,6 +12,7 @@ import {
   Select,
   FormHelperText,
   Code,
+  IconButton,
 } from "@chakra-ui/react";
 import { type Evaluation, EvalType } from "@prisma/client";
 import { useCallback, useState } from "react";
@@ -183,46 +184,37 @@ export default function EditEvaluations() {
                 <Text flex={1}>
                   {evaluation.evalType}: &quot;{evaluation.value}&quot;
                 </Text>
-                <Button
+
+                <IconButton
+                  aria-label="Edit"
                   variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                   minW="unset"
+                  color="gray.400"
                   onClick={() => setEditingId(evaluation.id)}
-                  _hover={{
-                    color: "gray.800",
-                    cursor: "pointer",
-                  }}
-                >
-                  <Icon as={BsPencil} boxSize={4} />
-                </Button>
-                <Button
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
+                  icon={<Icon as={BsPencil} />}
+                />
+                <IconButton
+                  aria-label="Delete"
                   variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                   minW="unset"
+                  color="gray.400"
                   onClick={() => onDelete(evaluation.id)}
-                  _hover={{
-                    color: "gray.800",
-                    cursor: "pointer",
-                  }}
-                >
-                  <Icon as={BsX} boxSize={6} />
-                </Button>
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
+                  icon={<Icon as={BsX} boxSize={6} />}
+                />
               </HStack>
             ),
           )}
           {editingId == null && (
             <Button
               onClick={() => setEditingId("new")}
-              alignSelf="flex-start"
+              alignSelf="end"
               size="sm"
               mt={4}
               colorScheme="blue"
             >
-              Add Evaluation
+              New Evaluation
             </Button>
           )}
           {editingId == "new" && (

app/src/components/OutputsTable/EditScenarioVars.tsx

@@ -1,103 +1,185 @@
-import { Text, Button, HStack, Heading, Icon, Input, Stack } from "@chakra-ui/react";
-import { useState } from "react";
-import { BsCheck, BsX } from "react-icons/bs";
+import { Text, Button, HStack, Heading, Icon, IconButton, Stack, VStack } from "@chakra-ui/react";
+import { type TemplateVariable } from "@prisma/client";
+import { useEffect, useState } from "react";
+import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+
+export const ScenarioVar = ({
+  variable,
+  isEditing,
+  setIsEditing,
+}: {
+  variable: Pick<TemplateVariable, "id" | "label">;
+  isEditing: boolean;
+  setIsEditing: (isEditing: boolean) => void;
+}) => {
+  const utils = api.useContext();
+
+  const [label, setLabel] = useState(variable.label);
+
+  useEffect(() => {
+    setLabel(variable.label);
+  }, [variable.label]);
+
+  const renameVarMutation = api.scenarioVars.rename.useMutation();
+  const [onRename] = useHandledAsyncCallback(async () => {
+    const resp = await renameVarMutation.mutateAsync({ id: variable.id, label });
+    if (maybeReportError(resp)) return;
+
+    setIsEditing(false);
+    await utils.scenarioVars.list.invalidate();
+    await utils.scenarios.list.invalidate();
+  }, [label, variable.id]);
+
+  const deleteMutation = api.scenarioVars.delete.useMutation();
+  const [onDeleteVar] = useHandledAsyncCallback(async () => {
+    await deleteMutation.mutateAsync({ id: variable.id });
+    await utils.scenarioVars.list.invalidate();
+  }, [variable.id]);
+
+  if (isEditing) {
+    return (
+      <HStack w="full">
+        <FloatingLabelInput
+          flex={1}
+          label="Renamed Variable"
+          value={label}
+          onChange={(e) => setLabel(e.target.value)}
+          onKeyDown={(e) => {
+            if (e.key === "Enter") {
+              e.preventDefault();
+              onRename();
+            }
+            // If the user types a space, replace it with an underscore
+            if (e.key === " ") {
+              e.preventDefault();
+              setLabel((label) => label && `${label}_`);
+            }
+          }}
+        />
+        <Button size="sm" onClick={() => setIsEditing(false)}>
+          Cancel
+        </Button>
+        <Button size="sm" colorScheme="blue" onClick={onRename}>
+          Save
+        </Button>
+      </HStack>
+    );
+  } else {
+    return (
+      <HStack w="full" borderTopWidth={1} borderColor="gray.200">
+        <Text flex={1}>{variable.label}</Text>
+        <IconButton
+          aria-label="Edit"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={() => setIsEditing(true)}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsPencil} />}
+        />
+        <IconButton
+          aria-label="Delete"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={onDeleteVar}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsX} boxSize={6} />}
+        />
+      </HStack>
+    );
+  }
+};
 
 export default function EditScenarioVars() {
   const experiment = useExperiment();
-  const vars =
-    api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+  const vars = useScenarioVars();
+
+  const [currentlyEditingId, setCurrentlyEditingId] = useState<string | null>(null);
 
   const [newVariable, setNewVariable] = useState<string>("");
-  const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
+  const newVarIsValid = newVariable?.length ?? 0 > 0;
 
   const utils = api.useContext();
-  const addVarMutation = api.templateVars.create.useMutation();
+  const addVarMutation = api.scenarioVars.create.useMutation();
   const [onAddVar] = useHandledAsyncCallback(async () => {
     if (!experiment.data?.id) return;
-    if (!newVarIsValid) return;
-    await addVarMutation.mutateAsync({
+    if (!newVariable) return;
+    const resp = await addVarMutation.mutateAsync({
       experimentId: experiment.data.id,
       label: newVariable,
     });
-    await utils.templateVars.list.invalidate();
+    if (maybeReportError(resp)) return;
+
+    await utils.scenarioVars.list.invalidate();
     setNewVariable("");
   }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
 
-  const deleteMutation = api.templateVars.delete.useMutation();
-  const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
-    await deleteMutation.mutateAsync({ id });
-    await utils.templateVars.list.invalidate();
-  }, []);
-
   return (
     <Stack>
       <Heading size="sm">Scenario Variables</Heading>
-      <Stack spacing={2}>
+      <VStack spacing={4}>
         <Text fontSize="sm">
           Scenario variables can be used in your prompt variants as well as evaluations.
         </Text>
-        <HStack spacing={0}>
-          <Input
-            placeholder="Add Scenario Variable"
-            size="sm"
-            borderTopRadius={0}
-            borderRightRadius={0}
-            value={newVariable}
-            onChange={(e) => setNewVariable(e.target.value)}
-            onKeyDown={(e) => {
-              if (e.key === "Enter") {
-                e.preventDefault();
-                onAddVar();
-              }
-              // If the user types a space, replace it with an underscore
-              if (e.key === " ") {
-                e.preventDefault();
-                setNewVariable((v) => v + "_");
-              }
-            }}
-          />
-          <Button
-            size="xs"
-            height="100%"
-            borderLeftRadius={0}
-            isDisabled={!newVarIsValid}
-            onClick={onAddVar}
-          >
-            <Icon as={BsCheck} boxSize={8} />
-          </Button>
-        </HStack>
-
-        <HStack spacing={2} py={4} wrap="wrap">
-          {vars.map((variable) => (
-            <HStack
+        <VStack spacing={0} w="full">
+          {vars.data?.map((variable) => (
+            <ScenarioVar
+              variable={variable}
               key={variable.id}
-              spacing={0}
-              bgColor="blue.100"
-              color="blue.600"
-              pl={2}
-              pr={0}
-              fontWeight="bold"
-            >
-              <Text fontSize="sm" flex={1}>
-                {variable.label}
-              </Text>
-              <Button
-                size="xs"
-                variant="ghost"
-                colorScheme="blue"
-                p="unset"
-                minW="unset"
-                px="unset"
-                onClick={() => onDeleteVar(variable.id)}
-              >
-                <Icon as={BsX} boxSize={6} color="blue.800" />
-              </Button>
-            </HStack>
+              isEditing={currentlyEditingId === variable.id}
+              setIsEditing={(isEditing) => {
+                if (isEditing) {
+                  setCurrentlyEditingId(variable.id);
+                } else {
+                  setCurrentlyEditingId(null);
+                }
+              }}
+            />
           ))}
-        </HStack>
-      </Stack>
+        </VStack>
+        {currentlyEditingId !== "new" && (
+          <Button
+            colorScheme="blue"
+            size="sm"
+            onClick={() => setCurrentlyEditingId("new")}
+            alignSelf="end"
+          >
+            New Variable
+          </Button>
+        )}
+        {currentlyEditingId === "new" && (
+          <HStack w="full">
+            <FloatingLabelInput
+              flex={1}
+              label="New Variable"
+              value={newVariable}
+              onChange={(e) => setNewVariable(e.target.value)}
+              onKeyDown={(e) => {
+                if (e.key === "Enter") {
+                  e.preventDefault();
+                  onAddVar();
+                }
+                // If the user types a space, replace it with an underscore
+                if (e.key === " ") {
+                  e.preventDefault();
+                  setNewVariable((v) => v && `${v}_`);
+                }
+              }}
+            />
+            <Button size="sm" onClick={() => setCurrentlyEditingId(null)}>
+              Cancel
+            </Button>
+            <Button size="sm" colorScheme="blue" onClick={onAddVar}>
+              Save
+            </Button>
+          </HStack>
+        )}
+      </VStack>
     </Stack>
   );
 }

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,7 +1,7 @@
 import { api } from "~/utils/api";
 import { type PromptVariant, type Scenario } from "../types";
 import { type StackProps, Text, VStack } from "@chakra-ui/react";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
 import stringify from "json-stringify-pretty-compact";
@@ -23,10 +23,7 @@ export default function OutputCell({
   variant: PromptVariant;
 }): ReactElement | null {
   const utils = api.useContext();
-  const experiment = useExperiment();
-  const vars = api.templateVars.list.useQuery({
-    experimentId: experiment.data?.id ?? "",
-  }).data;
+  const vars = useScenarioVars().data;
 
   const scenarioVariables = scenario.variableValues as Record<string, string>;
   const templateHasVariables =
@@ -110,7 +107,7 @@ export default function OutputCell({
 
   if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
 
-  const showLogs = !streamedMessage && !mostRecentResponse?.output;
+  const showLogs = !streamedMessage && !mostRecentResponse?.respPayload;
 
   if (showLogs)
     return (
@@ -163,13 +160,13 @@ export default function OutputCell({
       </CellWrapper>
     );
 
-  const normalizedOutput = mostRecentResponse?.output
-    ? provider.normalizeOutput(mostRecentResponse?.output)
+  const normalizedOutput = mostRecentResponse?.respPayload
+    ? provider.normalizeOutput(mostRecentResponse?.respPayload)
     : streamedMessage
     ? provider.normalizeOutput(streamedMessage)
     : null;
 
-  if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
+  if (mostRecentResponse?.respPayload && normalizedOutput?.type === "json") {
     return (
       <CellWrapper>
         <SyntaxHighlighter
@@ -191,7 +188,7 @@ export default function OutputCell({
 
   return (
     <CellWrapper>
-      <Text>{contentToDisplay}</Text>
+      <Text whiteSpace="pre-wrap">{contentToDisplay}</Text>
     </CellWrapper>
   );
 }

app/src/components/OutputsTable/OutputCell/OutputStats.tsx

@@ -19,8 +19,8 @@ export const OutputStats = ({
       ? modelResponse.receivedAt.getTime() - modelResponse.requestedAt.getTime()
       : 0;
 
-  const promptTokens = modelResponse.promptTokens;
-  const completionTokens = modelResponse.completionTokens;
+  const inputTokens = modelResponse.inputTokens;
+  const outputTokens = modelResponse.outputTokens;
 
   return (
     <HStack
@@ -55,8 +55,8 @@ export const OutputStats = ({
       </HStack>
       {modelResponse.cost && (
         <CostTooltip
-          promptTokens={promptTokens}
-          completionTokens={completionTokens}
+          inputTokens={inputTokens}
+          outputTokens={outputTokens}
           cost={modelResponse.cost}
         >
           <HStack spacing={0}>

app/src/components/OutputsTable/OutputCell/TopActions.tsx

@@ -18,13 +18,13 @@ export const CellOptions = ({
   const modalDisclosure = useDisclosure();
 
   return (
-    <HStack justifyContent="flex-end" w="full">
+    <HStack justifyContent="flex-end" w="full" spacing={1}>
       {cell && (
         <>
           <Tooltip label="See Prompt">
             <IconButton
               aria-label="See Prompt"
-              icon={<Icon as={BsInfoCircle} boxSize={4} />}
+              icon={<Icon as={BsInfoCircle} boxSize={3.5} />}
               onClick={modalDisclosure.onOpen}
               size="xs"
               colorScheme="gray"

app/src/components/OutputsTable/ScenarioEditor.tsx

@@ -1,7 +1,7 @@
 import { isEqual } from "lodash-es";
 import { useEffect, useState, type DragEvent } from "react";
 import { api } from "~/utils/api";
-import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperimentAccess, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
 import { type Scenario } from "./types";
 
 import {
@@ -41,8 +41,7 @@ export default function ScenarioEditor({
     if (savedValues) setValues(savedValues);
   }, [savedValues]);
 
-  const experiment = useExperiment();
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = useScenarioVars();
 
   const variableLabels = vars.data?.map((v) => v.label) ?? [];
 

app/src/components/OutputsTable/ScenarioEditorModal.tsx

@@ -58,7 +58,7 @@ export const ScenarioEditorModal = ({
     await utils.scenarios.list.invalidate();
   }, [mutation, values]);
 
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = api.scenarioVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
   const variableLabels = vars.data?.map((v) => v.label) ?? [];
 
   return (

app/src/components/OutputsTable/ScenarioPaginator.tsx

@@ -1,21 +1,16 @@
+import { type StackProps } from "@chakra-ui/react";
+
 import { useScenarios } from "~/utils/hooks";
 import Paginator from "../Paginator";
 
-const ScenarioPaginator = () => {
+const ScenarioPaginator = (props: StackProps) => {
   const { data } = useScenarios();
 
   if (!data) return null;
 
-  const { scenarios, startIndex, lastPage, count } = data;
+  const { count } = data;
 
-  return (
-    <Paginator
-      numItemsLoaded={scenarios.length}
-      startIndex={startIndex}
-      lastPage={lastPage}
-      count={count}
-    />
-  );
+  return <Paginator count={count} condense {...props} />;
 };
 
 export default ScenarioPaginator;

app/src/components/OutputsTable/ScenarioRow.tsx

@@ -10,6 +10,7 @@ const ScenarioRow = (props: {
   variants: PromptVariant[];
   canHide: boolean;
   rowStart: number;
+  isLast: boolean;
 }) => {
   const [isHovered, setIsHovered] = useState(false);
 
@@ -21,10 +22,12 @@ const ScenarioRow = (props: {
         onMouseEnter={() => setIsHovered(true)}
         onMouseLeave={() => setIsHovered(false)}
         sx={isHovered ? highlightStyle : undefined}
+        bgColor="white"
         borderLeftWidth={1}
         {...borders}
         rowStart={props.rowStart}
         colStart={1}
+        borderBottomLeftRadius={props.isLast ? 8 : 0}
       >
         <ScenarioEditor scenario={props.scenario} hovered={isHovered} canHide={props.canHide} />
       </GridItem>
@@ -34,8 +37,10 @@ const ScenarioRow = (props: {
           onMouseEnter={() => setIsHovered(true)}
           onMouseLeave={() => setIsHovered(false)}
           sx={isHovered ? highlightStyle : undefined}
+          bgColor="white"
           rowStart={props.rowStart}
           colStart={i + 2}
+          borderBottomRightRadius={props.isLast && i === props.variants.length - 1 ? 8 : 0}
           {...borders}
         >
           <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -48,7 +48,20 @@ export const ScenariosHeader = () => {
   );
 
   return (
-    <HStack w="100%" pb={cellPadding.y} pt={0} align="center" spacing={0}>
+    <HStack
+      w="100%"
+      py={cellPadding.y}
+      px={cellPadding.x}
+      align="center"
+      spacing={0}
+      borderTopRightRadius={8}
+      borderTopLeftRadius={8}
+      bgColor="white"
+      borderWidth={1}
+      borderBottomWidth={0}
+      borderColor="gray.300"
+      mt={8}
+    >
       <Text fontSize={16} fontWeight="bold">
         Scenarios ({scenarios.data?.count})
       </Text>
@@ -57,11 +70,16 @@ export const ScenariosHeader = () => {
           <MenuButton
             as={IconButton}
             mt={1}
+            ml={2}
             variant="ghost"
             aria-label="Edit Scenarios"
             icon={<Icon as={loading ? Spinner : BsGear} />}
+            maxW={8}
+            minW={8}
+            minH={8}
+            maxH={8}
           />
-          <MenuList fontSize="md" zIndex="dropdown" mt={-3}>
+          <MenuList fontSize="md" zIndex="dropdown" mt={-1}>
             <MenuItem
               icon={<Icon as={BsPlus} boxSize={6} mx="-5px" />}
               onClick={() => onAddScenario(false)}
@@ -72,7 +90,7 @@ export const ScenariosHeader = () => {
               Autogenerate Scenario
             </MenuItem>
             <MenuItem icon={<BsPencil />} onClick={openDrawer}>
-              Edit Vars
+              Add or Remove Variables
             </MenuItem>
           </MenuList>
         </Menu>

app/src/components/OutputsTable/VariantStats.tsx

@@ -17,8 +17,8 @@ export default function VariantStats(props: { variant: PromptVariant }) {
       initialData: {
         evalResults: [],
         overallCost: 0,
-        promptTokens: 0,
-        completionTokens: 0,
+        inputTokens: 0,
+        outputTokens: 0,
         scenarioCount: 0,
         outputCount: 0,
         awaitingEvals: false,
@@ -68,8 +68,8 @@ export default function VariantStats(props: { variant: PromptVariant }) {
       </HStack>
       {data.overallCost && (
         <CostTooltip
-          promptTokens={data.promptTokens}
-          completionTokens={data.completionTokens}
+          inputTokens={data.inputTokens}
+          outputTokens={data.outputTokens}
           cost={data.overallCost}
         >
           <HStack spacing={0} align="center" color="gray.500">

app/src/components/OutputsTable/index.tsx

@@ -53,20 +53,29 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
           colStart: i + 2,
           borderLeftWidth: i === 0 ? 1 : 0,
           marginLeft: i === 0 ? "-1px" : 0,
-          backgroundColor: "gray.100",
+          backgroundColor: "white",
         };
+        const isFirst = i === 0;
+        const isLast = i === variants.data.length - 1;
         return (
           <Fragment key={variant.uiId}>
             <VariantHeader
               variant={variant}
               canHide={variants.data.length > 1}
               rowStart={1}
+              borderTopLeftRadius={isFirst ? 8 : 0}
+              borderTopRightRadius={isLast ? 8 : 0}
               {...sharedProps}
             />
             <GridItem rowStart={2} {...sharedProps}>
               <VariantEditor variant={variant} />
             </GridItem>
-            <GridItem rowStart={3} {...sharedProps}>
+            <GridItem
+              rowStart={3}
+              {...sharedProps}
+              borderBottomLeftRadius={isFirst ? 8 : 0}
+              borderBottomRightRadius={isLast ? 8 : 0}
+            >
               <VariantStats variant={variant} />
             </GridItem>
           </Fragment>
@@ -90,6 +99,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
           scenario={scenario}
           variants={variants.data}
           canHide={visibleScenariosCount > 1}
+          isLast={i === visibleScenariosCount - 1}
         />
       ))}
       <GridItem

app/src/components/Paginator.tsx

@@ -1,77 +1,117 @@
-import { Box, HStack, IconButton } from "@chakra-ui/react";
-import {
-  BsChevronDoubleLeft,
-  BsChevronDoubleRight,
-  BsChevronLeft,
-  BsChevronRight,
-} from "react-icons/bs";
-import { usePage } from "~/utils/hooks";
+import { HStack, IconButton, Text, Select, type StackProps, Icon } from "@chakra-ui/react";
+import React, { useCallback } from "react";
+import { FiChevronsLeft, FiChevronsRight, FiChevronLeft, FiChevronRight } from "react-icons/fi";
+import { usePageParams } from "~/utils/hooks";
+
+const pageSizeOptions = [10, 25, 50, 100];
 
 const Paginator = ({
-  numItemsLoaded,
-  startIndex,
-  lastPage,
   count,
-}: {
-  numItemsLoaded: number;
-  startIndex: number;
-  lastPage: number;
-  count: number;
-}) => {
-  const [page, setPage] = usePage();
+  condense,
+  ...props
+}: { count: number; condense?: boolean } & StackProps) => {
+  const { page, pageSize, setPageParams } = usePageParams();
+
+  const lastPage = Math.ceil(count / pageSize);
+
+  const updatePageSize = useCallback(
+    (newPageSize: number) => {
+      const newPage = Math.floor(((page - 1) * pageSize) / newPageSize) + 1;
+      setPageParams({ page: newPage, pageSize: newPageSize }, "replace");
+    },
+    [page, pageSize, setPageParams],
+  );
 
   const nextPage = () => {
     if (page < lastPage) {
-      setPage(page + 1, "replace");
+      setPageParams({ page: page + 1 }, "replace");
     }
   };
 
   const prevPage = () => {
     if (page > 1) {
-      setPage(page - 1, "replace");
+      setPageParams({ page: page - 1 }, "replace");
     }
   };
 
-  const goToLastPage = () => setPage(lastPage, "replace");
-  const goToFirstPage = () => setPage(1, "replace");
+  const goToLastPage = () => setPageParams({ page: lastPage }, "replace");
+  const goToFirstPage = () => setPageParams({ page: 1 }, "replace");
 
   return (
-    <HStack pt={4}>
-      <IconButton
-        variant="ghost"
-        size="sm"
-        onClick={goToFirstPage}
-        isDisabled={page === 1}
-        aria-label="Go to first page"
-        icon={<BsChevronDoubleLeft />}
-      />
-      <IconButton
-        variant="ghost"
-        size="sm"
-        onClick={prevPage}
-        isDisabled={page === 1}
-        aria-label="Previous page"
-        icon={<BsChevronLeft />}
-      />
-      <Box>
-        {startIndex}-{startIndex + numItemsLoaded - 1} / {count}
-      </Box>
-      <IconButton
-        variant="ghost"
-        size="sm"
-        onClick={nextPage}
-        isDisabled={page === lastPage}
-        aria-label="Next page"
-        icon={<BsChevronRight />}
-      />
-      <IconButton
-        variant="ghost"
-        size="sm"
-        onClick={goToLastPage}
-        isDisabled={page === lastPage}
-        aria-label="Go to last page"
-        icon={<BsChevronDoubleRight />}
-      />
+    <HStack
+      pt={4}
+      spacing={8}
+      justifyContent={condense ? "flex-start" : "space-between"}
+      alignItems="center"
+      w="full"
+      {...props}
+    >
+      {!condense && (
+        <>
+          <HStack>
+            <Text>Rows</Text>
+            <Select
+              value={pageSize}
+              onChange={(e) => updatePageSize(parseInt(e.target.value))}
+              w={20}
+              backgroundColor="white"
+            >
+              {pageSizeOptions.map((option) => (
+                <option key={option} value={option}>
+                  {option}
+                </option>
+              ))}
+            </Select>
+          </HStack>
+          <Text>
+            Page {page} of {lastPage}
+          </Text>
+        </>
+      )}
+
+      <HStack>
+        <IconButton
+          variant="outline"
+          size="sm"
+          onClick={goToFirstPage}
+          isDisabled={page === 1}
+          aria-label="Go to first page"
+          icon={<Icon as={FiChevronsLeft} boxSize={5} strokeWidth={1.5} />}
+          bgColor="white"
+        />
+        <IconButton
+          variant="outline"
+          size="sm"
+          onClick={prevPage}
+          isDisabled={page === 1}
+          aria-label="Previous page"
+          icon={<Icon as={FiChevronLeft} boxSize={5} strokeWidth={1.5} />}
+          bgColor="white"
+        />
+        {condense && (
+          <Text>
+            Page {page} of {lastPage}
+          </Text>
+        )}
+        <IconButton
+          variant="outline"
+          size="sm"
+          onClick={nextPage}
+          isDisabled={page === lastPage}
+          aria-label="Next page"
+          icon={<Icon as={FiChevronRight} boxSize={5} strokeWidth={1.5} />}
+          bgColor="white"
+        />
+        <IconButton
+          variant="outline"
+          size="sm"
+          onClick={goToLastPage}
+          isDisabled={page === lastPage}
+          aria-label="Go to last page"
+          icon={<Icon as={FiChevronsRight} boxSize={5} strokeWidth={1.5} />}
+          bgColor="white"
+        />
+      </HStack>
     </HStack>
   );
 };

app/src/components/StatsCard.tsx

@@ -0,0 +1,26 @@
+import { VStack, HStack, type StackProps, Text, Divider } from "@chakra-ui/react";
+import Link, { type LinkProps } from "next/link";
+
+const StatsCard = ({
+  title,
+  href,
+  children,
+  ...rest
+}: { title: string; href: string } & StackProps & LinkProps) => {
+  return (
+    <VStack flex={1} borderWidth={1} padding={4} borderRadius={4} borderColor="gray.300" {...rest}>
+      <HStack w="full" justifyContent="space-between">
+        <Text fontSize="md" fontWeight="bold">
+          {title}
+        </Text>
+        <Link href={href}>
+          <Text color="blue">View all</Text>
+        </Link>
+      </HStack>
+      <Divider />
+      {children}
+    </VStack>
+  );
+};
+
+export default StatsCard;

app/src/components/VariantHeader/VariantHeader.tsx

@@ -75,7 +75,7 @@ export default function VariantHeader(
       padding={0}
       sx={{
         position: "sticky",
-        top: "0",
+        top: "-2",
         // Ensure that the menu always appears above the sticky header of other variants
         zIndex: menuOpen ? "dropdown" : 10,
       }}
@@ -84,6 +84,7 @@ export default function VariantHeader(
     >
       <HStack
         spacing={2}
+        py={2}
         alignItems="flex-start"
         minH={headerMinHeight}
         draggable={!isInputHovered}
@@ -102,7 +103,9 @@ export default function VariantHeader(
           setIsDragTarget(false);
         }}
         onDrop={onReorder}
-        backgroundColor={isDragTarget ? "gray.200" : "gray.100"}
+        backgroundColor={isDragTarget ? "gray.200" : "white"}
+        borderTopLeftRadius={gridItemProps.borderTopLeftRadius}
+        borderTopRightRadius={gridItemProps.borderTopRightRadius}
         h="full"
       >
         <Icon

app/src/components/dashboard/LoggedCallsTable.tsx

@@ -0,0 +1,46 @@
+import { Card, CardHeader, Heading, Table, Tbody, HStack, Button, Text } from "@chakra-ui/react";
+import { useState } from "react";
+import Link from "next/link";
+import { useLoggedCalls } from "~/utils/hooks";
+import { TableHeader, TableRow } from "../requestLogs/TableRow";
+
+export default function LoggedCallsTable() {
+  const [expandedRow, setExpandedRow] = useState<string | null>(null);
+  const { data: loggedCalls } = useLoggedCalls();
+
+  return (
+    <Card width="100%" overflow="hidden">
+      <CardHeader>
+        <HStack justifyContent="space-between">
+          <Heading as="h3" size="sm">
+            Request Logs
+          </Heading>
+          <Button as={Link} href="/request-logs" variant="ghost" colorScheme="blue">
+            <Text>View All</Text>
+          </Button>
+        </HStack>
+      </CardHeader>
+      <Table>
+        <TableHeader />
+        <Tbody>
+          {loggedCalls?.calls.map((loggedCall) => {
+            return (
+              <TableRow
+                key={loggedCall.id}
+                loggedCall={loggedCall}
+                isExpanded={loggedCall.id === expandedRow}
+                onToggle={() => {
+                  if (loggedCall.id === expandedRow) {
+                    setExpandedRow(null);
+                  } else {
+                    setExpandedRow(loggedCall.id);
+                  }
+                }}
+              />
+            );
+          })}
+        </Tbody>
+      </Table>
+    </Card>
+  );
+}

app/src/components/dashboard/UsageGraph.tsx

@@ -0,0 +1,61 @@
+import {
+  ResponsiveContainer,
+  LineChart,
+  Line,
+  XAxis,
+  YAxis,
+  CartesianGrid,
+  Tooltip,
+  Legend,
+} from "recharts";
+import { useMemo } from "react";
+
+import { useSelectedProject } from "~/utils/hooks";
+import dayjs from "~/utils/dayjs";
+import { api } from "~/utils/api";
+
+export default function UsageGraph() {
+  const { data: selectedProject } = useSelectedProject();
+
+  const stats = api.dashboard.stats.useQuery(
+    { projectId: selectedProject?.id ?? "" },
+    { enabled: !!selectedProject },
+  );
+
+  const data = useMemo(() => {
+    return (
+      stats.data?.periods.map(({ period, numQueries, cost }) => ({
+        period,
+        Requests: numQueries,
+        "Total Spent (USD)": parseFloat(cost.toString()),
+      })) || []
+    );
+  }, [stats.data]);
+
+  return (
+    <ResponsiveContainer width="100%" height={400}>
+      <LineChart data={data} margin={{ top: 5, right: 20, left: 10, bottom: 5 }}>
+        <XAxis dataKey="period" tickFormatter={(str: string) => dayjs(str).format("MMM D")} />
+        <YAxis yAxisId="left" dataKey="Requests" orientation="left" stroke="#8884d8" />
+        <YAxis
+          yAxisId="right"
+          dataKey="Total Spent (USD)"
+          orientation="right"
+          unit="$"
+          stroke="#82ca9d"
+        />
+        <Tooltip />
+        <Legend />
+        <CartesianGrid stroke="#f5f5f5" />
+        <Line dataKey="Requests" stroke="#8884d8" yAxisId="left" dot={false} strokeWidth={2} />
+        <Line
+          dataKey="Total Spent (USD)"
+          stroke="#82ca9d"
+          yAxisId="right"
+          dot={false}
+          strokeWidth={2}
+        />
+      </LineChart>
+    </ResponsiveContainer>
+  );
+}

app/src/components/datasets/DatasetCard.tsx

@@ -15,6 +15,7 @@ import { useRouter } from "next/router";
 import { BsPlusSquare } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
 
 type DatasetData = {
   name: string;
@@ -71,11 +72,12 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
 
 export const NewDatasetCard = () => {
   const router = useRouter();
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
   const createMutation = api.datasets.create.useMutation();
   const [createDataset, isLoading] = useHandledAsyncCallback(async () => {
-    const newDataset = await createMutation.mutateAsync({ label: "New Dataset" });
+    const newDataset = await createMutation.mutateAsync({ projectId: selectedProjectId ?? "" });
     await router.push({ pathname: "/data/[id]", query: { id: newDataset.id } });
-  }, [createMutation, router]);
+  }, [createMutation, router, selectedProjectId]);
 
   return (
     <AspectRatio ratio={1.2} w="full">

app/src/components/datasets/DatasetEntriesPaginator.tsx

@@ -1,21 +1,16 @@
+import { type StackProps } from "@chakra-ui/react";
+
 import { useDatasetEntries } from "~/utils/hooks";
 import Paginator from "../Paginator";
 
-const DatasetEntriesPaginator = () => {
+const DatasetEntriesPaginator = (props: StackProps) => {
   const { data } = useDatasetEntries();
 
   if (!data) return null;
 
-  const { entries, startIndex, lastPage, count } = data;
+  const { count } = data;
 
-  return (
-    <Paginator
-      numItemsLoaded={entries.length}
-      startIndex={startIndex}
-      lastPage={lastPage}
-      count={count}
-    />
-  );
+  return <Paginator count={count} {...props} />;
 };
 
 export default DatasetEntriesPaginator;

app/src/components/experiments/ExperimentCard.tsx

@@ -7,6 +7,7 @@ import {
   Spinner,
   AspectRatio,
   SkeletonText,
+  Card,
 } from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
 import { formatTimePast } from "~/utils/dayjs";
@@ -15,6 +16,7 @@ import { useRouter } from "next/router";
 import { BsPlusSquare } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
 
 type ExperimentData = {
   testScenarioCount: number;
@@ -28,17 +30,22 @@ type ExperimentData = {
 
 export const ExperimentCard = ({ exp }: { exp: ExperimentData }) => {
   return (
-    <AspectRatio ratio={1.2} w="full">
+    <Card
+      w="full"
+      h="full"
+      cursor="pointer"
+      p={4}
+      bg="white"
+      borderRadius={4}
+      _hover={{ bg: "gray.100" }}
+      transition="background 0.2s"
+      aspectRatio={1.2}
+    >
       <VStack
         as={Link}
+        w="full"
+        h="full"
         href={{ pathname: "/experiments/[id]", query: { id: exp.id } }}
-        bg="gray.50"
-        _hover={{ bg: "gray.100" }}
-        transition="background 0.2s"
-        cursor="pointer"
-        borderColor="gray.200"
-        borderWidth={1}
-        p={4}
         justify="space-between"
       >
         <HStack w="full" color="gray.700" justify="center">
@@ -56,7 +63,7 @@ export const ExperimentCard = ({ exp }: { exp: ExperimentData }) => {
           <Text flex={1}>Updated {formatTimePast(exp.updatedAt)}</Text>
         </HStack>
       </VStack>
-    </AspectRatio>
+    </Card>
   );
 };
 
@@ -75,37 +82,43 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
 
 export const NewExperimentCard = () => {
   const router = useRouter();
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
   const createMutation = api.experiments.create.useMutation();
   const [createExperiment, isLoading] = useHandledAsyncCallback(async () => {
-    const newExperiment = await createMutation.mutateAsync({ label: "New Experiment" });
-    await router.push({ pathname: "/experiments/[id]", query: { id: newExperiment.id } });
-  }, [createMutation, router]);
+    const newExperiment = await createMutation.mutateAsync({
+      projectId: selectedProjectId ?? "",
+    });
+    await router.push({
+      pathname: "/experiments/[id]",
+      query: { id: newExperiment.id },
+    });
+  }, [createMutation, router, selectedProjectId]);
 
   return (
-    <AspectRatio ratio={1.2} w="full">
-      <VStack
-        align="center"
-        justify="center"
-        _hover={{ cursor: "pointer", bg: "gray.50" }}
-        transition="background 0.2s"
-        cursor="pointer"
-        borderColor="gray.200"
-        borderWidth={1}
-        p={4}
-        onClick={createExperiment}
-      >
+    <Card
+      w="full"
+      h="full"
+      cursor="pointer"
+      p={4}
+      bg="white"
+      borderRadius={4}
+      _hover={{ bg: "gray.100" }}
+      transition="background 0.2s"
+      aspectRatio={1.2}
+    >
+      <VStack align="center" justify="center" w="full" h="full" p={4} onClick={createExperiment}>
         <Icon as={isLoading ? Spinner : BsPlusSquare} boxSize={8} />
         <Text display={{ base: "none", md: "block" }} ml={2}>
           New Experiment
         </Text>
       </VStack>
-    </AspectRatio>
+    </Card>
   );
 };
 
 export const ExperimentCardSkeleton = () => (
   <AspectRatio ratio={1.2} w="full">
-    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="gray.50">
+    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="white">
       <SkeletonText noOfLines={1} w="80%" />
       <SkeletonText noOfLines={2} w="60%" />
       <SkeletonText noOfLines={1} w="80%" />

app/src/components/experiments/ExperimentHeaderButtons/useOnForkButtonPressed.tsx

@@ -3,18 +3,23 @@ import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import { signIn, useSession } from "next-auth/react";
 import { useRouter } from "next/router";
+import { useAppStore } from "~/state/store";
 
 export const useOnForkButtonPressed = () => {
   const router = useRouter();
 
   const user = useSession().data;
   const experiment = useExperiment();
+  const selectedProjectId = useAppStore((state) => state.selectedProjectId);
 
   const forkMutation = api.experiments.fork.useMutation();
 
   const [onFork, isForking] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id) return;
-    const forkedExperimentId = await forkMutation.mutateAsync({ id: experiment.data.id });
+    if (!experiment.data?.id || !selectedProjectId) return;
+    const forkedExperimentId = await forkMutation.mutateAsync({
+      id: experiment.data.id,
+      projectId: selectedProjectId,
+    });
     await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
   }, [forkMutation, experiment.data?.id, router]);
 

app/src/components/nav/AppShell.tsx

@@ -1,4 +1,4 @@
-import { useState, useEffect } from "react";
+import { useState, useEffect, useRef } from "react";
 import {
   Heading,
   VStack,
@@ -7,101 +7,121 @@ import {
   Image,
   Text,
   Box,
-  type BoxProps,
   Link as ChakraLink,
   Flex,
+  useBreakpointValue,
 } from "@chakra-ui/react";
 import Head from "next/head";
-import Link, { type LinkProps } from "next/link";
-import { BsGithub, BsPersonCircle } from "react-icons/bs";
-import { useRouter } from "next/router";
-import { type IconType } from "react-icons";
-import { RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
+import Link from "next/link";
+import { BsGearFill, BsGithub, BsPersonCircle } from "react-icons/bs";
+import { IoStatsChartOutline } from "react-icons/io5";
+import { RiHome3Line, RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
 import { signIn, useSession } from "next-auth/react";
-import UserMenu from "./UserMenu";
 import { env } from "~/env.mjs";
+import ProjectMenu from "./ProjectMenu";
+import NavSidebarOption from "./NavSidebarOption";
+import IconLink from "./IconLink";
 
-type IconLinkProps = BoxProps & LinkProps & { label?: string; icon: IconType; href: string };
-
-const IconLink = ({ icon, label, href, color, ...props }: IconLinkProps) => {
-  const router = useRouter();
-  const isActive = href && router.pathname.startsWith(href);
-  return (
-    <Link href={href} style={{ width: "100%" }}>
-      <HStack
-        w="full"
-        p={4}
-        color={color}
-        as={ChakraLink}
-        bgColor={isActive ? "gray.200" : "transparent"}
-        _hover={{ bgColor: "gray.300", textDecoration: "none" }}
-        justifyContent="start"
-        cursor="pointer"
-        {...props}
-      >
-        <Icon as={icon} boxSize={6} mr={2} />
-        <Text fontWeight="bold" fontSize="sm">
-          {label}
-        </Text>
-      </HStack>
-    </Link>
-  );
-};
-
-const Divider = () => <Box h="1px" bgColor="gray.200" />;
+const Divider = () => <Box h="1px" bgColor="gray.300" w="full" />;
 
 const NavSidebar = () => {
   const user = useSession().data;
 
+  // Hack to get around initial flash, see https://github.com/chakra-ui/chakra-ui/issues/6452
+  const isMobile = useBreakpointValue({ base: true, md: false, ssr: false });
+  const renderCount = useRef(0);
+  renderCount.current++;
+
+  const displayLogo = isMobile && renderCount.current > 1;
+
   return (
     <VStack
       align="stretch"
-      bgColor="gray.100"
       py={2}
+      px={2}
       pb={0}
       height="100%"
-      w={{ base: "56px", md: "200px" }}
+      w={{ base: "56px", md: "240px" }}
       overflow="hidden"
+      borderRightWidth={1}
+      borderColor="gray.300"
     >
-      <HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={4} py={2}>
-        <Image src="/logo.svg" alt="" boxSize={6} mr={4} />
-        <Heading size="md" fontFamily="inconsolata, monospace">
-          OpenPipe
-        </Heading>
-      </HStack>
-      <VStack spacing={0} align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
+      {displayLogo && (
+        <>
+          <HStack
+            as={Link}
+            href="/"
+            _hover={{ textDecoration: "none" }}
+            spacing={{ base: 1, md: 0 }}
+            mx={2}
+            py={{ base: 1, md: 2 }}
+          >
+            <Image src="/logo.svg" alt="" boxSize={6} mr={4} ml={{ base: 0.5, md: 0 }} />
+            <Heading size="md" fontFamily="inconsolata, monospace">
+              OpenPipe
+            </Heading>
+          </HStack>
+          <Divider />
+        </>
+      )}
+
+      <VStack align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
         {user != null && (
           <>
+            <ProjectMenu />
+            <Divider />
+
+            {env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS && (
+              <>
+                <IconLink icon={RiHome3Line} label="Dashboard" href="/dashboard" beta />
+                <IconLink
+                  icon={IoStatsChartOutline}
+                  label="Request Logs"
+                  href="/request-logs"
+                  beta
+                />
+              </>
+            )}
             <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
             {env.NEXT_PUBLIC_SHOW_DATA && (
               <IconLink icon={RiDatabase2Line} label="Data" href="/data" />
             )}
+            <VStack w="full" alignItems="flex-start" spacing={0} pt={8}>
+              <Text
+                pl={2}
+                pb={2}
+                fontSize="xs"
+                fontWeight="bold"
+                color="gray.500"
+                display={{ base: "none", md: "flex" }}
+              >
+                CONFIGURATION
+              </Text>
+              <IconLink icon={BsGearFill} label="Project Settings" href="/project/settings" />
+            </VStack>
           </>
         )}
         {user === null && (
-          <HStack
-            w="full"
-            p={4}
-            as={ChakraLink}
-            _hover={{ bgColor: "gray.300", textDecoration: "none" }}
-            justifyContent="start"
-            cursor="pointer"
-            onClick={() => {
-              signIn("github").catch(console.error);
-            }}
-          >
-            <Icon as={BsPersonCircle} boxSize={6} mr={2} />
-            <Text fontWeight="bold" fontSize="sm">
-              Sign In
-            </Text>
-          </HStack>
+          <NavSidebarOption>
+            <HStack
+              w="full"
+              p={4}
+              as={ChakraLink}
+              justifyContent="start"
+              onClick={() => {
+                signIn("github").catch(console.error);
+              }}
+            >
+              <Icon as={BsPersonCircle} boxSize={6} mr={2} />
+              <Text fontWeight="bold" fontSize="sm">
+                Sign In
+              </Text>
+            </HStack>
+          </NavSidebarOption>
         )}
       </VStack>
-      {user ? (
-        <UserMenu user={user} borderColor={"gray.200"} borderTopWidth={1} borderBottomWidth={1} />
-      ) : (
-        <Divider />
-      )}
+
+      <Divider />
       <VStack spacing={0} align="center">
         <ChakraLink
           href="https://github.com/openpipe/openpipe"
@@ -117,7 +137,15 @@ const NavSidebar = () => {
   );
 };
 
-export default function AppShell(props: { children: React.ReactNode; title?: string }) {
+export default function AppShell({
+  children,
+  title,
+  requireAuth,
+}: {
+  children: React.ReactNode;
+  title?: string;
+  requireAuth?: boolean;
+}) {
   const [vh, setVh] = useState("100vh"); // Default height to prevent flicker on initial render
 
   useEffect(() => {
@@ -137,14 +165,23 @@ export default function AppShell(props: { children: React.ReactNode; title?: str
     };
   }, []);
 
+  const user = useSession().data;
+  const authLoading = useSession().status === "loading";
+
+  useEffect(() => {
+    if (requireAuth && user === null && !authLoading) {
+      signIn("github").catch(console.error);
+    }
+  }, [requireAuth, user, authLoading]);
+
   return (
     <Flex h={vh} w="100vw">
       <Head>
-        <title>{props.title ? `${props.title} | OpenPipe` : "OpenPipe"}</title>
+        <title>{title ? `${title} | OpenPipe` : "OpenPipe"}</title>
       </Head>
       <NavSidebar />
-      <Box h="100%" flex={1} overflowY="auto">
-        {props.children}
+      <Box h="100%" flex={1} overflowY="auto" bgColor="gray.50">
+        {children}
       </Box>
     </Flex>
   );

app/src/components/nav/IconLink.tsx

@@ -0,0 +1,31 @@
+import { Icon, HStack, Text, type BoxProps } from "@chakra-ui/react";
+import Link, { type LinkProps } from "next/link";
+import { type IconType } from "react-icons";
+import NavSidebarOption from "./NavSidebarOption";
+
+type IconLinkProps = BoxProps &
+  LinkProps & { label?: string; icon: IconType; href: string; beta?: boolean };
+
+const IconLink = ({ icon, label, href, color, beta, ...props }: IconLinkProps) => {
+  return (
+    <Link href={href} style={{ width: "100%" }}>
+      <NavSidebarOption activeHrefPattern={href}>
+        <HStack w="full" justifyContent="space-between" p={2} color={color} {...props}>
+          <HStack w="full" justifyContent="start">
+            <Icon as={icon} boxSize={6} mr={2} />
+            <Text fontSize="sm" display={{ base: "none", md: "block" }}>
+              {label}
+            </Text>
+          </HStack>
+          {beta && (
+            <Text fontSize="xs" ml={2} fontWeight="bold" color="orange.400">
+              BETA
+            </Text>
+          )}
+        </HStack>
+      </NavSidebarOption>
+    </Link>
+  );
+};
+
+export default IconLink;

app/src/components/nav/NavSidebarOption.tsx

@@ -0,0 +1,29 @@
+import { Box, type BoxProps, forwardRef } from "@chakra-ui/react";
+import { useRouter } from "next/router";
+
+const NavSidebarOption = forwardRef<
+  { activeHrefPattern?: string; disableHoverEffect?: boolean } & BoxProps,
+  "div"
+>(({ activeHrefPattern, disableHoverEffect, ...props }, ref) => {
+  const router = useRouter();
+  const isActive = activeHrefPattern && router.pathname.startsWith(activeHrefPattern);
+  return (
+    <Box
+      w="full"
+      fontWeight={isActive ? "bold" : "500"}
+      bgColor={isActive ? "gray.200" : "transparent"}
+      _hover={disableHoverEffect ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
+      justifyContent="start"
+      cursor="pointer"
+      borderRadius={4}
+      {...props}
+      ref={ref}
+    >
+      {props.children}
+    </Box>
+  );
+});
+
+NavSidebarOption.displayName = "NavSidebarOption";
+
+export default NavSidebarOption;

app/src/components/nav/PageHeaderContainer.tsx

@@ -0,0 +1,19 @@
+import { Flex, type FlexProps } from "@chakra-ui/react";
+
+const PageHeaderContainer = (props: FlexProps) => {
+  return (
+    <Flex
+      px={8}
+      py={2}
+      minH={16}
+      w="full"
+      direction={{ base: "column", sm: "row" }}
+      alignItems={{ base: "flex-start", sm: "center" }}
+      justifyContent="space-between"
+      fontWeight="500"
+      {...props}
+    />
+  );
+};
+
+export default PageHeaderContainer;

app/src/components/nav/ProjectBreadcrumbContents.tsx

@@ -0,0 +1,28 @@
+import { HStack, Flex, Text } from "@chakra-ui/react";
+import { useSelectedProject } from "~/utils/hooks";
+
+// Have to export only contents here instead of full BreadcrumbItem because Chakra doesn't
+// recognize a BreadcrumbItem exported with this component as a valid child of Breadcrumb.
+export default function ProjectBreadcrumbContents({ projectName = "" }: { projectName?: string }) {
+  const { data: selectedProject } = useSelectedProject();
+
+  projectName = projectName || selectedProject?.name || "";
+
+  return (
+    <HStack w="full">
+      <Flex
+        p={1}
+        borderRadius={4}
+        backgroundColor="orange.100"
+        boxSize={6}
+        alignItems="center"
+        justifyContent="center"
+      >
+        <Text>{projectName[0]?.toUpperCase()}</Text>
+      </Flex>
+      <Text display={{ base: "none", md: "block" }} py={1}>
+        {projectName}
+      </Text>
+    </HStack>
+  );
+}

app/src/components/nav/ProjectMenu.tsx

@@ -0,0 +1,201 @@
+import {
+  HStack,
+  VStack,
+  Text,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  Flex,
+  Icon,
+  Divider,
+  Button,
+  useDisclosure,
+  Spinner,
+  Link as ChakraLink,
+  Image,
+  Box,
+} from "@chakra-ui/react";
+import React, { useEffect, useState } from "react";
+import Link from "next/link";
+import { BsPlus, BsPersonCircle } from "react-icons/bs";
+import { type Project } from "@prisma/client";
+
+import { useAppStore } from "~/state/store";
+import { api } from "~/utils/api";
+import NavSidebarOption from "./NavSidebarOption";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+import { useRouter } from "next/router";
+import { useSession, signOut } from "next-auth/react";
+
+export default function ProjectMenu() {
+  const router = useRouter();
+  const utils = api.useContext();
+
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
+  const setSelectedProjectId = useAppStore((s) => s.setSelectedProjectId);
+
+  const { data: projects } = api.projects.list.useQuery();
+
+  useEffect(() => {
+    if (
+      projects &&
+      projects[0] &&
+      (!selectedProjectId || !projects.find((proj) => proj.id === selectedProjectId))
+    ) {
+      setSelectedProjectId(projects[0].id);
+    }
+  }, [selectedProjectId, setSelectedProjectId, projects]);
+
+  const { data: selectedProject } = useSelectedProject();
+
+  const popover = useDisclosure();
+
+  const createMutation = api.projects.create.useMutation();
+  const [createProject, isLoading] = useHandledAsyncCallback(async () => {
+    const newProj = await createMutation.mutateAsync({ name: "Untitled Project" });
+    await utils.projects.list.invalidate();
+    setSelectedProjectId(newProj.id);
+    await router.push({ pathname: "/project/settings" });
+  }, [createMutation, router]);
+
+  const user = useSession().data;
+
+  const profileImage = user?.user.image ? (
+    <Image src={user.user.image} alt="profile picture" boxSize={6} borderRadius="50%" />
+  ) : (
+    <Icon as={BsPersonCircle} boxSize={6} />
+  );
+
+  return (
+    <VStack w="full" alignItems="flex-start" spacing={0} py={1}>
+      <Popover
+        placement="bottom"
+        isOpen={popover.isOpen}
+        onOpen={popover.onOpen}
+        onClose={popover.onClose}
+        closeOnBlur
+      >
+        <PopoverTrigger>
+          <NavSidebarOption>
+            <HStack w="full">
+              <Flex
+                p={1}
+                borderRadius={4}
+                backgroundColor="orange.100"
+                minW={{ base: 10, md: 8 }}
+                minH={{ base: 10, md: 8 }}
+                m={{ base: 0, md: 1 }}
+                alignItems="center"
+                justifyContent="center"
+              >
+                <Text>{selectedProject?.name[0]?.toUpperCase()}</Text>
+              </Flex>
+              <Text
+                fontSize="sm"
+                display={{ base: "none", md: "block" }}
+                py={1}
+                flex={1}
+                fontWeight="bold"
+              >
+                {selectedProject?.name}
+              </Text>
+              <Box mr={2}>{profileImage}</Box>
+            </HStack>
+          </NavSidebarOption>
+        </PopoverTrigger>
+        <PopoverContent
+          _focusVisible={{ outline: "unset" }}
+          ml={-1}
+          w={224}
+          boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
+          fontSize="sm"
+        >
+          <VStack alignItems="flex-start" spacing={1} py={1}>
+            <Text px={3} py={2}>
+              {user?.user.email}
+            </Text>
+            <Divider />
+            <Text alignSelf="flex-start" fontWeight="bold" px={3} pt={2}>
+              Your Projects
+            </Text>
+            <VStack spacing={0} w="full" px={1}>
+              {projects?.map((proj) => (
+                <ProjectOption
+                  key={proj.id}
+                  proj={proj}
+                  isActive={proj.id === selectedProjectId}
+                  onClose={popover.onClose}
+                />
+              ))}
+              <HStack
+                as={Button}
+                variant="ghost"
+                colorScheme="blue"
+                color="blue.400"
+                fontSize="sm"
+                justifyContent="flex-start"
+                onClick={createProject}
+                w="full"
+                borderRadius={4}
+                spacing={0}
+              >
+                <Text>Add project</Text>
+                <Icon as={isLoading ? Spinner : BsPlus} boxSize={4} strokeWidth={0.5} />
+              </HStack>
+            </VStack>
+
+            <Divider />
+            <VStack w="full" px={1}>
+              <ChakraLink
+                onClick={() => {
+                  signOut().catch(console.error);
+                }}
+                _hover={{ bgColor: "gray.200", textDecoration: "none" }}
+                w="full"
+                py={2}
+                px={2}
+                borderRadius={4}
+              >
+                <Text>Sign out</Text>
+              </ChakraLink>
+            </VStack>
+          </VStack>
+        </PopoverContent>
+      </Popover>
+    </VStack>
+  );
+}
+
+const ProjectOption = ({
+  proj,
+  isActive,
+  onClose,
+}: {
+  proj: Project;
+  isActive: boolean;
+  onClose: () => void;
+}) => {
+  const setSelectedProjectId = useAppStore((s) => s.setSelectedProjectId);
+  const [gearHovered, setGearHovered] = useState(false);
+
+  return (
+    <HStack
+      as={Link}
+      href="/experiments"
+      onClick={() => {
+        setSelectedProjectId(proj.id);
+        onClose();
+      }}
+      w="full"
+      justifyContent="space-between"
+      _hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
+      color={isActive ? "blue.400" : undefined}
+      py={2}
+      px={4}
+      borderRadius={4}
+      spacing={4}
+    >
+      <Text>{proj.name}</Text>
+    </HStack>
+  );
+};

app/src/components/nav/UserMenu.tsx

@@ -8,16 +8,14 @@ import {
   PopoverTrigger,
   PopoverContent,
   Link,
-  useColorMode,
   type StackProps,
 } from "@chakra-ui/react";
 import { type Session } from "next-auth";
 import { signOut } from "next-auth/react";
 import { BsBoxArrowRight, BsChevronRight, BsPersonCircle } from "react-icons/bs";
+import NavSidebarOption from "./NavSidebarOption";
 
 export default function UserMenu({ user, ...rest }: { user: Session } & StackProps) {
-  const { colorMode } = useColorMode();
-
   const profileImage = user.user.image ? (
     <Image src={user.user.image} alt="profile picture" boxSize={8} borderRadius="50%" />
   ) : (
@@ -28,30 +26,28 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
     <>
       <Popover placement="right">
         <PopoverTrigger>
-          <HStack
-            // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
-            px={3}
-            spacing={3}
-            py={2}
-            {...rest}
-            cursor="pointer"
-            _hover={{
-              bgColor: colorMode === "light" ? "gray.200" : "gray.700",
-            }}
-          >
-            {profileImage}
-            <VStack spacing={0} align="start" flex={1} flexShrink={1}>
-              <Text fontWeight="bold" fontSize="sm">
-                {user.user.name}
-              </Text>
-              <Text color="gray.500" fontSize="xs">
-                {user.user.email}
-              </Text>
-            </VStack>
-            <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
-          </HStack>
+          <NavSidebarOption>
+            <HStack
+              // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
+              py={2}
+              px={1}
+              spacing={3}
+              {...rest}
+            >
+              {profileImage}
+              <VStack spacing={0} align="start" flex={1} flexShrink={1}>
+                <Text fontWeight="bold" fontSize="sm">
+                  {user.user.name}
+                </Text>
+                <Text color="gray.500" fontSize="xs">
+                  {/* {user.user.email} */}
+                </Text>
+              </VStack>
+              <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+            </HStack>
+          </NavSidebarOption>
         </PopoverTrigger>
-        <PopoverContent _focusVisible={{ boxShadow: "unset", outline: "unset" }} maxW="200px">
+        <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} minW={48} w="full">
           <VStack align="stretch" spacing={0}>
             {/* sign out */}
             <HStack

app/src/components/projectSettings/DeleteProjectDialog.tsx

@@ -0,0 +1,89 @@
+import {
+  Button,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+  Input,
+  Text,
+  VStack,
+  Box,
+  Spinner,
+} from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import { useRef, useState } from "react";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+
+export const DeleteProjectDialog = ({
+  isOpen,
+  onClose,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+}) => {
+  const selectedProject = useSelectedProject();
+  const deleteMutation = api.projects.delete.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onDeleteConfirm, isDeleting] = useHandledAsyncCallback(async () => {
+    if (!selectedProject.data?.id) return;
+    await deleteMutation.mutateAsync({ id: selectedProject.data.id });
+    await utils.projects.list.invalidate();
+    await router.push({ pathname: "/experiments" });
+    onClose();
+  }, [deleteMutation, selectedProject, router]);
+
+  const [nameToDelete, setNameToDelete] = useState("");
+
+  return (
+    <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Delete Project
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            <VStack spacing={4} alignItems="flex-start">
+              <Text>
+                If you delete this project all the associated data and experiments will be deleted
+                as well. If you are sure that you want to delete this project, please type the name
+                of the project below.
+              </Text>
+              <Box bgColor="orange.100" w="full" p={2} borderRadius={4}>
+                <Text fontFamily="inconsolata">{selectedProject.data?.name}</Text>
+              </Box>
+              <Input
+                placeholder={selectedProject.data?.name}
+                value={nameToDelete}
+                onChange={(e) => setNameToDelete(e.target.value)}
+              />
+            </VStack>
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={onClose}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="red"
+              onClick={onDeleteConfirm}
+              ml={3}
+              isDisabled={nameToDelete !== selectedProject.data?.name}
+              w={20}
+            >
+              {isDeleting ? <Spinner /> : "Delete"}
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};

app/src/components/requestLogs/ActionButton.tsx

@@ -0,0 +1,30 @@
+import { Button, HStack, type ButtonProps, Icon, Text } from "@chakra-ui/react";
+import { type IconType } from "react-icons";
+
+const ActionButton = ({
+  icon,
+  label,
+  ...buttonProps
+}: { icon: IconType; label: string } & ButtonProps) => {
+  return (
+    <Button
+      colorScheme="blue"
+      color="black"
+      bgColor="white"
+      borderColor="gray.300"
+      borderRadius={4}
+      variant="outline"
+      size="sm"
+      fontSize="sm"
+      fontWeight="normal"
+      {...buttonProps}
+    >
+      <HStack spacing={1}>
+        {icon && <Icon as={icon} />}
+        <Text>{label}</Text>
+      </HStack>
+    </Button>
+  );
+};
+
+export default ActionButton;

app/src/components/requestLogs/FormattedJson.tsx

@@ -0,0 +1,55 @@
+import { Box, IconButton, useToast } from "@chakra-ui/react";
+import { CopyIcon } from "lucide-react";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import { atelierCaveLight } from "react-syntax-highlighter/dist/cjs/styles/hljs";
+import stringify from "json-stringify-pretty-compact";
+
+const FormattedJson = ({ json }: { json: any }) => {
+  const jsonString = stringify(json, { maxLength: 40 });
+  const toast = useToast();
+
+  const copyToClipboard = async (text: string) => {
+    try {
+      await navigator.clipboard.writeText(text);
+      toast({
+        title: "Copied to clipboard",
+        status: "success",
+        duration: 2000,
+      });
+    } catch (err) {
+      toast({
+        title: "Failed to copy to clipboard",
+        status: "error",
+        duration: 2000,
+      });
+    }
+  };
+
+  return (
+    <Box position="relative" fontSize="sm" borderRadius="md" overflow="hidden">
+      <SyntaxHighlighter
+        customStyle={{ overflowX: "unset" }}
+        language="json"
+        style={atelierCaveLight}
+        lineProps={{
+          style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
+        }}
+        wrapLines
+      >
+        {jsonString}
+      </SyntaxHighlighter>
+      <IconButton
+        aria-label="Copy"
+        icon={<CopyIcon />}
+        position="absolute"
+        top={1}
+        right={1}
+        size="xs"
+        variant="ghost"
+        onClick={() => void copyToClipboard(jsonString)}
+      />
+    </Box>
+  );
+};
+
+export { FormattedJson };

app/src/components/requestLogs/LoggedCallsPaginator.tsx

@@ -0,0 +1,16 @@
+import { type StackProps } from "@chakra-ui/react";
+
+import { useLoggedCalls } from "~/utils/hooks";
+import Paginator from "../Paginator";
+
+const LoggedCallsPaginator = (props: StackProps) => {
+  const { data } = useLoggedCalls();
+
+  if (!data) return null;
+
+  const { count } = data;
+
+  return <Paginator count={count} {...props} />;
+};
+
+export default LoggedCallsPaginator;

app/src/components/requestLogs/LoggedCallsTable.tsx

@@ -0,0 +1,36 @@
+import { Card, Table, Tbody } from "@chakra-ui/react";
+import { useState } from "react";
+import { useLoggedCalls } from "~/utils/hooks";
+import { TableHeader, TableRow } from "./TableRow";
+
+export default function LoggedCallsTable() {
+  const [expandedRow, setExpandedRow] = useState<string | null>(null);
+  const { data: loggedCalls } = useLoggedCalls();
+
+  return (
+    <Card width="100%" overflow="hidden">
+      <Table>
+        <TableHeader showCheckbox />
+        <Tbody>
+          {loggedCalls?.calls.map((loggedCall) => {
+            return (
+              <TableRow
+                key={loggedCall.id}
+                loggedCall={loggedCall}
+                isExpanded={loggedCall.id === expandedRow}
+                onToggle={() => {
+                  if (loggedCall.id === expandedRow) {
+                    setExpandedRow(null);
+                  } else {
+                    setExpandedRow(loggedCall.id);
+                  }
+                }}
+                showCheckbox
+              />
+            );
+          })}
+        </Tbody>
+      </Table>
+    </Card>
+  );
+}

app/src/components/requestLogs/TableRow.tsx

@@ -0,0 +1,164 @@
+import {
+  Box,
+  Heading,
+  Td,
+  Tr,
+  Thead,
+  Th,
+  Tooltip,
+  Collapse,
+  HStack,
+  VStack,
+  Button,
+  ButtonGroup,
+  Text,
+  Checkbox,
+} from "@chakra-ui/react";
+import dayjs from "dayjs";
+import relativeTime from "dayjs/plugin/relativeTime";
+import Link from "next/link";
+
+import { type RouterOutputs } from "~/utils/api";
+import { FormattedJson } from "./FormattedJson";
+import { useAppStore } from "~/state/store";
+import { useLoggedCalls } from "~/utils/hooks";
+import { useMemo } from "react";
+
+dayjs.extend(relativeTime);
+
+type LoggedCall = RouterOutputs["loggedCalls"]["list"]["calls"][0];
+
+export const TableHeader = ({ showCheckbox }: { showCheckbox?: boolean }) => {
+  const matchingLogIds = useLoggedCalls().data?.matchingLogIds;
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const addAll = useAppStore((s) => s.selectedLogs.addSelectedLogIds);
+  const clearAll = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
+  const allSelected = useMemo(() => {
+    if (!matchingLogIds) return false;
+    return matchingLogIds.every((id) => selectedLogIds.has(id));
+  }, [selectedLogIds, matchingLogIds]);
+  return (
+    <Thead>
+      <Tr>
+        {showCheckbox && (
+          <Th>
+            <HStack w={8}>
+              <Checkbox
+                isChecked={allSelected}
+                onChange={() => {
+                  allSelected ? clearAll() : addAll(matchingLogIds || []);
+                }}
+              />
+              <Text>({selectedLogIds.size})</Text>
+            </HStack>
+          </Th>
+        )}
+        <Th>Time</Th>
+        <Th>Model</Th>
+        <Th isNumeric>Duration</Th>
+        <Th isNumeric>Input tokens</Th>
+        <Th isNumeric>Output tokens</Th>
+        <Th isNumeric>Status</Th>
+      </Tr>
+    </Thead>
+  );
+};
+
+export const TableRow = ({
+  loggedCall,
+  isExpanded,
+  onToggle,
+  showCheckbox,
+}: {
+  loggedCall: LoggedCall;
+  isExpanded: boolean;
+  onToggle: () => void;
+  showCheckbox?: boolean;
+}) => {
+  const isError = loggedCall.modelResponse?.statusCode !== 200;
+  const timeAgo = dayjs(loggedCall.requestedAt).fromNow();
+  const fullTime = dayjs(loggedCall.requestedAt).toString();
+
+  const durationCell = (
+    <Td isNumeric>
+      {loggedCall.cacheHit ? (
+        <Text color="gray.500">Cached</Text>
+      ) : (
+        ((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2) + "s"
+      )}
+    </Td>
+  );
+
+  const isChecked = useAppStore((s) => s.selectedLogs.selectedLogIds.has(loggedCall.id));
+  const toggleChecked = useAppStore((s) => s.selectedLogs.toggleSelectedLogId);
+
+  return (
+    <>
+      <Tr
+        onClick={onToggle}
+        key={loggedCall.id}
+        _hover={{ bgColor: "gray.50", cursor: "pointer" }}
+        sx={{
+          "> td": { borderBottom: "none" },
+        }}
+      >
+        {showCheckbox && (
+          <Td>
+            <Checkbox isChecked={isChecked} onChange={() => toggleChecked(loggedCall.id)} />
+          </Td>
+        )}
+        <Td>
+          <Tooltip label={fullTime} placement="top">
+            <Box whiteSpace="nowrap" minW="120px">
+              {timeAgo}
+            </Box>
+          </Tooltip>
+        </Td>
+        <Td width="100%">
+          <HStack justifyContent="flex-start">
+            <Text
+              colorScheme="purple"
+              color="purple.500"
+              borderColor="purple.500"
+              px={1}
+              borderRadius={4}
+              borderWidth={1}
+              fontSize="xs"
+            >
+              {loggedCall.model}
+            </Text>
+          </HStack>
+        </Td>
+        {durationCell}
+        <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
+        <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
+        <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
+          {loggedCall.modelResponse?.statusCode ?? "No response"}
+        </Td>
+      </Tr>
+      <Tr>
+        <Td colSpan={8} p={0}>
+          <Collapse in={isExpanded} unmountOnExit={true}>
+            <VStack p={4} align="stretch">
+              <HStack align="stretch">
+                <VStack flex={1} align="stretch">
+                  <Heading size="sm">Input</Heading>
+                  <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
+                </VStack>
+                <VStack flex={1} align="stretch">
+                  <Heading size="sm">Output</Heading>
+                  <FormattedJson json={loggedCall.modelResponse?.respPayload} />
+                </VStack>
+              </HStack>
+              <ButtonGroup alignSelf="flex-end">
+                <Button as={Link} colorScheme="blue" href={{ pathname: "/experiments" }}>
+                  Experiments
+                </Button>
+              </ButtonGroup>
+            </VStack>
+          </Collapse>
+        </Td>
+      </Tr>
+    </>
+  );
+};

app/src/components/tooltip/CostTooltip.tsx

@@ -2,14 +2,14 @@ import { HStack, Icon, Text, Tooltip, type TooltipProps, VStack, Divider } from
 import { BsCurrencyDollar } from "react-icons/bs";
 
 type CostTooltipProps = {
-  promptTokens: number | null;
-  completionTokens: number | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
   cost: number;
 } & TooltipProps;
 
 export const CostTooltip = ({
-  promptTokens,
-  completionTokens,
+  inputTokens,
+  outputTokens,
   cost,
   children,
   ...props
@@ -36,12 +36,12 @@ export const CostTooltip = ({
             <HStack>
               <VStack w="28" spacing={1}>
                 <Text>Prompt</Text>
-                <Text>{promptTokens ?? 0}</Text>
+                <Text>{inputTokens ?? 0}</Text>
               </VStack>
               <Divider borderColor="gray.200" h={8} orientation="vertical" />
               <VStack w="28" spacing={1}>
                 <Text whiteSpace="nowrap">Completion</Text>
-                <Text>{completionTokens ?? 0}</Text>
+                <Text>{outputTokens ?? 0}</Text>
               </VStack>
             </HStack>
           </VStack>

app/src/env.mjs

@@ -20,6 +20,7 @@ export const env = createEnv({
     REPLICATE_API_TOKEN: z.string().default("placeholder"),
     ANTHROPIC_API_KEY: z.string().default("placeholder"),
     SENTRY_AUTH_TOKEN: z.string().optional(),
+    OPENPIPE_API_KEY: z.string().optional(),
   },
 
   /**
@@ -33,6 +34,7 @@ export const env = createEnv({
     NEXT_PUBLIC_HOST: z.string().url().default("http://localhost:3000"),
     NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
     NEXT_PUBLIC_SHOW_DATA: z.string().optional(),
+    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: z.string().optional(),
   },
 
   /**
@@ -54,6 +56,8 @@ export const env = createEnv({
     ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
     NEXT_PUBLIC_SENTRY_DSN: process.env.NEXT_PUBLIC_SENTRY_DSN,
     SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
+    OPENPIPE_API_KEY: process.env.OPENPIPE_API_KEY,
+    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: process.env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/modelProviders/anthropic-completion/codegen/input.schema.json

@@ -9,7 +9,8 @@
         "claude-2",
         "claude-2.0",
         "claude-instant-1",
-        "claude-instant-1.1"
+        "claude-instant-1.1",
+        "claude-instant-1.2"
       ]
     },
     "prompt": {

app/src/modelProviders/anthropic-completion/index.ts

@@ -28,6 +28,10 @@ const modelProvider: AnthropicProvider = {
   inputSchema: inputSchema as JSONSchema4,
   canStream: true,
   getCompletion,
+  getUsage: (input, output) => {
+    // TODO: add usage logic
+    return null;
+  },
   ...frontendModelProvider,
 };
 

app/src/modelProviders/openai-ChatCompletion/getCompletion.ts

@@ -4,14 +4,10 @@ import {
   type ChatCompletion,
   type CompletionCreateParams,
 } from "openai/resources/chat";
-import { countOpenAIChatTokens } from "~/utils/countTokens";
 import { type CompletionResponse } from "../types";
 import { isArray, isString, omit } from "lodash-es";
 import { openai } from "~/server/utils/openai";
-import { truthyFilter } from "~/utils/utils";
 import { APIError } from "openai";
-import frontendModelProvider from "./frontend";
-import modelProvider, { type SupportedModel } from ".";
 
 const mergeStreamedChunks = (
   base: ChatCompletion | null,
@@ -60,9 +56,6 @@ export async function getCompletion(
 ): Promise<CompletionResponse<ChatCompletion>> {
   const start = Date.now();
   let finalCompletion: ChatCompletion | null = null;
-  let promptTokens: number | undefined = undefined;
-  let completionTokens: number | undefined = undefined;
-  const modelName = modelProvider.getModel(input) as SupportedModel;
 
   try {
     if (onStream) {
@@ -86,16 +79,6 @@ export async function getCompletion(
           autoRetry: false,
         };
       }
-      try {
-        promptTokens = countOpenAIChatTokens(modelName, input.messages);
-        completionTokens = countOpenAIChatTokens(
-          modelName,
-          finalCompletion.choices.map((c) => c.message).filter(truthyFilter),
-        );
-      } catch (err) {
-        // TODO handle this, library seems like maybe it doesn't work with function calls?
-        console.error(err);
-      }
     } else {
       const resp = await openai.chat.completions.create(
         { ...input, stream: false },
@@ -104,25 +87,14 @@ export async function getCompletion(
         },
       );
       finalCompletion = resp;
-      promptTokens = resp.usage?.prompt_tokens ?? 0;
-      completionTokens = resp.usage?.completion_tokens ?? 0;
     }
     const timeToComplete = Date.now() - start;
 
-    const { promptTokenPrice, completionTokenPrice } = frontendModelProvider.models[modelName];
-    let cost = undefined;
-    if (promptTokenPrice && completionTokenPrice && promptTokens && completionTokens) {
-      cost = promptTokens * promptTokenPrice + completionTokens * completionTokenPrice;
-    }
-
     return {
       type: "success",
       statusCode: 200,
       value: finalCompletion,
       timeToComplete,
-      promptTokens,
-      completionTokens,
-      cost,
     };
   } catch (error: unknown) {
     if (error instanceof APIError) {

app/src/modelProviders/openai-ChatCompletion/index.ts

@@ -4,6 +4,8 @@ import inputSchema from "./codegen/input.schema.json";
 import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
 import { getCompletion } from "./getCompletion";
 import frontendModelProvider from "./frontend";
+import { countOpenAIChatTokens } from "~/utils/countTokens";
+import { truthyFilter } from "~/utils/utils";
 
 const supportedModels = [
   "gpt-4-0613",
@@ -39,6 +41,41 @@ const modelProvider: OpenaiChatModelProvider = {
   inputSchema: inputSchema as JSONSchema4,
   canStream: true,
   getCompletion,
+  getUsage: (input, output) => {
+    if (output.choices.length === 0) return null;
+
+    const model = modelProvider.getModel(input);
+    if (!model) return null;
+
+    let inputTokens: number;
+    let outputTokens: number;
+
+    if (output.usage) {
+      inputTokens = output.usage.prompt_tokens;
+      outputTokens = output.usage.completion_tokens;
+    } else {
+      try {
+        inputTokens = countOpenAIChatTokens(model, input.messages);
+        outputTokens = countOpenAIChatTokens(
+          model,
+          output.choices.map((c) => c.message).filter(truthyFilter),
+        );
+      } catch (err) {
+        inputTokens = 0;
+        outputTokens = 0;
+        // TODO handle this, library seems like maybe it doesn't work with function calls?
+        console.error(err);
+      }
+    }
+
+    const { promptTokenPrice, completionTokenPrice } = frontendModelProvider.models[model];
+    let cost = undefined;
+    if (promptTokenPrice && completionTokenPrice && inputTokens && outputTokens) {
+      cost = inputTokens * promptTokenPrice + outputTokens * completionTokenPrice;
+    }
+
+    return { inputTokens: inputTokens, outputTokens: outputTokens, cost };
+  },
   ...frontendModelProvider,
 };
 

app/src/modelProviders/replicate-llama2/index.ts

@@ -75,6 +75,10 @@ const modelProvider: ReplicateLlama2Provider = {
   },
   canStream: true,
   getCompletion,
+  getUsage: (input, output) => {
+    // TODO: add usage logic
+    return null;
+  },
   ...frontendModelProvider,
 };
 

app/src/modelProviders/types.ts

@@ -43,9 +43,6 @@ export type CompletionResponse<T> =
       value: T;
       timeToComplete: number;
       statusCode: number;
-      promptTokens?: number;
-      completionTokens?: number;
-      cost?: number;
     };
 
 export type ModelProvider<SupportedModels extends string, InputSchema, OutputSchema> = {
@@ -56,6 +53,10 @@ export type ModelProvider<SupportedModels extends string, InputSchema, OutputSch
     input: InputSchema,
     onStream: ((partialOutput: OutputSchema) => void) | null,
   ) => Promise<CompletionResponse<OutputSchema>>;
+  getUsage: (
+    input: InputSchema,
+    output: OutputSchema,
+  ) => { gpuRuntime?: number; inputTokens?: number; outputTokens?: number; cost?: number } | null;
 
   // This is just a convenience for type inference, don't use it at runtime
   _outputSchema?: OutputSchema | null;

app/src/pages/_app.tsx

@@ -8,7 +8,7 @@ import { ChakraThemeProvider } from "~/theme/ChakraThemeProvider";
 import { SyncAppStore } from "~/state/sync";
 import NextAdapterApp from "next-query-params/app";
 import { QueryParamProvider } from "use-query-params";
-import { SessionIdentifier } from "~/utils/analytics/clientAnalytics";
+import { PosthogAppProvider } from "~/utils/analytics/posthog";
 
 const MyApp: AppType<{ session: Session | null }> = ({
   Component,
@@ -34,14 +34,15 @@ const MyApp: AppType<{ session: Session | null }> = ({
         <meta name="twitter:image" content="/og.png" />
       </Head>
       <SessionProvider session={session}>
-        <SyncAppStore />
-        <Favicon />
-        <SessionIdentifier />
-        <ChakraThemeProvider>
-          <QueryParamProvider adapter={NextAdapterApp}>
-            <Component {...pageProps} />
-          </QueryParamProvider>
-        </ChakraThemeProvider>
+        <PosthogAppProvider>
+          <SyncAppStore />
+          <Favicon />
+          <ChakraThemeProvider>
+            <QueryParamProvider adapter={NextAdapterApp}>
+              <Component {...pageProps} />
+            </QueryParamProvider>
+          </ChakraThemeProvider>
+        </PosthogAppProvider>
       </SessionProvider>
     </>
   );

app/src/pages/api/[...trpc].ts

@@ -0,0 +1,22 @@
+import { type NextApiRequest, type NextApiResponse } from "next";
+import cors from "nextjs-cors";
+import { createOpenApiNextHandler } from "trpc-openapi";
+import { createProcedureCache } from "trpc-openapi/dist/adapters/node-http/procedures";
+import { appRouter } from "~/server/api/root.router";
+import { createTRPCContext } from "~/server/api/trpc";
+
+const openApiHandler = createOpenApiNextHandler({
+  router: appRouter,
+  createContext: createTRPCContext,
+});
+
+const cache = createProcedureCache(appRouter);
+
+const handler = async (req: NextApiRequest, res: NextApiResponse) => {
+  // Setup CORS
+  await cors(req, res);
+
+  return openApiHandler(req, res);
+};
+
+export default handler;

app/src/pages/api/openapi.json.ts

@@ -0,0 +1,16 @@
+import { type NextApiRequest, type NextApiResponse } from "next";
+import { generateOpenApiDocument } from "trpc-openapi";
+import { appRouter } from "~/server/api/root.router";
+
+export const openApiDocument = generateOpenApiDocument(appRouter, {
+  title: "OpenPipe API",
+  description: "The public API for reporting API calls to OpenPipe",
+  version: "0.1.0",
+  baseUrl: "https://app.openpipe.ai/api",
+});
+// Respond with our OpenAPI schema
+const hander = (req: NextApiRequest, res: NextApiResponse) => {
+  res.status(200).send(openApiDocument);
+};
+
+export default hander;

app/src/pages/dashboard/index.tsx

@@ -0,0 +1,114 @@
+import {
+  Heading,
+  Text,
+  Stat,
+  StatLabel,
+  StatNumber,
+  VStack,
+  HStack,
+  Card,
+  CardBody,
+  CardHeader,
+  Icon,
+  Table,
+  Tbody,
+  Tr,
+  Td,
+  Divider,
+} from "@chakra-ui/react";
+import { Ban, DollarSign, Hash } from "lucide-react";
+
+import AppShell from "~/components/nav/AppShell";
+import { useSelectedProject } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import LoggedCallsTable from "~/components/dashboard/LoggedCallsTable";
+import UsageGraph from "~/components/dashboard/UsageGraph";
+
+export default function Dashboard() {
+  const { data: selectedProject } = useSelectedProject();
+
+  const stats = api.dashboard.stats.useQuery(
+    { projectId: selectedProject?.id ?? "" },
+    { enabled: !!selectedProject },
+  );
+
+  return (
+    <AppShell title="Dashboard" requireAuth>
+      <VStack px={8} py={8} alignItems="flex-start" spacing={4}>
+        <Text fontSize="2xl" fontWeight="bold">
+          Dashboard
+        </Text>
+        <Divider />
+        <VStack margin="auto" spacing={4} align="stretch" w="full">
+          <HStack gap={4} align="start">
+            <Card flex={1}>
+              <CardHeader>
+                <Heading as="h3" size="sm">
+                  Usage Statistics
+                </Heading>
+              </CardHeader>
+              <CardBody>
+                <UsageGraph />
+              </CardBody>
+            </Card>
+            <VStack spacing="4" width="300px" align="stretch">
+              <Card>
+                <CardBody>
+                  <Stat>
+                    <HStack>
+                      <StatLabel flex={1}>Total Spent</StatLabel>
+                      <Icon as={DollarSign} boxSize={4} color="gray.500" />
+                    </HStack>
+                    <StatNumber>
+                      ${parseFloat(stats.data?.totals?.cost?.toString() ?? "0").toFixed(3)}
+                    </StatNumber>
+                  </Stat>
+                </CardBody>
+              </Card>
+              <Card>
+                <CardBody>
+                  <Stat>
+                    <HStack>
+                      <StatLabel flex={1}>Total Requests</StatLabel>
+                      <Icon as={Hash} boxSize={4} color="gray.500" />
+                    </HStack>
+                    <StatNumber>
+                      {stats.data?.totals?.numQueries
+                        ? parseInt(stats.data?.totals?.numQueries.toString())?.toLocaleString()
+                        : undefined}
+                    </StatNumber>
+                  </Stat>
+                </CardBody>
+              </Card>
+              <Card overflow="hidden">
+                <Stat>
+                  <CardHeader>
+                    <HStack>
+                      <StatLabel flex={1}>Errors</StatLabel>
+                      <Icon as={Ban} boxSize={4} color="gray.500" />
+                    </HStack>
+                  </CardHeader>
+                  <Table variant="simple">
+                    <Tbody>
+                      {stats.data?.errors?.map((error) => (
+                        <Tr key={error.code}>
+                          <Td>
+                            {error.name} ({error.code})
+                          </Td>
+                          <Td isNumeric color="red.600">
+                            {parseInt(error.count.toString()).toLocaleString()}
+                          </Td>
+                        </Tr>
+                      ))}
+                    </Tbody>
+                  </Table>
+                </Stat>
+              </Card>
+            </VStack>
+          </HStack>
+          <LoggedCallsTable />
+        </VStack>
+      </VStack>
+    </AppShell>
+  );
+}

app/src/pages/data/[id].tsx

@@ -18,6 +18,8 @@ import { api } from "~/utils/api";
 import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
 import DatasetEntriesTable from "~/components/datasets/DatasetEntriesTable";
 import { DatasetHeaderButtons } from "~/components/datasets/DatasetHeaderButtons/DatasetHeaderButtons";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
 export default function Dataset() {
   const router = useRouter();
@@ -55,15 +57,11 @@ export default function Dataset() {
   return (
     <AppShell title={dataset.data?.name}>
       <VStack h="full">
-        <Flex
-          pl={4}
-          pr={8}
-          py={2}
-          w="full"
-          direction={{ base: "column", sm: "row" }}
-          alignItems={{ base: "flex-start", sm: "center" }}
-        >
-          <Breadcrumb flex={1} mt={1}>
+        <PageHeaderContainer>
+          <Breadcrumb>
+            <BreadcrumbItem>
+              <ProjectBreadcrumbContents projectName={dataset.data?.project?.name} />
+            </BreadcrumbItem>
             <BreadcrumbItem>
               <Link href="/data">
                 <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
@@ -89,8 +87,8 @@ export default function Dataset() {
             </BreadcrumbItem>
           </Breadcrumb>
           <DatasetHeaderButtons />
-        </Flex>
-        <Box w="full" overflowX="auto" flex={1} pl={4} pr={8} pt={8} pb={16}>
+        </PageHeaderContainer>
+        <Box w="full" overflowX="auto" flex={1} px={8} pt={8} pb={16}>
           {datasetId && <DatasetEntriesTable />}
         </Box>
       </VStack>

app/src/pages/data/index.tsx

@@ -1,83 +1,49 @@
-import {
-  SimpleGrid,
-  Icon,
-  VStack,
-  Breadcrumb,
-  BreadcrumbItem,
-  Flex,
-  Center,
-  Text,
-  Link,
-  HStack,
-} from "@chakra-ui/react";
+import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
 import AppShell from "~/components/nav/AppShell";
-import { api } from "~/utils/api";
-import { signIn, useSession } from "next-auth/react";
 import { RiDatabase2Line } from "react-icons/ri";
 import {
   DatasetCard,
   DatasetCardSkeleton,
   NewDatasetCard,
 } from "~/components/datasets/DatasetCard";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import { useDatasets } from "~/utils/hooks";
 
 export default function DatasetsPage() {
-  const datasets = api.datasets.list.useQuery();
-
-  const user = useSession().data;
-  const authLoading = useSession().status === "loading";
-
-  if (user === null || authLoading) {
-    return (
-      <AppShell title="Data">
-        <Center h="100%">
-          {!authLoading && (
-            <Text>
-              <Link
-                onClick={() => {
-                  signIn("github").catch(console.error);
-                }}
-                textDecor="underline"
-              >
-                Sign in
-              </Link>{" "}
-              to view or create new datasets!
-            </Text>
-          )}
-        </Center>
-      </AppShell>
-    );
-  }
+  const datasets = useDatasets();
 
   return (
-    <AppShell title="Data">
-      <VStack alignItems={"flex-start"} px={4} py={2}>
-        <HStack minH={8} align="center" pt={2}>
-          <Breadcrumb flex={1}>
-            <BreadcrumbItem>
-              <Flex alignItems="center">
-                <Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
-              </Flex>
-            </BreadcrumbItem>
-          </Breadcrumb>
-        </HStack>
-        <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} p="4">
-          <NewDatasetCard />
-          {datasets.data && !datasets.isLoading ? (
-            datasets?.data?.map((dataset) => (
-              <DatasetCard
-                key={dataset.id}
-                dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
-              />
-            ))
-          ) : (
-            <>
-              <DatasetCardSkeleton />
-              <DatasetCardSkeleton />
-              <DatasetCardSkeleton />
-            </>
-          )}
-        </SimpleGrid>
-      </VStack>
+    <AppShell title="Data" requireAuth>
+      <PageHeaderContainer>
+        <Breadcrumb>
+          <BreadcrumbItem>
+            <ProjectBreadcrumbContents />
+          </BreadcrumbItem>
+          <BreadcrumbItem minH={8}>
+            <Flex alignItems="center">
+              <Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
+            </Flex>
+          </BreadcrumbItem>
+        </Breadcrumb>
+      </PageHeaderContainer>
+      <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py={4} px={8}>
+        <NewDatasetCard />
+        {datasets.data && !datasets.isLoading ? (
+          datasets?.data?.map((dataset) => (
+            <DatasetCard
+              key={dataset.id}
+              dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
+            />
+          ))
+        ) : (
+          <>
+            <DatasetCardSkeleton />
+            <DatasetCardSkeleton />
+            <DatasetCardSkeleton />
+          </>
+        )}
+      </SimpleGrid>
     </AppShell>
   );
 }

app/src/pages/experiments/[id].tsx

@@ -23,6 +23,8 @@ import { useAppStore } from "~/state/store";
 import { useSyncVariantEditor } from "~/state/sync";
 import { ExperimentHeaderButtons } from "~/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons";
 import Head from "next/head";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
 // TODO: import less to fix deployment with server side props
 // export const getServerSideProps = async (context: GetServerSidePropsContext<{ id: string }>) => {
@@ -60,7 +62,7 @@ export default function Experiment() {
 
   useEffect(() => {
     useAppStore.getState().sharedVariantEditor.loadMonaco().catch(console.error);
-  });
+  }, []);
 
   const [label, setLabel] = useState(experiment.data?.label || "");
   useEffect(() => {
@@ -104,14 +106,11 @@ export default function Experiment() {
       )}
       <AppShell title={experiment.data?.label}>
         <VStack h="full">
-          <Flex
-            px={4}
-            py={2}
-            w="full"
-            direction={{ base: "column", sm: "row" }}
-            alignItems={{ base: "flex-start", sm: "center" }}
-          >
-            <Breadcrumb flex={1}>
+          <PageHeaderContainer>
+            <Breadcrumb>
+              <BreadcrumbItem>
+                <ProjectBreadcrumbContents projectName={experiment.data?.project?.name} />
+              </BreadcrumbItem>
               <BreadcrumbItem>
                 <Link href="/experiments">
                   <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
@@ -143,7 +142,7 @@ export default function Experiment() {
               </BreadcrumbItem>
             </Breadcrumb>
             <ExperimentHeaderButtons />
-          </Flex>
+          </PageHeaderContainer>
           <ExperimentSettingsDrawer />
           <Box w="100%" overflowX="auto" flex={1}>
             <OutputsTable experimentId={router.query.id as string | undefined} />

app/src/pages/experiments/index.tsx

@@ -1,78 +1,44 @@
-import {
-  SimpleGrid,
-  Icon,
-  VStack,
-  Breadcrumb,
-  BreadcrumbItem,
-  Flex,
-  Center,
-  Text,
-  Link,
-  HStack,
-} from "@chakra-ui/react";
+import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
 import AppShell from "~/components/nav/AppShell";
-import { api } from "~/utils/api";
 import {
   ExperimentCard,
   ExperimentCardSkeleton,
   NewExperimentCard,
 } from "~/components/experiments/ExperimentCard";
-import { signIn, useSession } from "next-auth/react";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import { useExperiments } from "~/utils/hooks";
 
 export default function ExperimentsPage() {
-  const experiments = api.experiments.list.useQuery();
-
-  const user = useSession().data;
-  const authLoading = useSession().status === "loading";
-
-  if (user === null || authLoading) {
-    return (
-      <AppShell title="Experiments">
-        <Center h="100%">
-          {!authLoading && (
-            <Text>
-              <Link
-                onClick={() => {
-                  signIn("github").catch(console.error);
-                }}
-                textDecor="underline"
-              >
-                Sign in
-              </Link>{" "}
-              to view or create new experiments!
-            </Text>
-          )}
-        </Center>
-      </AppShell>
-    );
-  }
+  const experiments = useExperiments();
 
   return (
-    <AppShell title="Experiments">
-      <VStack alignItems={"flex-start"} px={4} py={2}>
-        <HStack minH={8} align="center" pt={2}>
-          <Breadcrumb flex={1}>
-            <BreadcrumbItem>
-              <Flex alignItems="center">
-                <Icon as={RiFlaskLine} boxSize={4} mr={2} /> Experiments
-              </Flex>
-            </BreadcrumbItem>
-          </Breadcrumb>
-        </HStack>
-        <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} p="4">
-          <NewExperimentCard />
-          {experiments.data && !experiments.isLoading ? (
-            experiments?.data?.map((exp) => <ExperimentCard key={exp.id} exp={exp} />)
-          ) : (
-            <>
-              <ExperimentCardSkeleton />
-              <ExperimentCardSkeleton />
-              <ExperimentCardSkeleton />
-            </>
-          )}
-        </SimpleGrid>
-      </VStack>
+    <AppShell title="Experiments" requireAuth>
+      <PageHeaderContainer>
+        <Breadcrumb>
+          <BreadcrumbItem>
+            <ProjectBreadcrumbContents />
+          </BreadcrumbItem>
+          <BreadcrumbItem minH={8}>
+            <Flex alignItems="center">
+              <Icon as={RiFlaskLine} boxSize={4} mr={2} /> Experiments
+            </Flex>
+          </BreadcrumbItem>
+        </Breadcrumb>
+      </PageHeaderContainer>
+      <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py="4" px={8}>
+        <NewExperimentCard />
+        {experiments.data && !experiments.isLoading ? (
+          experiments?.data?.map((exp) => <ExperimentCard key={exp.id} exp={exp} />)
+        ) : (
+          <>
+            <ExperimentCardSkeleton />
+            <ExperimentCardSkeleton />
+            <ExperimentCardSkeleton />
+          </>
+        )}
+      </SimpleGrid>
     </AppShell>
   );
 }

app/src/pages/project/settings/index.tsx

@@ -0,0 +1,161 @@
+import {
+  Breadcrumb,
+  BreadcrumbItem,
+  Text,
+  type TextProps,
+  VStack,
+  HStack,
+  Button,
+  Divider,
+  Icon,
+  useDisclosure,
+} from "@chakra-ui/react";
+import { useEffect, useState } from "react";
+import { BsTrash } from "react-icons/bs";
+
+import AppShell from "~/components/nav/AppShell";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import CopiableCode from "~/components/CopiableCode";
+import { DeleteProjectDialog } from "~/components/projectSettings/DeleteProjectDialog";
+import AutoResizeTextArea from "~/components/AutoResizeTextArea";
+
+export default function Settings() {
+  const utils = api.useContext();
+  const { data: selectedProject } = useSelectedProject();
+
+  const apiKey =
+    selectedProject?.apiKeys?.length && selectedProject?.apiKeys[0]
+      ? selectedProject?.apiKeys[0].apiKey
+      : "";
+
+  const updateMutation = api.projects.update.useMutation();
+  const [onSaveName] = useHandledAsyncCallback(async () => {
+    if (name && name !== selectedProject?.name && selectedProject?.id) {
+      await updateMutation.mutateAsync({
+        id: selectedProject.id,
+        updates: { name },
+      });
+      await Promise.all([
+        utils.projects.get.invalidate({ id: selectedProject.id }),
+        utils.projects.list.invalidate(),
+      ]);
+    }
+  }, [updateMutation, selectedProject]);
+
+  const [name, setName] = useState(selectedProject?.name);
+  useEffect(() => {
+    setName(selectedProject?.name);
+  }, [selectedProject?.name]);
+
+  const deleteProjectOpen = useDisclosure();
+
+  return (
+    <>
+      <AppShell>
+        <PageHeaderContainer>
+          <Breadcrumb>
+            <BreadcrumbItem>
+              <ProjectBreadcrumbContents />
+            </BreadcrumbItem>
+            <BreadcrumbItem isCurrentPage>
+              <Text>Project Settings</Text>
+            </BreadcrumbItem>
+          </Breadcrumb>
+        </PageHeaderContainer>
+        <VStack px={8} py={4} alignItems="flex-start" spacing={4}>
+          <VStack spacing={0} alignItems="flex-start">
+            <Text fontSize="2xl" fontWeight="bold">
+              Project Settings
+            </Text>
+            <Text fontSize="sm">
+              Configure your project settings. These settings only apply to {selectedProject?.name}.
+            </Text>
+          </VStack>
+          <VStack
+            w="full"
+            alignItems="flex-start"
+            borderWidth={1}
+            borderRadius={4}
+            borderColor="gray.300"
+            bgColor="white"
+            p={6}
+            spacing={6}
+          >
+            <VStack alignItems="flex-start" w="full">
+              <Text fontWeight="bold" fontSize="xl">
+                Display Name
+              </Text>
+              <AutoResizeTextArea
+                w="full"
+                maxW={600}
+                value={name}
+                onChange={(e) => setName(e.target.value)}
+                borderColor="gray.300"
+              />
+              <Button
+                isDisabled={!name || name === selectedProject?.name}
+                colorScheme="orange"
+                borderRadius={4}
+                mt={2}
+                _disabled={{
+                  opacity: 0.6,
+                }}
+                onClick={onSaveName}
+              >
+                Rename Project
+              </Button>
+            </VStack>
+            <Divider backgroundColor="gray.300" />
+            <VStack alignItems="flex-start">
+              <Subtitle>Project API Key</Subtitle>
+              <Text fontSize="sm">
+                Use your project API key to authenticate your requests when sending data to
+                OpenPipe. You can set this key in your environment variables, or use it directly in
+                your code.
+              </Text>
+            </VStack>
+            <CopiableCode code={apiKey} />
+            <Divider />
+            {selectedProject?.personalProjectUserId ? (
+              <VStack alignItems="flex-start">
+                <Subtitle>Personal Project</Subtitle>
+                <Text fontSize="sm">
+                  This project is {selectedProject?.personalProjectUser?.name}'s personal project.
+                  It cannot be deleted.
+                </Text>
+              </VStack>
+            ) : (
+              <VStack alignItems="flex-start">
+                <Subtitle color="red.600">Danger Zone</Subtitle>
+                <Text fontSize="sm">
+                  Permanently delete your project and all of its data. This action cannot be undone.
+                </Text>
+                <HStack
+                  as={Button}
+                  isDisabled={selectedProject?.role !== "ADMIN"}
+                  colorScheme="red"
+                  variant="outline"
+                  borderRadius={4}
+                  mt={2}
+                  height="auto"
+                  onClick={deleteProjectOpen.onOpen}
+                >
+                  <Icon as={BsTrash} />
+                  <Text overflowWrap="break-word" whiteSpace="normal" py={2}>
+                    Delete {selectedProject?.name}
+                  </Text>
+                </HStack>
+              </VStack>
+            )}
+          </VStack>
+        </VStack>
+      </AppShell>
+      <DeleteProjectDialog isOpen={deleteProjectOpen.isOpen} onClose={deleteProjectOpen.onClose} />
+    </>
+  );
+}
+
+const Subtitle = (props: TextProps) => <Text fontWeight="bold" fontSize="xl" {...props} />;

app/src/pages/request-logs/index.tsx

@@ -0,0 +1,34 @@
+import { Text, VStack, Divider, HStack } from "@chakra-ui/react";
+
+import AppShell from "~/components/nav/AppShell";
+import LoggedCallTable from "~/components/requestLogs/LoggedCallsTable";
+import LoggedCallsPaginator from "~/components/requestLogs/LoggedCallsPaginator";
+import ActionButton from "~/components/requestLogs/ActionButton";
+import { useAppStore } from "~/state/store";
+import { RiFlaskLine } from "react-icons/ri";
+
+export default function LoggedCalls() {
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  return (
+    <AppShell title="Request Logs" requireAuth>
+      <VStack px={8} py={8} alignItems="flex-start" spacing={4} w="full">
+        <Text fontSize="2xl" fontWeight="bold">
+          Request Logs
+        </Text>
+        <Divider />
+        <HStack w="full" justifyContent="flex-end">
+          <ActionButton
+            onClick={() => {
+              console.log("experimenting with these ids", selectedLogIds);
+            }}
+            label="Experiment"
+            icon={RiFlaskLine}
+            isDisabled={selectedLogIds.size === 0}
+          />
+        </HStack>
+        <LoggedCallTable />
+        <LoggedCallsPaginator />
+      </VStack>
+    </AppShell>
+  );
+}

app/src/pages/world-champs/signup.tsx

@@ -19,6 +19,8 @@ import {
   useInterval,
   Image,
   Flex,
+  Alert,
+  AlertIcon,
 } from "@chakra-ui/react";
 import { signIn, useSession } from "next-auth/react";
 import Head from "next/head";
@@ -188,12 +190,18 @@ export default function Signup() {
           <Heading size="lg" textAlign="center">
             🏆 Prompt Engineering World Championships
           </Heading>
-          <CountdownTimer
+          {/* <CountdownTimer
             date={new Date("2023-08-14T00:00:00Z")}
             fontSize="2xl"
             alignSelf="center"
             color="gray.500"
-          />
+          /> */}
+          <Alert status="warning" mt={4}>
+            <AlertIcon />
+            We've decided to pause the World Championships for the moment because our systems aren't
+            quite ready. You can still sign up if you're interested and we'll notify you once we
+            reschedule!
+          </Alert>
 
           <ApplicationStatus py={8} alignSelf="center" />
 

app/src/promptConstructor/format.ts

@@ -4,11 +4,15 @@ import parserTypescript from "prettier/plugins/typescript";
 // @ts-expect-error for some reason missing from types
 import parserEstree from "prettier/plugins/estree";
 
+// This emits a warning in the browser "Critical dependency: the request of a
+// dependency is an expression". Unfortunately doesn't seem to be a way to get
+// around it if we want to use Babel client-side for now. One solution would be
+// to just do the formatting server-side in a trpc call.
+// https://github.com/babel/babel/issues/14301
 import * as babel from "@babel/standalone";
 
 export function stripTypes(tsCode: string): string {
   const options = {
-    presets: ["typescript"],
     filename: "file.ts",
   };
 

app/src/server/api/root.router.ts

@@ -3,11 +3,15 @@ import { createTRPCRouter } from "~/server/api/trpc";
 import { experimentsRouter } from "./routers/experiments.router";
 import { scenariosRouter } from "./routers/scenarios.router";
 import { scenarioVariantCellsRouter } from "./routers/scenarioVariantCells.router";
-import { templateVarsRouter } from "./routers/templateVariables.router";
+import { scenarioVarsRouter } from "./routers/scenarioVariables.router";
 import { evaluationsRouter } from "./routers/evaluations.router";
 import { worldChampsRouter } from "./routers/worldChamps.router";
 import { datasetsRouter } from "./routers/datasets.router";
 import { datasetEntries } from "./routers/datasetEntries.router";
+import { externalApiRouter } from "./routers/externalApi.router";
+import { projectsRouter } from "./routers/projects.router";
+import { dashboardRouter } from "./routers/dashboard.router";
+import { loggedCallsRouter } from "./routers/loggedCalls.router";
 
 /**
  * This is the primary router for your server.
@@ -19,11 +23,15 @@ export const appRouter = createTRPCRouter({
   experiments: experimentsRouter,
   scenarios: scenariosRouter,
   scenarioVariantCells: scenarioVariantCellsRouter,
-  templateVars: templateVarsRouter,
+  scenarioVars: scenarioVarsRouter,
   evaluations: evaluationsRouter,
   worldChamps: worldChampsRouter,
   datasets: datasetsRouter,
   datasetEntries: datasetEntries,
+  projects: projectsRouter,
+  dashboard: dashboardRouter,
+  loggedCalls: loggedCallsRouter,
+  externalApi: externalApiRouter,
 });
 
 // export type definition of API

app/src/server/api/routers/dashboard.router.ts

@@ -0,0 +1,108 @@
+import { sql } from "kysely";
+import { z } from "zod";
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { kysely } from "~/server/db";
+import { requireCanViewProject } from "~/utils/accessControl";
+import dayjs from "~/utils/dayjs";
+
+export const dashboardRouter = createTRPCRouter({
+  stats: protectedProcedure
+    .input(
+      z.object({
+        // TODO: actually take startDate into account
+        startDate: z.string().optional(),
+        projectId: z.string(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      await requireCanViewProject(input.projectId, ctx);
+      // Return the stats group by hour
+      const periods = await kysely
+        .selectFrom("LoggedCall")
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .where("projectId", "=", input.projectId)
+        .select(({ fn }) => [
+          sql<Date>`date_trunc('day', "LoggedCallModelResponse"."requestedAt")`.as("period"),
+          sql<number>`count("LoggedCall"."id")::int`.as("numQueries"),
+          fn.sum(fn.coalesce("LoggedCallModelResponse.cost", sql<number>`0`)).as("cost"),
+        ])
+        .groupBy("period")
+        .orderBy("period")
+        .execute();
+
+      let originalDataIndex = periods.length - 1;
+      // *SLAMS DOWN GLASS OF WHISKEY* timezones, amirite?
+      let dayToMatch = dayjs(input.startDate || new Date());
+      // Ensure that the initial date we're matching against is never before the first period
+      if (
+        periods[originalDataIndex] &&
+        dayToMatch.isBefore(periods[originalDataIndex]?.period, "day")
+      ) {
+        dayToMatch = dayjs(periods[originalDataIndex]?.period);
+      }
+      const backfilledPeriods: typeof periods = [];
+
+      // Backfill from now to 14 days ago or the date of the first logged call, whichever is earlier
+      while (
+        backfilledPeriods.length < 14 ||
+        (periods[0]?.period && !dayToMatch.isBefore(periods[0]?.period, "day"))
+      ) {
+        const nextOriginalPeriod = periods[originalDataIndex];
+        if (nextOriginalPeriod && dayjs(nextOriginalPeriod?.period).isSame(dayToMatch, "day")) {
+          backfilledPeriods.unshift(nextOriginalPeriod);
+          originalDataIndex--;
+        } else {
+          backfilledPeriods.unshift({
+            period: dayjs(dayToMatch).toDate(),
+            numQueries: 0,
+            cost: 0,
+          });
+        }
+        dayToMatch = dayToMatch.subtract(1, "day");
+      }
+
+      const totals = await kysely
+        .selectFrom("LoggedCall")
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .where("projectId", "=", input.projectId)
+        .select(({ fn }) => [
+          fn.sum(fn.coalesce("LoggedCallModelResponse.cost", sql<number>`0`)).as("cost"),
+          fn.count("LoggedCall.id").as("numQueries"),
+        ])
+        .executeTakeFirst();
+
+      const errors = await kysely
+        .selectFrom("LoggedCall")
+        .where("projectId", "=", input.projectId)
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .select(({ fn }) => [fn.count("LoggedCall.id").as("count"), "statusCode as code"])
+        .where("statusCode", ">", 200)
+        .groupBy("code")
+        .orderBy("count", "desc")
+        .execute();
+
+      const namedErrors = errors.map((e) => {
+        if (e.code === 429) {
+          return { ...e, name: "Rate limited" };
+        } else if (e.code === 500) {
+          return { ...e, name: "Internal server error" };
+        } else {
+          return { ...e, name: "Other" };
+        }
+      });
+
+      return { periods: backfilledPeriods, totals, errors: namedErrors };
+    }),
+});

app/src/server/api/routers/datasetEntries.router.ts

@@ -4,23 +4,21 @@ import { prisma } from "~/server/db";
 import { requireCanModifyDataset, requireCanViewDataset } from "~/utils/accessControl";
 import { autogenerateDatasetEntries } from "../autogenerate/autogenerateDatasetEntries";
 
-const PAGE_SIZE = 10;
-
 export const datasetEntries = createTRPCRouter({
   list: protectedProcedure
-    .input(z.object({ datasetId: z.string(), page: z.number() }))
+    .input(z.object({ datasetId: z.string(), page: z.number(), pageSize: z.number() }))
     .query(async ({ input, ctx }) => {
       await requireCanViewDataset(input.datasetId, ctx);
 
-      const { datasetId, page } = input;
+      const { datasetId, page, pageSize } = input;
 
       const entries = await prisma.datasetEntry.findMany({
         where: {
           datasetId,
         },
         orderBy: { createdAt: "desc" },
-        skip: (page - 1) * PAGE_SIZE,
-        take: PAGE_SIZE,
+        skip: (page - 1) * pageSize,
+        take: pageSize,
       });
 
       const count = await prisma.datasetEntry.count({
@@ -31,8 +29,6 @@ export const datasetEntries = createTRPCRouter({
 
       return {
         entries,
-        startIndex: (page - 1) * PAGE_SIZE + 1,
-        lastPage: Math.ceil(count / PAGE_SIZE),
         count,
       };
     }),

app/src/server/api/routers/datasets.router.ts

@@ -3,65 +3,62 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
 import { prisma } from "~/server/db";
 import {
   requireCanModifyDataset,
+  requireCanModifyProject,
   requireCanViewDataset,
-  requireNothing,
+  requireCanViewProject,
 } from "~/utils/accessControl";
-import userOrg from "~/server/utils/userOrg";
 
 export const datasetsRouter = createTRPCRouter({
-  list: protectedProcedure.query(async ({ ctx }) => {
-    // Anyone can list experiments
-    requireNothing(ctx);
+  list: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewProject(input.projectId, ctx);
 
-    const datasets = await prisma.dataset.findMany({
-      where: {
-        organization: {
-          organizationUsers: {
-            some: { userId: ctx.session.user.id },
+      const datasets = await prisma.dataset.findMany({
+        where: {
+          projectId: input.projectId,
+        },
+        orderBy: {
+          createdAt: "desc",
+        },
+        include: {
+          _count: {
+            select: { datasetEntries: true },
           },
         },
-      },
-      orderBy: {
-        createdAt: "desc",
-      },
-      include: {
-        _count: {
-          select: { datasetEntries: true },
-        },
-      },
-    });
+      });
 
-    return datasets;
-  }),
+      return datasets;
+    }),
 
   get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
     await requireCanViewDataset(input.id, ctx);
     return await prisma.dataset.findFirstOrThrow({
       where: { id: input.id },
+      include: {
+        project: true,
+      },
     });
   }),
 
-  create: protectedProcedure.input(z.object({})).mutation(async ({ ctx }) => {
-    // Anyone can create an experiment
-    requireNothing(ctx);
+  create: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyProject(input.projectId, ctx);
 
-    const numDatasets = await prisma.dataset.count({
-      where: {
-        organization: {
-          organizationUsers: {
-            some: { userId: ctx.session.user.id },
-          },
+      const numDatasets = await prisma.dataset.count({
+        where: {
+          projectId: input.projectId,
         },
-      },
-    });
+      });
 
-    return await prisma.dataset.create({
-      data: {
-        name: `Dataset ${numDatasets + 1}`,
-        organizationId: (await userOrg(ctx.session.user.id)).id,
-      },
-    });
-  }),
+      return await prisma.dataset.create({
+        data: {
+          name: `Dataset ${numDatasets + 1}`,
+          projectId: input.projectId,
+        },
+      });
+    }),
 
   update: protectedProcedure
     .input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))

app/src/server/api/routers/experiments.router.ts

@@ -8,10 +8,10 @@ import { generateNewCell } from "~/server/utils/generateNewCell";
 import {
   canModifyExperiment,
   requireCanModifyExperiment,
+  requireCanModifyProject,
   requireCanViewExperiment,
-  requireNothing,
+  requireCanViewProject,
 } from "~/utils/accessControl";
-import userOrg from "~/server/utils/userOrg";
 import generateTypes from "~/modelProviders/generateTypes";
 import { promptConstructorVersion } from "~/promptConstructor/version";
 
@@ -43,55 +43,55 @@ export const experimentsRouter = createTRPCRouter({
       testScenarioCount,
     };
   }),
-  list: protectedProcedure.query(async ({ ctx }) => {
-    // Anyone can list experiments
-    requireNothing(ctx);
+  list: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewProject(input.projectId, ctx);
 
-    const experiments = await prisma.experiment.findMany({
-      where: {
-        organization: {
-          organizationUsers: {
-            some: { userId: ctx.session.user.id },
-          },
+      const experiments = await prisma.experiment.findMany({
+        where: {
+          projectId: input.projectId,
         },
-      },
-      orderBy: {
-        sortIndex: "desc",
-      },
-    });
+        orderBy: {
+          sortIndex: "desc",
+        },
+      });
 
-    // TODO: look for cleaner way to do this. Maybe aggregate?
-    const experimentsWithCounts = await Promise.all(
-      experiments.map(async (experiment) => {
-        const visibleTestScenarioCount = await prisma.testScenario.count({
-          where: {
-            experimentId: experiment.id,
-            visible: true,
-          },
-        });
+      // TODO: look for cleaner way to do this. Maybe aggregate?
+      const experimentsWithCounts = await Promise.all(
+        experiments.map(async (experiment) => {
+          const visibleTestScenarioCount = await prisma.testScenario.count({
+            where: {
+              experimentId: experiment.id,
+              visible: true,
+            },
+          });
 
-        const visiblePromptVariantCount = await prisma.promptVariant.count({
-          where: {
-            experimentId: experiment.id,
-            visible: true,
-          },
-        });
+          const visiblePromptVariantCount = await prisma.promptVariant.count({
+            where: {
+              experimentId: experiment.id,
+              visible: true,
+            },
+          });
 
-        return {
-          ...experiment,
-          testScenarioCount: visibleTestScenarioCount,
-          promptVariantCount: visiblePromptVariantCount,
-        };
-      }),
-    );
+          return {
+            ...experiment,
+            testScenarioCount: visibleTestScenarioCount,
+            promptVariantCount: visiblePromptVariantCount,
+          };
+        }),
+      );
 
-    return experimentsWithCounts;
-  }),
+      return experimentsWithCounts;
+    }),
 
   get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
     await requireCanViewExperiment(input.id, ctx);
     const experiment = await prisma.experiment.findFirstOrThrow({
       where: { id: input.id },
+      include: {
+        project: true,
+      },
     });
 
     const canModify = ctx.session?.user.id
@@ -107,222 +107,224 @@ export const experimentsRouter = createTRPCRouter({
     };
   }),
 
-  fork: protectedProcedure.input(z.object({ id: z.string() })).mutation(async ({ input, ctx }) => {
-    await requireCanViewExperiment(input.id, ctx);
+  fork: protectedProcedure
+    .input(z.object({ id: z.string(), projectId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanViewExperiment(input.id, ctx);
+      await requireCanModifyProject(input.projectId, ctx);
 
-    const [
-      existingExp,
-      existingVariants,
-      existingScenarios,
-      existingCells,
-      evaluations,
-      templateVariables,
-    ] = await prisma.$transaction([
-      prisma.experiment.findUniqueOrThrow({
-        where: {
-          id: input.id,
-        },
-      }),
-      prisma.promptVariant.findMany({
-        where: {
-          experimentId: input.id,
-          visible: true,
-        },
-      }),
-      prisma.testScenario.findMany({
-        where: {
-          experimentId: input.id,
-          visible: true,
-        },
-      }),
-      prisma.scenarioVariantCell.findMany({
-        where: {
-          testScenario: {
-            visible: true,
+      const [
+        existingExp,
+        existingVariants,
+        existingScenarios,
+        existingCells,
+        evaluations,
+        templateVariables,
+      ] = await prisma.$transaction([
+        prisma.experiment.findUniqueOrThrow({
+          where: {
+            id: input.id,
           },
-          promptVariant: {
+        }),
+        prisma.promptVariant.findMany({
+          where: {
             experimentId: input.id,
             visible: true,
           },
-        },
-        include: {
-          modelResponses: {
-            include: {
-              outputEvaluations: true,
+        }),
+        prisma.testScenario.findMany({
+          where: {
+            experimentId: input.id,
+            visible: true,
+          },
+        }),
+        prisma.scenarioVariantCell.findMany({
+          where: {
+            testScenario: {
+              visible: true,
+            },
+            promptVariant: {
+              experimentId: input.id,
+              visible: true,
             },
           },
-        },
-      }),
-      prisma.evaluation.findMany({
-        where: {
-          experimentId: input.id,
-        },
-      }),
-      prisma.templateVariable.findMany({
-        where: {
-          experimentId: input.id,
-        },
-      }),
-    ]);
+          include: {
+            modelResponses: {
+              include: {
+                outputEvaluations: true,
+              },
+            },
+          },
+        }),
+        prisma.evaluation.findMany({
+          where: {
+            experimentId: input.id,
+          },
+        }),
+        prisma.templateVariable.findMany({
+          where: {
+            experimentId: input.id,
+          },
+        }),
+      ]);
 
-    const newExperimentId = uuidv4();
+      const newExperimentId = uuidv4();
 
-    const existingToNewVariantIds = new Map<string, string>();
-    const variantsToCreate: Prisma.PromptVariantCreateManyInput[] = [];
-    for (const variant of existingVariants) {
-      const newVariantId = uuidv4();
-      existingToNewVariantIds.set(variant.id, newVariantId);
-      variantsToCreate.push({
-        ...variant,
-        id: newVariantId,
-        experimentId: newExperimentId,
-      });
-    }
-
-    const existingToNewScenarioIds = new Map<string, string>();
-    const scenariosToCreate: Prisma.TestScenarioCreateManyInput[] = [];
-    for (const scenario of existingScenarios) {
-      const newScenarioId = uuidv4();
-      existingToNewScenarioIds.set(scenario.id, newScenarioId);
-      scenariosToCreate.push({
-        ...scenario,
-        id: newScenarioId,
-        experimentId: newExperimentId,
-        variableValues: scenario.variableValues as Prisma.InputJsonValue,
-      });
-    }
-
-    const existingToNewEvaluationIds = new Map<string, string>();
-    const evaluationsToCreate: Prisma.EvaluationCreateManyInput[] = [];
-    for (const evaluation of evaluations) {
-      const newEvaluationId = uuidv4();
-      existingToNewEvaluationIds.set(evaluation.id, newEvaluationId);
-      evaluationsToCreate.push({
-        ...evaluation,
-        id: newEvaluationId,
-        experimentId: newExperimentId,
-      });
-    }
-
-    const cellsToCreate: Prisma.ScenarioVariantCellCreateManyInput[] = [];
-    const modelResponsesToCreate: Prisma.ModelResponseCreateManyInput[] = [];
-    const outputEvaluationsToCreate: Prisma.OutputEvaluationCreateManyInput[] = [];
-    for (const cell of existingCells) {
-      const newCellId = uuidv4();
-      const { modelResponses, ...cellData } = cell;
-      cellsToCreate.push({
-        ...cellData,
-        id: newCellId,
-        promptVariantId: existingToNewVariantIds.get(cell.promptVariantId) ?? "",
-        testScenarioId: existingToNewScenarioIds.get(cell.testScenarioId) ?? "",
-        prompt: (cell.prompt as Prisma.InputJsonValue) ?? undefined,
-      });
-      for (const modelResponse of modelResponses) {
-        const newModelResponseId = uuidv4();
-        const { outputEvaluations, ...modelResponseData } = modelResponse;
-        modelResponsesToCreate.push({
-          ...modelResponseData,
-          id: newModelResponseId,
-          scenarioVariantCellId: newCellId,
-          output: (modelResponse.output as Prisma.InputJsonValue) ?? undefined,
+      const existingToNewVariantIds = new Map<string, string>();
+      const variantsToCreate: Prisma.PromptVariantCreateManyInput[] = [];
+      for (const variant of existingVariants) {
+        const newVariantId = uuidv4();
+        existingToNewVariantIds.set(variant.id, newVariantId);
+        variantsToCreate.push({
+          ...variant,
+          id: newVariantId,
+          experimentId: newExperimentId,
         });
-        for (const evaluation of outputEvaluations) {
-          outputEvaluationsToCreate.push({
-            ...evaluation,
-            id: uuidv4(),
-            modelResponseId: newModelResponseId,
-            evaluationId: existingToNewEvaluationIds.get(evaluation.evaluationId) ?? "",
+      }
+
+      const existingToNewScenarioIds = new Map<string, string>();
+      const scenariosToCreate: Prisma.TestScenarioCreateManyInput[] = [];
+      for (const scenario of existingScenarios) {
+        const newScenarioId = uuidv4();
+        existingToNewScenarioIds.set(scenario.id, newScenarioId);
+        scenariosToCreate.push({
+          ...scenario,
+          id: newScenarioId,
+          experimentId: newExperimentId,
+          variableValues: scenario.variableValues as Prisma.InputJsonValue,
+        });
+      }
+
+      const existingToNewEvaluationIds = new Map<string, string>();
+      const evaluationsToCreate: Prisma.EvaluationCreateManyInput[] = [];
+      for (const evaluation of evaluations) {
+        const newEvaluationId = uuidv4();
+        existingToNewEvaluationIds.set(evaluation.id, newEvaluationId);
+        evaluationsToCreate.push({
+          ...evaluation,
+          id: newEvaluationId,
+          experimentId: newExperimentId,
+        });
+      }
+
+      const cellsToCreate: Prisma.ScenarioVariantCellCreateManyInput[] = [];
+      const modelResponsesToCreate: Prisma.ModelResponseCreateManyInput[] = [];
+      const outputEvaluationsToCreate: Prisma.OutputEvaluationCreateManyInput[] = [];
+      for (const cell of existingCells) {
+        const newCellId = uuidv4();
+        const { modelResponses, ...cellData } = cell;
+        cellsToCreate.push({
+          ...cellData,
+          id: newCellId,
+          promptVariantId: existingToNewVariantIds.get(cell.promptVariantId) ?? "",
+          testScenarioId: existingToNewScenarioIds.get(cell.testScenarioId) ?? "",
+          prompt: (cell.prompt as Prisma.InputJsonValue) ?? undefined,
+        });
+        for (const modelResponse of modelResponses) {
+          const newModelResponseId = uuidv4();
+          const { outputEvaluations, ...modelResponseData } = modelResponse;
+          modelResponsesToCreate.push({
+            ...modelResponseData,
+            id: newModelResponseId,
+            scenarioVariantCellId: newCellId,
+            respPayload: (modelResponse.respPayload as Prisma.InputJsonValue) ?? undefined,
           });
+          for (const evaluation of outputEvaluations) {
+            outputEvaluationsToCreate.push({
+              ...evaluation,
+              id: uuidv4(),
+              modelResponseId: newModelResponseId,
+              evaluationId: existingToNewEvaluationIds.get(evaluation.evaluationId) ?? "",
+            });
+          }
         }
       }
-    }
 
-    const templateVariablesToCreate: Prisma.TemplateVariableCreateManyInput[] = [];
-    for (const templateVariable of templateVariables) {
-      templateVariablesToCreate.push({
-        ...templateVariable,
-        id: uuidv4(),
-        experimentId: newExperimentId,
-      });
-    }
+      const templateVariablesToCreate: Prisma.TemplateVariableCreateManyInput[] = [];
+      for (const templateVariable of templateVariables) {
+        templateVariablesToCreate.push({
+          ...templateVariable,
+          id: uuidv4(),
+          experimentId: newExperimentId,
+        });
+      }
 
-    const maxSortIndex =
-      (
-        await prisma.experiment.aggregate({
-          _max: {
-            sortIndex: true,
+      const maxSortIndex =
+        (
+          await prisma.experiment.aggregate({
+            _max: {
+              sortIndex: true,
+            },
+          })
+        )._max?.sortIndex ?? 0;
+
+      await prisma.$transaction([
+        prisma.experiment.create({
+          data: {
+            id: newExperimentId,
+            sortIndex: maxSortIndex + 1,
+            label: `${existingExp.label} (forked)`,
+            projectId: input.projectId,
           },
-        })
-      )._max?.sortIndex ?? 0;
+        }),
+        prisma.promptVariant.createMany({
+          data: variantsToCreate,
+        }),
+        prisma.testScenario.createMany({
+          data: scenariosToCreate,
+        }),
+        prisma.scenarioVariantCell.createMany({
+          data: cellsToCreate,
+        }),
+        prisma.modelResponse.createMany({
+          data: modelResponsesToCreate,
+        }),
+        prisma.evaluation.createMany({
+          data: evaluationsToCreate,
+        }),
+        prisma.outputEvaluation.createMany({
+          data: outputEvaluationsToCreate,
+        }),
+        prisma.templateVariable.createMany({
+          data: templateVariablesToCreate,
+        }),
+      ]);
 
-    await prisma.$transaction([
-      prisma.experiment.create({
+      return newExperimentId;
+    }),
+
+  create: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyProject(input.projectId, ctx);
+
+      const maxSortIndex =
+        (
+          await prisma.experiment.aggregate({
+            _max: {
+              sortIndex: true,
+            },
+            where: { projectId: input.projectId },
+          })
+        )._max?.sortIndex ?? 0;
+
+      const exp = await prisma.experiment.create({
         data: {
-          id: newExperimentId,
           sortIndex: maxSortIndex + 1,
-          label: `${existingExp.label} (forked)`,
-          organizationId: (await userOrg(ctx.session.user.id)).id,
+          label: `Experiment ${maxSortIndex + 1}`,
+          projectId: input.projectId,
         },
-      }),
-      prisma.promptVariant.createMany({
-        data: variantsToCreate,
-      }),
-      prisma.testScenario.createMany({
-        data: scenariosToCreate,
-      }),
-      prisma.scenarioVariantCell.createMany({
-        data: cellsToCreate,
-      }),
-      prisma.modelResponse.createMany({
-        data: modelResponsesToCreate,
-      }),
-      prisma.evaluation.createMany({
-        data: evaluationsToCreate,
-      }),
-      prisma.outputEvaluation.createMany({
-        data: outputEvaluationsToCreate,
-      }),
-      prisma.templateVariable.createMany({
-        data: templateVariablesToCreate,
-      }),
-    ]);
+      });
 
-    return newExperimentId;
-  }),
-
-  create: protectedProcedure.input(z.object({})).mutation(async ({ ctx }) => {
-    // Anyone can create an experiment
-    requireNothing(ctx);
-
-    const organizationId = (await userOrg(ctx.session.user.id)).id;
-
-    const maxSortIndex =
-      (
-        await prisma.experiment.aggregate({
-          _max: {
-            sortIndex: true,
-          },
-          where: { organizationId },
-        })
-      )._max?.sortIndex ?? 0;
-
-    const exp = await prisma.experiment.create({
-      data: {
-        sortIndex: maxSortIndex + 1,
-        label: `Experiment ${maxSortIndex + 1}`,
-        organizationId,
-      },
-    });
-
-    const [variant, _, scenario1, scenario2, scenario3] = await prisma.$transaction([
-      prisma.promptVariant.create({
-        data: {
-          experimentId: exp.id,
-          label: "Prompt Variant 1",
-          sortIndex: 0,
-          // The interpolated $ is necessary until dedent incorporates
-          // https://github.com/dmnd/dedent/pull/46
-          promptConstructor: dedent`
+      const [variant, _, scenario1, scenario2, scenario3] = await prisma.$transaction([
+        prisma.promptVariant.create({
+          data: {
+            experimentId: exp.id,
+            label: "Prompt Variant 1",
+            sortIndex: 0,
+            // The interpolated $ is necessary until dedent incorporates
+            // https://github.com/dmnd/dedent/pull/46
+            promptConstructor: dedent`
           /**
            * Use Javascript to define an OpenAI chat completion
            * (https://platform.openai.com/docs/api-reference/chat/create).
@@ -341,49 +343,49 @@ export const experimentsRouter = createTRPCRouter({
               },
             ],
           });`,
-          model: "gpt-3.5-turbo-0613",
-          modelProvider: "openai/ChatCompletion",
-          promptConstructorVersion,
-        },
-      }),
-      prisma.templateVariable.create({
-        data: {
-          experimentId: exp.id,
-          label: "language",
-        },
-      }),
-      prisma.testScenario.create({
-        data: {
-          experimentId: exp.id,
-          variableValues: {
-            language: "English",
+            model: "gpt-3.5-turbo-0613",
+            modelProvider: "openai/ChatCompletion",
+            promptConstructorVersion,
           },
-        },
-      }),
-      prisma.testScenario.create({
-        data: {
-          experimentId: exp.id,
-          variableValues: {
-            language: "Spanish",
+        }),
+        prisma.templateVariable.create({
+          data: {
+            experimentId: exp.id,
+            label: "language",
           },
-        },
-      }),
-      prisma.testScenario.create({
-        data: {
-          experimentId: exp.id,
-          variableValues: {
-            language: "German",
+        }),
+        prisma.testScenario.create({
+          data: {
+            experimentId: exp.id,
+            variableValues: {
+              language: "English",
+            },
           },
-        },
-      }),
-    ]);
+        }),
+        prisma.testScenario.create({
+          data: {
+            experimentId: exp.id,
+            variableValues: {
+              language: "Spanish",
+            },
+          },
+        }),
+        prisma.testScenario.create({
+          data: {
+            experimentId: exp.id,
+            variableValues: {
+              language: "German",
+            },
+          },
+        }),
+      ]);
 
-    await generateNewCell(variant.id, scenario1.id);
-    await generateNewCell(variant.id, scenario2.id);
-    await generateNewCell(variant.id, scenario3.id);
+      await generateNewCell(variant.id, scenario1.id);
+      await generateNewCell(variant.id, scenario2.id);
+      await generateNewCell(variant.id, scenario3.id);
 
-    return exp;
-  }),
+      return exp;
+    }),
 
   update: protectedProcedure
     .input(z.object({ id: z.string(), updates: z.object({ label: z.string() }) }))

app/src/server/api/routers/externalApi.router.ts

@@ -0,0 +1,198 @@
+import { type Prisma } from "@prisma/client";
+import { type JsonValue } from "type-fest";
+import { z } from "zod";
+import { v4 as uuidv4 } from "uuid";
+import { TRPCError } from "@trpc/server";
+
+import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { hashRequest } from "~/server/utils/hashObject";
+import modelProvider from "~/modelProviders/openai-ChatCompletion";
+import {
+  type ChatCompletion,
+  type CompletionCreateParams,
+} from "openai/resources/chat/completions";
+
+const reqValidator = z.object({
+  model: z.string(),
+  messages: z.array(z.any()),
+});
+
+const respValidator = z.object({
+  id: z.string(),
+  model: z.string(),
+  choices: z.array(
+    z.object({
+      finish_reason: z.string(),
+    }),
+  ),
+});
+
+export const externalApiRouter = createTRPCRouter({
+  checkCache: publicProcedure
+    .meta({
+      openapi: {
+        method: "POST",
+        path: "/v1/check-cache",
+        description: "Check if a prompt is cached",
+        protect: true,
+      },
+    })
+    .input(
+      z.object({
+        requestedAt: z.number().describe("Unix timestamp in milliseconds"),
+        reqPayload: z.unknown().describe("JSON-encoded request payload"),
+        tags: z
+          .record(z.string())
+          .optional()
+          .describe(
+            'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
+          ),
+      }),
+    )
+    .output(
+      z.object({
+        respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const apiKey = ctx.apiKey;
+      if (!apiKey) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const key = await prisma.apiKey.findUnique({
+        where: { apiKey },
+      });
+      if (!key) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const reqPayload = await reqValidator.spa(input.reqPayload);
+      const cacheKey = hashRequest(key.projectId, reqPayload as JsonValue);
+
+      const existingResponse = await prisma.loggedCallModelResponse.findFirst({
+        where: { cacheKey },
+        include: { originalLoggedCall: true },
+        orderBy: { requestedAt: "desc" },
+      });
+
+      if (!existingResponse) return { respPayload: null };
+
+      await prisma.loggedCall.create({
+        data: {
+          projectId: key.projectId,
+          requestedAt: new Date(input.requestedAt),
+          cacheHit: true,
+          modelResponseId: existingResponse.id,
+        },
+      });
+
+      return {
+        respPayload: existingResponse.respPayload,
+      };
+    }),
+
+  report: publicProcedure
+    .meta({
+      openapi: {
+        method: "POST",
+        path: "/v1/report",
+        description: "Report an API call",
+        protect: true,
+      },
+    })
+    .input(
+      z.object({
+        requestedAt: z.number().describe("Unix timestamp in milliseconds"),
+        receivedAt: z.number().describe("Unix timestamp in milliseconds"),
+        reqPayload: z.unknown().describe("JSON-encoded request payload"),
+        respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
+        statusCode: z.number().optional().describe("HTTP status code of response"),
+        errorMessage: z.string().optional().describe("User-friendly error message"),
+        tags: z
+          .record(z.string())
+          .optional()
+          .describe(
+            'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
+          ),
+      }),
+    )
+    .output(z.void())
+    .mutation(async ({ input, ctx }) => {
+      console.log("GOT TAGS", input.tags);
+      const apiKey = ctx.apiKey;
+      if (!apiKey) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const key = await prisma.apiKey.findUnique({
+        where: { apiKey },
+      });
+      if (!key) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const reqPayload = await reqValidator.spa(input.reqPayload);
+      const respPayload = await respValidator.spa(input.respPayload);
+
+      const requestHash = hashRequest(key.projectId, reqPayload as JsonValue);
+
+      const newLoggedCallId = uuidv4();
+      const newModelResponseId = uuidv4();
+
+      let usage;
+      let model;
+      if (reqPayload.success && respPayload.success) {
+        usage = modelProvider.getUsage(
+          input.reqPayload as CompletionCreateParams,
+          input.respPayload as ChatCompletion,
+        );
+        model = reqPayload.data.model;
+      }
+
+      await prisma.$transaction([
+        prisma.loggedCall.create({
+          data: {
+            id: newLoggedCallId,
+            projectId: key.projectId,
+            requestedAt: new Date(input.requestedAt),
+            cacheHit: false,
+            model,
+          },
+        }),
+        prisma.loggedCallModelResponse.create({
+          data: {
+            id: newModelResponseId,
+            originalLoggedCallId: newLoggedCallId,
+            requestedAt: new Date(input.requestedAt),
+            receivedAt: new Date(input.receivedAt),
+            reqPayload: input.reqPayload as Prisma.InputJsonValue,
+            respPayload: input.respPayload as Prisma.InputJsonValue,
+            statusCode: input.statusCode,
+            errorMessage: input.errorMessage,
+            durationMs: input.receivedAt - input.requestedAt,
+            cacheKey: respPayload.success ? requestHash : null,
+            inputTokens: usage?.inputTokens,
+            outputTokens: usage?.outputTokens,
+            cost: usage?.cost,
+          },
+        }),
+        // Avoid foreign key constraint error by updating the logged call after the model response is created
+        prisma.loggedCall.update({
+          where: {
+            id: newLoggedCallId,
+          },
+          data: {
+            modelResponseId: newModelResponseId,
+          },
+        }),
+      ]);
+
+      const tagsToCreate = Object.entries(input.tags ?? {}).map(([name, value]) => ({
+        loggedCallId: newLoggedCallId,
+        // sanitize tags
+        name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
+        value,
+      }));
+      await prisma.loggedCallTag.createMany({
+        data: tagsToCreate,
+      });
+    }),
+});

app/src/server/api/routers/loggedCalls.router.ts

@@ -0,0 +1,33 @@
+import { z } from "zod";
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { requireCanViewProject } from "~/utils/accessControl";
+
+export const loggedCallsRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(z.object({ projectId: z.string(), page: z.number(), pageSize: z.number() }))
+    .query(async ({ input, ctx }) => {
+      const { projectId, page, pageSize } = input;
+
+      await requireCanViewProject(projectId, ctx);
+
+      const calls = await prisma.loggedCall.findMany({
+        where: { projectId },
+        orderBy: { requestedAt: "desc" },
+        include: { tags: true, modelResponse: true },
+        skip: (page - 1) * pageSize,
+        take: pageSize,
+      });
+
+      const matchingLogs = await prisma.loggedCall.findMany({
+        where: { projectId },
+        select: { id: true },
+      });
+
+      const count = await prisma.loggedCall.count({
+        where: { projectId },
+      });
+
+      return { calls, count, matchingLogIds: matchingLogs.map((log) => log.id) };
+    }),
+});

app/src/server/api/routers/projects.router.ts

@@ -0,0 +1,128 @@
+import { TRPCError } from "@trpc/server";
+import { v4 as uuidv4 } from "uuid";
+import { z } from "zod";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { generateApiKey } from "~/server/utils/generateApiKey";
+import userProject from "~/server/utils/userProject";
+import {
+  requireCanModifyProject,
+  requireCanViewProject,
+  requireIsProjectAdmin,
+  requireNothing,
+} from "~/utils/accessControl";
+
+export const projectsRouter = createTRPCRouter({
+  list: protectedProcedure.query(async ({ ctx }) => {
+    const userId = ctx.session.user.id;
+    requireNothing(ctx);
+
+    if (!userId) {
+      return null;
+    }
+
+    const projects = await prisma.project.findMany({
+      where: {
+        projectUsers: {
+          some: { userId: ctx.session.user.id },
+        },
+      },
+      orderBy: {
+        createdAt: "asc",
+      },
+    });
+
+    if (!projects.length) {
+      // TODO: We should move this to a separate endpoint that is called on sign up
+      const personalProject = await userProject(userId);
+      projects.push(personalProject);
+    }
+
+    return projects;
+  }),
+  get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
+    await requireCanViewProject(input.id, ctx);
+    const [proj, userRole] = await prisma.$transaction([
+      prisma.project.findUnique({
+        where: {
+          id: input.id,
+        },
+        include: {
+          apiKeys: true,
+          personalProjectUser: true,
+        },
+      }),
+      prisma.projectUser.findFirst({
+        where: {
+          userId: ctx.session.user.id,
+          projectId: input.id,
+          role: {
+            in: ["ADMIN", "MEMBER"],
+          },
+        },
+      }),
+    ]);
+
+    if (!proj) {
+      throw new TRPCError({ code: "NOT_FOUND" });
+    }
+
+    return {
+      ...proj,
+      role: userRole?.role ?? null,
+    };
+  }),
+  update: protectedProcedure
+    .input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyProject(input.id, ctx);
+      return await prisma.project.update({
+        where: {
+          id: input.id,
+        },
+        data: {
+          name: input.updates.name,
+        },
+      });
+    }),
+  create: protectedProcedure
+    .input(z.object({ name: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      requireNothing(ctx);
+      const newProjectId = uuidv4();
+      const [newProject] = await prisma.$transaction([
+        prisma.project.create({
+          data: {
+            id: newProjectId,
+            name: input.name,
+          },
+        }),
+        prisma.projectUser.create({
+          data: {
+            userId: ctx.session.user.id,
+            projectId: newProjectId,
+            role: "ADMIN",
+          },
+        }),
+        prisma.apiKey.create({
+          data: {
+            name: "Default API Key",
+            projectId: newProjectId,
+            apiKey: generateApiKey(),
+          },
+        }),
+      ]);
+      return newProject;
+    }),
+  delete: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireIsProjectAdmin(input.id, ctx);
+      return await prisma.project.delete({
+        where: {
+          id: input.id,
+        },
+      });
+    }),
+});