Auto-resize project menu width

* Style project settings on mobile

* Use auto-resize text area for display name

* Remove unused import

README.md

@@ -65,7 +65,14 @@ OpenPipe includes a tool to generate new test scenarios based on your existing p
 4. Clone this repository: `git clone https://github.com/openpipe/openpipe`
 5. Install the dependencies: `cd openpipe && pnpm install`
 6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`.
-7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma db push` to create the database.
+7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma migrate dev` to create the database.
 8. Create a [GitHub OAuth App](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app) and update the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` values. (Note: a PR to make auth optional when running locally would be a great contribution!)
 9. Start the app: `pnpm dev`.
 10. Navigate to [http://localhost:3000](http://localhost:3000)
+
+## Testing Locally
+
+1. Copy your `.env` file to `.env.test`.
+2. Update the `DATABASE_URL` to have a different database name than your development one
+3. Run `DATABASE_URL=[your new datatase url] pnpm prisma migrate dev --skip-seed --skip-generate`
+4. Run `pnpm test`

app/.gitignore

@@ -34,6 +34,7 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env
 .env*.local
+.env.test
 
 # vercel
 .vercel

app/package.json

@@ -19,7 +19,7 @@
     "codegen": "tsx src/server/scripts/client-codegen.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
-    "test": "pnpm vitest --no-threads"
+    "test": "pnpm vitest"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.5.8",

app/prisma/migrations/20230809222649_rename_organization_to_project/migration.sql

@@ -0,0 +1,37 @@
+-- Rename Enum
+ALTER TYPE "OrganizationUserRole" RENAME TO "ProjectUserRole";
+
+-- Drop and recreate foreign keys
+ALTER TABLE "ApiKey" DROP CONSTRAINT "ApiKey_organizationId_fkey";
+ALTER TABLE "Dataset" DROP CONSTRAINT "Dataset_organizationId_fkey";
+ALTER TABLE "Experiment" DROP CONSTRAINT "Experiment_organizationId_fkey";
+ALTER TABLE "LoggedCall" DROP CONSTRAINT "LoggedCall_organizationId_fkey";
+ALTER TABLE "OrganizationUser" DROP CONSTRAINT "OrganizationUser_organizationId_fkey";
+ALTER TABLE "OrganizationUser" DROP CONSTRAINT "OrganizationUser_userId_fkey";
+
+-- Rename columns
+ALTER TABLE "ApiKey" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Dataset" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Experiment" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "LoggedCall" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "OrganizationUser" RENAME COLUMN "organizationId" TO "projectId";
+ALTER TABLE "Organization" RENAME COLUMN "personalOrgUserId" TO "personalProjectUserId";
+
+-- Rename table
+ALTER TABLE "Organization" RENAME TO "Project";
+ALTER TABLE "OrganizationUser" RENAME TO "ProjectUser";
+
+-- Recreate foreign keys
+ALTER TABLE "Experiment" ADD CONSTRAINT "Experiment_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "Dataset" ADD CONSTRAINT "Dataset_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ProjectUser" ADD CONSTRAINT "ProjectUser_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ProjectUser" ADD CONSTRAINT "ProjectUser_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- Rename indexes
+ALTER TABLE "Project" RENAME CONSTRAINT "Organization_pkey" TO "Project_pkey";
+ALTER TABLE "ProjectUser" RENAME CONSTRAINT "OrganizationUser_pkey" TO "ProjectUser_pkey";
+ALTER TABLE "Project" RENAME CONSTRAINT "Organization_personalOrgUserId_fkey" TO "Project_personalProjectUserId_fkey";
+ALTER INDEX "Organization_personalOrgUserId_key" RENAME TO "Project_personalProjectUserId_key";
+ALTER INDEX "OrganizationUser_organizationId_userId_key" RENAME TO "ProjectUser_projectId_userId_key";

app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

app/prisma/migrations/20230811023536_standardize_on_input_and_output_tokens_as_names/migration.sql

@@ -0,0 +1,15 @@
+/*
+  Warnings:
+
+  - You are about to rename the column `completionTokens` to `outputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `promptTokens` to `inputTokens` on the `ModelResponse` table.
+
+*/
+
+-- Rename completionTokens to outputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "completionTokens" TO "outputTokens";
+
+-- Rename promptTokens to inputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "promptTokens" TO "inputTokens";

app/prisma/schema.prisma

@@ -16,8 +16,8 @@ model Experiment {
 
     sortIndex Int @default(0)
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -117,8 +117,8 @@ model ModelResponse {
     receivedAt       DateTime?
     output           Json?
     cost             Float?
-    promptTokens     Int?
-    completionTokens Int?
+    inputTokens      Int?
+    outputTokens     Int?
     statusCode       Int?
     errorMessage     String?
     retryTime        DateTime?
@@ -180,8 +180,8 @@ model Dataset {
     name           String
     datasetEntries DatasetEntry[]
 
-    organizationId String       @db.Uuid
-    organization   Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String  @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -200,36 +200,35 @@ model DatasetEntry {
     updatedAt DateTime @updatedAt
 }
 
-// TODO rename Organization to Project
-model Organization {
-    id                String  @id @default(uuid()) @db.Uuid
-    name              String  @default("Project 1")
+model Project {
+    id   String @id @default(uuid()) @db.Uuid
+    name String @default("Project 1")
 
-    personalOrgUserId String? @unique @db.Uuid
-    personalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+    personalProjectUserId String? @unique @db.Uuid
+    personalProjectUser   User?   @relation(fields: [personalProjectUserId], references: [id], onDelete: Cascade)
 
-    createdAt         DateTime           @default(now())
-    updatedAt         DateTime           @updatedAt
-    organizationUsers OrganizationUser[]
-    experiments       Experiment[]
-    datasets          Dataset[]
-    loggedCalls       LoggedCall[]
-    apiKeys           ApiKey[]
+    createdAt    DateTime      @default(now())
+    updatedAt    DateTime      @updatedAt
+    projectUsers ProjectUser[]
+    experiments  Experiment[]
+    datasets     Dataset[]
+    loggedCalls  LoggedCall[]
+    apiKeys      ApiKey[]
 }
 
-enum OrganizationUserRole {
+enum ProjectUserRole {
     ADMIN
     MEMBER
     VIEWER
 }
 
-model OrganizationUser {
+model ProjectUser {
     id String @id @default(uuid()) @db.Uuid
 
-    role OrganizationUserRole
+    role ProjectUserRole
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     userId String @db.Uuid
     user   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
@@ -237,7 +236,7 @@ model OrganizationUser {
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    @@unique([organizationId, userId])
+    @@unique([projectId, userId])
 }
 
 model WorldChampEntrant {
@@ -265,14 +264,14 @@ model LoggedCall {
     // A LoggedCall is always associated with a LoggedCallModelResponse. If this
     // is a cache miss, we create a new LoggedCallModelResponse.
     // If it's a cache hit, it's a pre-existing LoggedCallModelResponse.    
-    modelResponseId String?                 @db.Uuid
+    modelResponseId String?                  @db.Uuid
     modelResponse   LoggedCallModelResponse? @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
 
     // The responses created by this LoggedCall. Will be empty if this LoggedCall was a cache hit.
     createdResponses LoggedCallModelResponse[] @relation(name: "ModelResponseOriginalCall")
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     tags LoggedCallTag[]
 
@@ -323,11 +322,11 @@ model LoggedCallModelResponse {
 }
 
 model LoggedCallTag {
-    id    String @id @default(uuid()) @db.Uuid
+    id    String  @id @default(uuid()) @db.Uuid
     name  String
     value String?
 
-    loggedCallId String        @db.Uuid
+    loggedCallId String     @db.Uuid
     loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
 
     @@index([name])
@@ -340,8 +339,8 @@ model ApiKey {
     name   String
     apiKey String @unique
 
-    organizationId String        @db.Uuid
-    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+    projectId String   @db.Uuid
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -390,8 +389,8 @@ model User {
 
     accounts          Account[]
     sessions          Session[]
-    organizationUsers OrganizationUser[]
-    organizations     Organization[]
+    projectUsers      ProjectUser[]
+    projects          Project[]
     worldChampEntrant WorldChampEntrant?
 
     createdAt DateTime @default(now())

app/prisma/seed.ts

@@ -5,14 +5,14 @@ import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111111";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
@@ -26,7 +26,7 @@ await prisma.experiment.create({
   data: {
     id: defaultId,
     label: "Country Capitals Example",
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 

app/prisma/seedAgiEval.ts

@@ -7,14 +7,14 @@ import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111112";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
@@ -47,7 +47,7 @@ for (const dataset of datasets) {
   const oldExperiment = await prisma.experiment.findFirst({
     where: {
       label: experimentName,
-      organizationId: org.id,
+      projectId: project.id,
     },
   });
   if (oldExperiment) {
@@ -60,7 +60,7 @@ for (const dataset of datasets) {
     data: {
       id: oldExperiment?.id ?? undefined,
       label: experimentName,
-      organizationId: org.id,
+      projectId: project.id,
     },
   });
 

app/prisma/seedDashboard.ts

@@ -311,9 +311,9 @@ const MODEL_RESPONSE_TEMPLATES: {
 
 await prisma.loggedCallModelResponse.deleteMany();
 
-const org = await prisma.organization.findFirst({
+const project = await prisma.project.findFirst({
   where: {
-    personalOrgUserId: {
+    personalProjectUserId: {
       not: null,
     },
   },
@@ -322,8 +322,8 @@ const org = await prisma.organization.findFirst({
   },
 });
 
-if (!org) {
-  console.error("No org found. Sign up to create your first org.");
+if (!project) {
+  console.error("No project found. Sign up to create your first project.");
   process.exit(1);
 }
 
@@ -348,7 +348,7 @@ for (let i = 0; i < 1437; i++) {
     id: loggedCallId,
     cacheHit: false,
     startTime,
-    organizationId: org.id,
+    projectId: project.id,
     createdAt: startTime,
   });
 
@@ -373,7 +373,7 @@ for (let i = 0; i < 1437; i++) {
     respStatus: template.respStatus,
     error: template.error,
     createdAt: startTime,
-    cacheKey: hashRequest(org.id, template.reqPayload as JsonValue),
+    cacheKey: hashRequest(project.id, template.reqPayload as JsonValue),
     durationMs: endTime.getTime() - startTime.getTime(),
     inputTokens: template.inputTokens,
     outputTokens: template.outputTokens,

app/prisma/seedTwitterSentiment.ts

@@ -6,14 +6,14 @@ import { promptConstructorVersion } from "~/promptConstructor/version";
 
 const defaultId = "11111111-1111-1111-1111-111111111112";
 
-await prisma.organization.deleteMany({
+await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
-// If there's an existing org, just seed into it
-const org =
-  (await prisma.organization.findFirst({})) ??
-  (await prisma.organization.create({
+// If there's an existing project, just seed into it
+const project =
+  (await prisma.project.findFirst({})) ??
+  (await prisma.project.create({
     data: { id: defaultId },
   }));
 
@@ -27,7 +27,7 @@ const experimentName = `Twitter Sentiment Analysis`;
 const oldExperiment = await prisma.experiment.findFirst({
   where: {
     label: experimentName,
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 if (oldExperiment) {
@@ -40,7 +40,7 @@ const experiment = await prisma.experiment.create({
   data: {
     id: oldExperiment?.id ?? undefined,
     label: experimentName,
-    organizationId: org.id,
+    projectId: project.id,
   },
 });
 

app/public/favicons/safari-pinned-tab.svg

@@ -9,10 +9,9 @@ Created by potrace 1.14, written by Peter Selinger 2001-2017
 </metadata>
 <g transform="translate(0.000000,550.000000) scale(0.100000,-0.100000)"
 fill="#000000" stroke="none">
-<path d="M813 5478 c-18 -13 -37 -36 -43 -52 -6 -19 -10 -236 -10 -603 0 -638
--1 -626 65 -657 25 -12 67 -16 179 -16 l146 0 0 -2032 0 -2032 23 -33 c12 -18
-35 -37 51 -43 19 -7 539 -10 1528 -10 1663 0 1549 -5 1582 65 14 30 16 235 16
-2059 l0 2026 156 0 156 0 39 39 39 39 0 587 c0 651 1 638 -65 669 -30 14 -223
-16 -1932 16 l-1898 0 -32 -22z"/>
+<path d="M785 5474 l-25 -27 0 -622 0 -622 25 -27 24 -26 171 0 170 0 0 -2050
+0 -2051 25 -25 24 -24 1557 2 1556 3 19 24 c19 23 19 70 19 2072 l0 2049 169
+0 c165 0 169 1 195 25 l26 24 0 626 0 626 -26 24 -27 25 -1939 0 -1939 0 -24
+-26z"/>
 </g>
 </svg>

app/public/logo.svg

@@ -1,5 +1,28 @@
-<svg width="380" height="320" viewBox="0 0 380 320" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M72 320L122.5 231L130.5 150.5L115 73L72 0H312L265 64.5L257 158.5L265 249L312 320H72Z" fill="#FF5733"/>
-<path d="M67.027 9.5C72.9909 9.5 79.5196 12.3449 86.3672 19.2588C93.2495 26.2075 99.8845 36.7468 105.66 50.5336C117.194 78.0671 124.554 116.764 124.554 160C124.554 203.236 117.194 241.933 105.66 269.466C99.8845 283.253 93.2495 293.793 86.3672 300.741C79.5196 307.655 72.9909 310.5 67.027 310.5C61.0632 310.5 54.5345 307.655 47.6868 300.741C40.8045 293.793 34.1695 283.253 28.394 269.466C16.8596 241.933 9.5 203.236 9.5 160C9.5 116.764 16.8596 78.0671 28.394 50.5336C34.1695 36.7468 40.8045 26.2075 47.6868 19.2588C54.5345 12.3449 61.0632 9.5 67.027 9.5Z" stroke="#FF5733" stroke-width="19"/>
-<path d="M312.027 9.5C317.991 9.5 324.52 12.3449 331.367 19.2588C338.25 26.2075 344.885 36.7468 350.66 50.5336C362.194 78.0671 369.554 116.764 369.554 160C369.554 203.236 362.194 241.933 350.66 269.466C344.885 283.253 338.25 293.793 331.367 300.741C324.52 307.655 317.991 310.5 312.027 310.5C306.063 310.5 299.534 307.655 292.687 300.741C285.805 293.793 279.17 283.253 273.394 269.466C261.86 241.933 254.5 203.236 254.5 160C254.5 116.764 261.86 78.0671 273.394 50.5336C279.17 36.7468 285.805 26.2075 292.687 19.2588C299.534 12.3449 306.063 9.5 312.027 9.5Z" stroke="#FF5733" stroke-width="19"/>
+<svg width="398" height="550" viewBox="0 0 398 550" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M39 125H359V542C359 546.418 355.418 550 351 550H47C42.5817 550 39 546.418 39 542V125Z" fill="black"/>
+<path d="M0 8C0 3.58172 3.58172 0 8 0H390C394.418 0 398 3.58172 398 8V127C398 131.418 394.418 135 390 135H7.99999C3.58171 135 0 131.418 0 127V8Z" fill="black"/>
+<path d="M50 135H348V535C348 537.209 346.209 539 344 539H54C51.7909 539 50 537.209 50 535V135Z" fill="#FF5733"/>
+<path d="M11 14.0001C11 11.791 12.7909 10.0001 15 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H15C12.7909 124 11 122.209 11 120V14.0001Z" fill="#FF5733"/>
+<path d="M11 14.0001C11 11.791 12.7909 10.0001 15 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H15C12.7909 124 11 122.209 11 120V14.0001Z" fill="url(#paint0_linear_102_49)"/>
+<path d="M50 134H348V535C348 537.209 346.209 539 344 539H54C51.7909 539 50 537.209 50 535V134Z" fill="url(#paint1_linear_102_49)"/>
+<path d="M108 142H156V535H108V142Z" fill="white"/>
+<path d="M300 135H348V535C348 537.209 346.209 539 344 539H300V135Z" fill="white" fill-opacity="0.25"/>
+<path d="M96 142H108V535H96V142Z" fill="white" fill-opacity="0.5"/>
+<path d="M84 10.0001H133V120H84V10.0001Z" fill="white"/>
+<path d="M339 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H339V10.0001Z" fill="white" fill-opacity="0.25"/>
+<path d="M71.9995 10.0001H83.9995V120H71.9995V10.0001Z" fill="white" fill-opacity="0.5"/>
+<path d="M108 534.529H156V539.019H108V534.529Z" fill="#AAAAAA"/>
+<path opacity="0.5" d="M95.9927 534.529H107.982V539.019H95.9927V534.529Z" fill="#AAAAAA"/>
+<path d="M84.0029 119.887H133.007V124.027H84.0029V119.887Z" fill="#AAAAAA"/>
+<path opacity="0.5" d="M71.9883 119.887H83.978V124.027H71.9883V119.887Z" fill="#AAAAAA"/>
+<defs>
+<linearGradient id="paint0_linear_102_49" x1="335" y1="67.0001" x2="137" y2="67.0001" gradientUnits="userSpaceOnUse">
+<stop stop-color="#D62600"/>
+<stop offset="1" stop-color="#FF5733" stop-opacity="0"/>
+</linearGradient>
+<linearGradient id="paint1_linear_102_49" x1="306.106" y1="336.5" x2="149.597" y2="336.5" gradientUnits="userSpaceOnUse">
+<stop stop-color="#D62600"/>
+<stop offset="1" stop-color="#FF5733" stop-opacity="0"/>
+</linearGradient>
+</defs>
 </svg>

app/src/components/CopiableCode.tsx

@@ -19,7 +19,7 @@ const CopiableCode = ({ code }: { code: string }) => {
       w="full"
       justifyContent="space-between"
     >
-      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5}>
+      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5} overflowX="auto">
         {code}
       </Text>
       <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>

app/src/components/Favicon.tsx

@@ -8,8 +8,8 @@ export default function Favicon() {
       <link rel="icon" type="image/png" sizes="16x16" href="/favicons/favicon-16x16.png" />
       <link rel="manifest" href="/favicons/site.webmanifest" />
       <link rel="shortcut icon" href="/favicons/favicon.ico" />
+      <link rel="mask-icon" href="/favicons/safari-pinned-tab.svg" color="#5bbad5" />
       <meta name="msapplication-TileColor" content="#da532c" />
-      <meta name="msapplication-config" content="/favicons/browserconfig.xml" />
       <meta name="theme-color" content="#ffffff" />
     </Head>
   );

app/src/components/OutputsTable/EditEvaluations.tsx

@@ -12,6 +12,7 @@ import {
   Select,
   FormHelperText,
   Code,
+  IconButton,
 } from "@chakra-ui/react";
 import { type Evaluation, EvalType } from "@prisma/client";
 import { useCallback, useState } from "react";
@@ -183,46 +184,37 @@ export default function EditEvaluations() {
                 <Text flex={1}>
                   {evaluation.evalType}: &quot;{evaluation.value}&quot;
                 </Text>
-                <Button
+
+                <IconButton
+                  aria-label="Edit"
                   variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                   minW="unset"
+                  color="gray.400"
                   onClick={() => setEditingId(evaluation.id)}
-                  _hover={{
-                    color: "gray.800",
-                    cursor: "pointer",
-                  }}
-                >
-                  <Icon as={BsPencil} boxSize={4} />
-                </Button>
-                <Button
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
+                  icon={<Icon as={BsPencil} />}
+                />
+                <IconButton
+                  aria-label="Delete"
                   variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                   minW="unset"
+                  color="gray.400"
                   onClick={() => onDelete(evaluation.id)}
-                  _hover={{
-                    color: "gray.800",
-                    cursor: "pointer",
-                  }}
-                >
-                  <Icon as={BsX} boxSize={6} />
-                </Button>
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
+                  icon={<Icon as={BsX} boxSize={6} />}
+                />
               </HStack>
             ),
           )}
           {editingId == null && (
             <Button
               onClick={() => setEditingId("new")}
-              alignSelf="flex-start"
+              alignSelf="end"
               size="sm"
               mt={4}
               colorScheme="blue"
             >
-              Add Evaluation
+              New Evaluation
             </Button>
           )}
           {editingId == "new" && (

app/src/components/OutputsTable/EditScenarioVars.tsx

@@ -1,103 +1,185 @@
-import { Text, Button, HStack, Heading, Icon, Input, Stack } from "@chakra-ui/react";
-import { useState } from "react";
-import { BsCheck, BsX } from "react-icons/bs";
+import { Text, Button, HStack, Heading, Icon, IconButton, Stack, VStack } from "@chakra-ui/react";
+import { type TemplateVariable } from "@prisma/client";
+import { useEffect, useState } from "react";
+import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import { maybeReportError } from "~/utils/standardResponses";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+
+export const ScenarioVar = ({
+  variable,
+  isEditing,
+  setIsEditing,
+}: {
+  variable: Pick<TemplateVariable, "id" | "label">;
+  isEditing: boolean;
+  setIsEditing: (isEditing: boolean) => void;
+}) => {
+  const utils = api.useContext();
+
+  const [label, setLabel] = useState(variable.label);
+
+  useEffect(() => {
+    setLabel(variable.label);
+  }, [variable.label]);
+
+  const renameVarMutation = api.scenarioVars.rename.useMutation();
+  const [onRename] = useHandledAsyncCallback(async () => {
+    const resp = await renameVarMutation.mutateAsync({ id: variable.id, label });
+    if (maybeReportError(resp)) return;
+
+    setIsEditing(false);
+    await utils.scenarioVars.list.invalidate();
+    await utils.scenarios.list.invalidate();
+  }, [label, variable.id]);
+
+  const deleteMutation = api.scenarioVars.delete.useMutation();
+  const [onDeleteVar] = useHandledAsyncCallback(async () => {
+    await deleteMutation.mutateAsync({ id: variable.id });
+    await utils.scenarioVars.list.invalidate();
+  }, [variable.id]);
+
+  if (isEditing) {
+    return (
+      <HStack w="full">
+        <FloatingLabelInput
+          flex={1}
+          label="Renamed Variable"
+          value={label}
+          onChange={(e) => setLabel(e.target.value)}
+          onKeyDown={(e) => {
+            if (e.key === "Enter") {
+              e.preventDefault();
+              onRename();
+            }
+            // If the user types a space, replace it with an underscore
+            if (e.key === " ") {
+              e.preventDefault();
+              setLabel((label) => label && `${label}_`);
+            }
+          }}
+        />
+        <Button size="sm" onClick={() => setIsEditing(false)}>
+          Cancel
+        </Button>
+        <Button size="sm" colorScheme="blue" onClick={onRename}>
+          Save
+        </Button>
+      </HStack>
+    );
+  } else {
+    return (
+      <HStack w="full" borderTopWidth={1} borderColor="gray.200">
+        <Text flex={1}>{variable.label}</Text>
+        <IconButton
+          aria-label="Edit"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={() => setIsEditing(true)}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsPencil} />}
+        />
+        <IconButton
+          aria-label="Delete"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={onDeleteVar}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsX} boxSize={6} />}
+        />
+      </HStack>
+    );
+  }
+};
 
 export default function EditScenarioVars() {
   const experiment = useExperiment();
-  const vars =
-    api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+  const vars = useScenarioVars();
+
+  const [currentlyEditingId, setCurrentlyEditingId] = useState<string | null>(null);
 
   const [newVariable, setNewVariable] = useState<string>("");
-  const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
+  const newVarIsValid = newVariable?.length ?? 0 > 0;
 
   const utils = api.useContext();
-  const addVarMutation = api.templateVars.create.useMutation();
+  const addVarMutation = api.scenarioVars.create.useMutation();
   const [onAddVar] = useHandledAsyncCallback(async () => {
     if (!experiment.data?.id) return;
-    if (!newVarIsValid) return;
-    await addVarMutation.mutateAsync({
+    if (!newVariable) return;
+    const resp = await addVarMutation.mutateAsync({
       experimentId: experiment.data.id,
       label: newVariable,
     });
-    await utils.templateVars.list.invalidate();
+    if (maybeReportError(resp)) return;
+
+    await utils.scenarioVars.list.invalidate();
     setNewVariable("");
   }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
 
-  const deleteMutation = api.templateVars.delete.useMutation();
-  const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
-    await deleteMutation.mutateAsync({ id });
-    await utils.templateVars.list.invalidate();
-  }, []);
-
   return (
     <Stack>
       <Heading size="sm">Scenario Variables</Heading>
-      <Stack spacing={2}>
+      <VStack spacing={4}>
         <Text fontSize="sm">
           Scenario variables can be used in your prompt variants as well as evaluations.
         </Text>
-        <HStack spacing={0}>
-          <Input
-            placeholder="Add Scenario Variable"
-            size="sm"
-            borderTopRadius={0}
-            borderRightRadius={0}
-            value={newVariable}
-            onChange={(e) => setNewVariable(e.target.value)}
-            onKeyDown={(e) => {
-              if (e.key === "Enter") {
-                e.preventDefault();
-                onAddVar();
-              }
-              // If the user types a space, replace it with an underscore
-              if (e.key === " ") {
-                e.preventDefault();
-                setNewVariable((v) => v + "_");
-              }
-            }}
-          />
-          <Button
-            size="xs"
-            height="100%"
-            borderLeftRadius={0}
-            isDisabled={!newVarIsValid}
-            onClick={onAddVar}
-          >
-            <Icon as={BsCheck} boxSize={8} />
-          </Button>
-        </HStack>
-
-        <HStack spacing={2} py={4} wrap="wrap">
-          {vars.map((variable) => (
-            <HStack
+        <VStack spacing={0} w="full">
+          {vars.data?.map((variable) => (
+            <ScenarioVar
+              variable={variable}
               key={variable.id}
-              spacing={0}
-              bgColor="blue.100"
-              color="blue.600"
-              pl={2}
-              pr={0}
-              fontWeight="bold"
-            >
-              <Text fontSize="sm" flex={1}>
-                {variable.label}
-              </Text>
-              <Button
-                size="xs"
-                variant="ghost"
-                colorScheme="blue"
-                p="unset"
-                minW="unset"
-                px="unset"
-                onClick={() => onDeleteVar(variable.id)}
-              >
-                <Icon as={BsX} boxSize={6} color="blue.800" />
-              </Button>
-            </HStack>
+              isEditing={currentlyEditingId === variable.id}
+              setIsEditing={(isEditing) => {
+                if (isEditing) {
+                  setCurrentlyEditingId(variable.id);
+                } else {
+                  setCurrentlyEditingId(null);
+                }
+              }}
+            />
           ))}
-        </HStack>
-      </Stack>
+        </VStack>
+        {currentlyEditingId !== "new" && (
+          <Button
+            colorScheme="blue"
+            size="sm"
+            onClick={() => setCurrentlyEditingId("new")}
+            alignSelf="end"
+          >
+            New Variable
+          </Button>
+        )}
+        {currentlyEditingId === "new" && (
+          <HStack w="full">
+            <FloatingLabelInput
+              flex={1}
+              label="New Variable"
+              value={newVariable}
+              onChange={(e) => setNewVariable(e.target.value)}
+              onKeyDown={(e) => {
+                if (e.key === "Enter") {
+                  e.preventDefault();
+                  onAddVar();
+                }
+                // If the user types a space, replace it with an underscore
+                if (e.key === " ") {
+                  e.preventDefault();
+                  setNewVariable((v) => v && `${v}_`);
+                }
+              }}
+            />
+            <Button size="sm" onClick={() => setCurrentlyEditingId(null)}>
+              Cancel
+            </Button>
+            <Button size="sm" colorScheme="blue" onClick={onAddVar}>
+              Save
+            </Button>
+          </HStack>
+        )}
+      </VStack>
     </Stack>
   );
 }

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,7 +1,7 @@
 import { api } from "~/utils/api";
 import { type PromptVariant, type Scenario } from "../types";
 import { type StackProps, Text, VStack } from "@chakra-ui/react";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
 import stringify from "json-stringify-pretty-compact";
@@ -23,10 +23,7 @@ export default function OutputCell({
   variant: PromptVariant;
 }): ReactElement | null {
   const utils = api.useContext();
-  const experiment = useExperiment();
-  const vars = api.templateVars.list.useQuery({
-    experimentId: experiment.data?.id ?? "",
-  }).data;
+  const vars = useScenarioVars().data;
 
   const scenarioVariables = scenario.variableValues as Record<string, string>;
   const templateHasVariables =
@@ -191,7 +188,7 @@ export default function OutputCell({
 
   return (
     <CellWrapper>
-      <Text>{contentToDisplay}</Text>
+      <Text whiteSpace="pre-wrap">{contentToDisplay}</Text>
     </CellWrapper>
   );
 }

app/src/components/OutputsTable/OutputCell/OutputStats.tsx

@@ -19,8 +19,8 @@ export const OutputStats = ({
       ? modelResponse.receivedAt.getTime() - modelResponse.requestedAt.getTime()
       : 0;
 
-  const promptTokens = modelResponse.promptTokens;
-  const completionTokens = modelResponse.completionTokens;
+  const inputTokens = modelResponse.inputTokens;
+  const outputTokens = modelResponse.outputTokens;
 
   return (
     <HStack
@@ -55,8 +55,8 @@ export const OutputStats = ({
       </HStack>
       {modelResponse.cost && (
         <CostTooltip
-          promptTokens={promptTokens}
-          completionTokens={completionTokens}
+          inputTokens={inputTokens}
+          outputTokens={outputTokens}
           cost={modelResponse.cost}
         >
           <HStack spacing={0}>

app/src/components/OutputsTable/ScenarioEditor.tsx

@@ -1,7 +1,7 @@
 import { isEqual } from "lodash-es";
 import { useEffect, useState, type DragEvent } from "react";
 import { api } from "~/utils/api";
-import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperimentAccess, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
 import { type Scenario } from "./types";
 
 import {
@@ -41,8 +41,7 @@ export default function ScenarioEditor({
     if (savedValues) setValues(savedValues);
   }, [savedValues]);
 
-  const experiment = useExperiment();
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = useScenarioVars();
 
   const variableLabels = vars.data?.map((v) => v.label) ?? [];
 

app/src/components/OutputsTable/ScenarioEditorModal.tsx

@@ -58,7 +58,7 @@ export const ScenarioEditorModal = ({
     await utils.scenarios.list.invalidate();
   }, [mutation, values]);
 
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = api.scenarioVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
   const variableLabels = vars.data?.map((v) => v.label) ?? [];
 
   return (

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -72,7 +72,7 @@ export const ScenariosHeader = () => {
               Autogenerate Scenario
             </MenuItem>
             <MenuItem icon={<BsPencil />} onClick={openDrawer}>
-              Edit Vars
+              Add or Remove Variables
             </MenuItem>
           </MenuList>
         </Menu>

app/src/components/OutputsTable/VariantStats.tsx

@@ -17,8 +17,8 @@ export default function VariantStats(props: { variant: PromptVariant }) {
       initialData: {
         evalResults: [],
         overallCost: 0,
-        promptTokens: 0,
-        completionTokens: 0,
+        inputTokens: 0,
+        outputTokens: 0,
         scenarioCount: 0,
         outputCount: 0,
         awaitingEvals: false,
@@ -68,8 +68,8 @@ export default function VariantStats(props: { variant: PromptVariant }) {
       </HStack>
       {data.overallCost && (
         <CostTooltip
-          promptTokens={data.promptTokens}
-          completionTokens={data.completionTokens}
+          inputTokens={data.inputTokens}
+          outputTokens={data.outputTokens}
           cost={data.overallCost}
         >
           <HStack spacing={0} align="center" color="gray.500">

app/src/components/datasets/DatasetCard.tsx

@@ -72,12 +72,12 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
 
 export const NewDatasetCard = () => {
   const router = useRouter();
-  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
   const createMutation = api.datasets.create.useMutation();
   const [createDataset, isLoading] = useHandledAsyncCallback(async () => {
-    const newDataset = await createMutation.mutateAsync({ organizationId: selectedOrgId ?? "" });
+    const newDataset = await createMutation.mutateAsync({ projectId: selectedProjectId ?? "" });
     await router.push({ pathname: "/data/[id]", query: { id: newDataset.id } });
-  }, [createMutation, router, selectedOrgId]);
+  }, [createMutation, router, selectedProjectId]);
 
   return (
     <AspectRatio ratio={1.2} w="full">

app/src/components/experiments/ExperimentCard.tsx

@@ -76,17 +76,17 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
 
 export const NewExperimentCard = () => {
   const router = useRouter();
-  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
   const createMutation = api.experiments.create.useMutation();
   const [createExperiment, isLoading] = useHandledAsyncCallback(async () => {
     const newExperiment = await createMutation.mutateAsync({
-      organizationId: selectedOrgId ?? "",
+      projectId: selectedProjectId ?? "",
     });
     await router.push({
       pathname: "/experiments/[id]",
       query: { id: newExperiment.id },
     });
-  }, [createMutation, router, selectedOrgId]);
+  }, [createMutation, router, selectedProjectId]);
 
   return (
     <AspectRatio ratio={1.2} w="full">

app/src/components/experiments/ExperimentHeaderButtons/useOnForkButtonPressed.tsx

@@ -10,15 +10,15 @@ export const useOnForkButtonPressed = () => {
 
   const user = useSession().data;
   const experiment = useExperiment();
-  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  const selectedProjectId = useAppStore((state) => state.selectedProjectId);
 
   const forkMutation = api.experiments.fork.useMutation();
 
   const [onFork, isForking] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id || !selectedOrgId) return;
+    if (!experiment.data?.id || !selectedProjectId) return;
     const forkedExperimentId = await forkMutation.mutateAsync({
       id: experiment.data.id,
-      organizationId: selectedOrgId,
+      projectId: selectedProjectId,
     });
     await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
   }, [forkMutation, experiment.data?.id, router]);

app/src/components/nav/AppShell.tsx

@@ -40,8 +40,15 @@ const NavSidebar = () => {
       borderRightWidth={1}
       borderColor="gray.300"
     >
-      <HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={2} py={2}>
-        <Image src="/logo.svg" alt="" boxSize={6} mr={4} />
+      <HStack
+        as={Link}
+        href="/"
+        _hover={{ textDecoration: "none" }}
+        spacing={{ base: 1, md: 0 }}
+        mx={2}
+        py={{ base: 1, md: 2 }}
+      >
+        <Image src="/logo.svg" alt="" boxSize={6} mr={4} ml={{ base: 0.5, md: 0 }} />
         <Heading size="md" fontFamily="inconsolata, monospace">
           OpenPipe
         </Heading>

app/src/components/nav/NavSidebarOption.tsx

@@ -1,11 +1,10 @@
-import { Box, type BoxProps } from "@chakra-ui/react";
+import { Box, type BoxProps, forwardRef } from "@chakra-ui/react";
 import { useRouter } from "next/router";
 
-const NavSidebarOption = ({
-  activeHrefPattern,
-  disableHoverEffect,
-  ...props
-}: { activeHrefPattern?: string; disableHoverEffect?: boolean } & BoxProps) => {
+const NavSidebarOption = forwardRef<
+  { activeHrefPattern?: string; disableHoverEffect?: boolean } & BoxProps,
+  "div"
+>(({ activeHrefPattern, disableHoverEffect, ...props }, ref) => {
   const router = useRouter();
   const isActive = activeHrefPattern && router.pathname.startsWith(activeHrefPattern);
   return (
@@ -18,10 +17,13 @@ const NavSidebarOption = ({
       cursor="pointer"
       borderRadius={4}
       {...props}
+      ref={ref}
     >
       {props.children}
     </Box>
   );
-};
+});
+
+NavSidebarOption.displayName = "NavSidebarOption";
 
 export default NavSidebarOption;

app/src/components/nav/ProjectBreadcrumbContents.tsx

@@ -1,12 +1,12 @@
 import { HStack, Flex, Text } from "@chakra-ui/react";
-import { useSelectedOrg } from "~/utils/hooks";
+import { useSelectedProject } from "~/utils/hooks";
 
 // Have to export only contents here instead of full BreadcrumbItem because Chakra doesn't
 // recognize a BreadcrumbItem exported with this component as a valid child of Breadcrumb.
-export default function ProjectBreadcrumbContents({ orgName = "" }: { orgName?: string }) {
-  const { data: selectedOrg } = useSelectedOrg();
+export default function ProjectBreadcrumbContents({ projectName = "" }: { projectName?: string }) {
+  const { data: selectedProject } = useSelectedProject();
 
-  orgName = orgName || selectedOrg?.name || "";
+  projectName = projectName || selectedProject?.name || "";
 
   return (
     <HStack w="full">
@@ -18,10 +18,10 @@ export default function ProjectBreadcrumbContents({ orgName = "" }: { orgName?:
         alignItems="center"
         justifyContent="center"
       >
-        <Text>{orgName[0]?.toUpperCase()}</Text>
+        <Text>{projectName[0]?.toUpperCase()}</Text>
       </Flex>
       <Text display={{ base: "none", md: "block" }} py={1}>
-        {orgName}
+        {projectName}
       </Text>
     </HStack>
   );

app/src/components/nav/ProjectMenu.tsx

@@ -15,41 +15,43 @@ import {
 } from "@chakra-ui/react";
 import React, { useEffect, useState } from "react";
 import Link from "next/link";
-import { AiFillCaretDown } from "react-icons/ai";
-import { BsGear, BsPlus } from "react-icons/bs";
-import { type Organization } from "@prisma/client";
+import { BsChevronRight, BsGear, BsPlus } from "react-icons/bs";
+import { type Project } from "@prisma/client";
 
 import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 import NavSidebarOption from "./NavSidebarOption";
-import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
 import { useRouter } from "next/router";
 
 export default function ProjectMenu() {
   const router = useRouter();
-  const isActive = router.pathname.startsWith("/home");
   const utils = api.useContext();
 
-  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
-  const setSelectedOrgId = useAppStore((s) => s.setSelectedOrgId);
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
+  const setselectedProjectId = useAppStore((s) => s.setselectedProjectId);
 
-  const { data: orgs } = api.organizations.list.useQuery();
+  const { data: projects } = api.projects.list.useQuery();
 
   useEffect(() => {
-    if (orgs && orgs[0] && (!selectedOrgId || !orgs.find((org) => org.id === selectedOrgId))) {
-      setSelectedOrgId(orgs[0].id);
+    if (
+      projects &&
+      projects[0] &&
+      (!selectedProjectId || !projects.find((proj) => proj.id === selectedProjectId))
+    ) {
+      setselectedProjectId(projects[0].id);
     }
-  }, [selectedOrgId, setSelectedOrgId, orgs]);
+  }, [selectedProjectId, setselectedProjectId, projects]);
 
-  const { data: selectedOrg } = useSelectedOrg();
+  const { data: selectedProject } = useSelectedProject();
 
   const popover = useDisclosure();
 
-  const createMutation = api.organizations.create.useMutation();
+  const createMutation = api.projects.create.useMutation();
   const [createProject, isLoading] = useHandledAsyncCallback(async () => {
-    const newOrg = await createMutation.mutateAsync({ name: "New Project" });
-    await utils.organizations.list.invalidate();
-    setSelectedOrgId(newOrg.id);
+    const newProj = await createMutation.mutateAsync({ name: "New Project" });
+    await utils.projects.list.invalidate();
+    setselectedProjectId(newProj.id);
     await router.push({ pathname: "/project/settings" });
   }, [createMutation, router]);
 
@@ -65,15 +67,10 @@ export default function ProjectMenu() {
       >
         PROJECT
       </Text>
-      <NavSidebarOption>
-        <Popover
-          placement="bottom-start"
-          isOpen={popover.isOpen}
-          onClose={popover.onClose}
-          closeOnBlur
-        >
-          <PopoverTrigger>
-            <HStack w="full" justifyContent="space-between" onClick={popover.onToggle}>
+      <Popover placement="right-end" isOpen={popover.isOpen} onClose={popover.onClose} closeOnBlur>
+        <PopoverTrigger>
+          <NavSidebarOption>
+            <HStack w="full" onClick={popover.onToggle}>
               <Flex
                 p={1}
                 borderRadius={4}
@@ -83,74 +80,68 @@ export default function ProjectMenu() {
                 m={{ base: 0, md: 1 }}
                 alignItems="center"
                 justifyContent="center"
-                // onClick={sidebarExpanded ? undefined : openMenu}
               >
-                <Text>{selectedOrg?.name[0]?.toUpperCase()}</Text>
+                <Text>{selectedProject?.name[0]?.toUpperCase()}</Text>
               </Flex>
-              <Text fontSize="sm" display={{ base: "none", md: "block" }} py={1}>
-                {selectedOrg?.name}
+              <Text fontSize="sm" display={{ base: "none", md: "block" }} py={1} flex={1}>
+                {selectedProject?.name}
               </Text>
-              <Icon as={AiFillCaretDown} boxSize={3} size="xs" color="gray.500" mr={2} />
+              <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
             </HStack>
-          </PopoverTrigger>
-          <PopoverContent
-            _focusVisible={{ boxShadow: "unset" }}
-            minW={0}
-            borderColor="blue.400"
-            w="full"
-          >
-            <VStack alignItems="flex-start" spacing={2} py={4} px={2}>
-              <Text color="gray.500" fontSize="xs" fontWeight="bold" pb={1}>
-                PROJECTS
-              </Text>
-              <Divider />
-              <VStack spacing={0} w="full">
-                {orgs?.map((org) => (
-                  <ProjectOption
-                    key={org.id}
-                    org={org}
-                    isActive={org.id === selectedOrgId}
-                    onClose={popover.onClose}
-                  />
-                ))}
-              </VStack>
-              <HStack
-                as={Button}
-                variant="ghost"
-                colorScheme="blue"
-                color="blue.400"
-                pr={8}
-                w="full"
-                onClick={createProject}
-              >
-                <Icon as={isLoading ? Spinner : BsPlus} boxSize={6} />
-                <Text>New project</Text>
-              </HStack>
+          </NavSidebarOption>
+        </PopoverTrigger>
+        <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} w="auto" minW={100} maxW={280}>
+          <VStack alignItems="flex-start" spacing={2} py={4} px={2}>
+            <Text color="gray.500" fontSize="xs" fontWeight="bold" pb={1}>
+              PROJECTS
+            </Text>
+            <Divider />
+            <VStack spacing={0} w="full">
+              {projects?.map((proj) => (
+                <ProjectOption
+                  key={proj.id}
+                  proj={proj}
+                  isActive={proj.id === selectedProjectId}
+                  onClose={popover.onClose}
+                />
+              ))}
             </VStack>
-          </PopoverContent>
-        </Popover>
-      </NavSidebarOption>
+            <HStack
+              as={Button}
+              variant="ghost"
+              colorScheme="blue"
+              color="blue.400"
+              pr={8}
+              w="full"
+              onClick={createProject}
+            >
+              <Icon as={isLoading ? Spinner : BsPlus} boxSize={6} />
+              <Text>New project</Text>
+            </HStack>
+          </VStack>
+        </PopoverContent>
+      </Popover>
     </VStack>
   );
 }
 
 const ProjectOption = ({
-  org,
+  proj,
   isActive,
   onClose,
 }: {
-  org: Organization;
+  proj: Project;
   isActive: boolean;
   onClose: () => void;
 }) => {
-  const setSelectedOrgId = useAppStore((s) => s.setSelectedOrgId);
+  const setselectedProjectId = useAppStore((s) => s.setselectedProjectId);
   const [gearHovered, setGearHovered] = useState(false);
   return (
     <HStack
       as={Link}
       href="/experiments"
       onClick={() => {
-        setSelectedOrgId(org.id);
+        setselectedProjectId(proj.id);
         onClose();
       }}
       w="full"
@@ -158,12 +149,14 @@ const ProjectOption = ({
       bgColor={isActive ? "gray.100" : "transparent"}
       _hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
       p={2}
+      borderRadius={4}
+      spacing={4}
     >
-      <Text>{org.name}</Text>
+      <Text>{proj.name}</Text>
       <IconButton
         as={Link}
         href="/project/settings"
-        aria-label={`Open ${org.name} settings`}
+        aria-label={`Open ${proj.name} settings`}
         icon={<Icon as={BsGear} boxSize={5} strokeWidth={0.5} color="gray.500" />}
         variant="ghost"
         size="xs"

app/src/components/nav/UserMenu.tsx

@@ -9,7 +9,6 @@ import {
   PopoverContent,
   Link,
   type StackProps,
-  Box,
 } from "@chakra-ui/react";
 import { type Session } from "next-auth";
 import { signOut } from "next-auth/react";
@@ -27,30 +26,28 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
     <>
       <Popover placement="right">
         <PopoverTrigger>
-          <Box>
-            <NavSidebarOption>
-              <HStack
-                // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
-                py={2}
-                px={1}
-                spacing={3}
-                {...rest}
-              >
-                {profileImage}
-                <VStack spacing={0} align="start" flex={1} flexShrink={1}>
-                  <Text fontWeight="bold" fontSize="sm">
-                    {user.user.name}
-                  </Text>
-                  <Text color="gray.500" fontSize="xs">
-                    {/* {user.user.email} */}
-                  </Text>
-                </VStack>
-                <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
-              </HStack>
-            </NavSidebarOption>
-          </Box>
+          <NavSidebarOption>
+            <HStack
+              // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
+              py={2}
+              px={1}
+              spacing={3}
+              {...rest}
+            >
+              {profileImage}
+              <VStack spacing={0} align="start" flex={1} flexShrink={1}>
+                <Text fontWeight="bold" fontSize="sm">
+                  {user.user.name}
+                </Text>
+                <Text color="gray.500" fontSize="xs">
+                  {/* {user.user.email} */}
+                </Text>
+              </VStack>
+              <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+            </HStack>
+          </NavSidebarOption>
         </PopoverTrigger>
-        <PopoverContent _focusVisible={{ boxShadow: "unset", outline: "unset" }} maxW="200px">
+        <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} minW={48} w="full">
           <VStack align="stretch" spacing={0}>
             {/* sign out */}
             <HStack

app/src/components/projectSettings/DeleteProjectDialog.tsx

@@ -16,7 +16,7 @@ import {
 import { useRouter } from "next/router";
 import { useRef, useState } from "react";
 import { api } from "~/utils/api";
-import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
 
 export const DeleteProjectDialog = ({
   isOpen,
@@ -25,20 +25,20 @@ export const DeleteProjectDialog = ({
   isOpen: boolean;
   onClose: () => void;
 }) => {
-  const selectedOrg = useSelectedOrg();
-  const deleteMutation = api.organizations.delete.useMutation();
+  const selectedProject = useSelectedProject();
+  const deleteMutation = api.projects.delete.useMutation();
   const utils = api.useContext();
   const router = useRouter();
 
   const cancelRef = useRef<HTMLButtonElement>(null);
 
   const [onDeleteConfirm, isDeleting] = useHandledAsyncCallback(async () => {
-    if (!selectedOrg.data?.id) return;
-    await deleteMutation.mutateAsync({ id: selectedOrg.data.id });
-    await utils.organizations.list.invalidate();
+    if (!selectedProject.data?.id) return;
+    await deleteMutation.mutateAsync({ id: selectedProject.data.id });
+    await utils.projects.list.invalidate();
     await router.push({ pathname: "/experiments" });
     onClose();
-  }, [deleteMutation, selectedOrg, router]);
+  }, [deleteMutation, selectedProject, router]);
 
   const [nameToDelete, setNameToDelete] = useState("");
 
@@ -58,10 +58,10 @@ export const DeleteProjectDialog = ({
                 of the project below.
               </Text>
               <Box bgColor="orange.100" w="full" p={2} borderRadius={4}>
-                <Text fontFamily="inconsolata">{selectedOrg.data?.name}</Text>
+                <Text fontFamily="inconsolata">{selectedProject.data?.name}</Text>
               </Box>
               <Input
-                placeholder={selectedOrg.data?.name}
+                placeholder={selectedProject.data?.name}
                 value={nameToDelete}
                 onChange={(e) => setNameToDelete(e.target.value)}
               />
@@ -76,7 +76,7 @@ export const DeleteProjectDialog = ({
               colorScheme="red"
               onClick={onDeleteConfirm}
               ml={3}
-              isDisabled={nameToDelete !== selectedOrg.data?.name}
+              isDisabled={nameToDelete !== selectedProject.data?.name}
               w={20}
             >
               {isDeleting ? <Spinner /> : "Delete"}

app/src/components/tooltip/CostTooltip.tsx

@@ -2,14 +2,14 @@ import { HStack, Icon, Text, Tooltip, type TooltipProps, VStack, Divider } from
 import { BsCurrencyDollar } from "react-icons/bs";
 
 type CostTooltipProps = {
-  promptTokens: number | null;
-  completionTokens: number | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
   cost: number;
 } & TooltipProps;
 
 export const CostTooltip = ({
-  promptTokens,
-  completionTokens,
+  inputTokens,
+  outputTokens,
   cost,
   children,
   ...props
@@ -36,12 +36,12 @@ export const CostTooltip = ({
             <HStack>
               <VStack w="28" spacing={1}>
                 <Text>Prompt</Text>
-                <Text>{promptTokens ?? 0}</Text>
+                <Text>{inputTokens ?? 0}</Text>
               </VStack>
               <Divider borderColor="gray.200" h={8} orientation="vertical" />
               <VStack w="28" spacing={1}>
                 <Text whiteSpace="nowrap">Completion</Text>
-                <Text>{completionTokens ?? 0}</Text>
+                <Text>{outputTokens ?? 0}</Text>
               </VStack>
             </HStack>
           </VStack>

app/src/modelProviders/anthropic-completion/codegen/input.schema.json

@@ -9,7 +9,8 @@
         "claude-2",
         "claude-2.0",
         "claude-instant-1",
-        "claude-instant-1.1"
+        "claude-instant-1.1",
+        "claude-instant-1.2"
       ]
     },
     "prompt": {

app/src/modelProviders/anthropic-completion/index.ts

@@ -28,6 +28,10 @@ const modelProvider: AnthropicProvider = {
   inputSchema: inputSchema as JSONSchema4,
   canStream: true,
   getCompletion,
+  getUsage: (input, output) => {
+    // TODO: add usage logic
+    return null;
+  },
   ...frontendModelProvider,
 };
 

app/src/modelProviders/openai-ChatCompletion/getCompletion.ts

@@ -4,11 +4,9 @@ import {
   type ChatCompletion,
   type CompletionCreateParams,
 } from "openai/resources/chat";
-import { countOpenAIChatTokens } from "~/utils/countTokens";
 import { type CompletionResponse } from "../types";
 import { isArray, isString, omit } from "lodash-es";
 import { openai } from "~/server/utils/openai";
-import { truthyFilter } from "~/utils/utils";
 import { APIError } from "openai";
 import frontendModelProvider from "./frontend";
 import modelProvider, { type SupportedModel } from ".";
@@ -60,9 +58,6 @@ export async function getCompletion(
 ): Promise<CompletionResponse<ChatCompletion>> {
   const start = Date.now();
   let finalCompletion: ChatCompletion | null = null;
-  let promptTokens: number | undefined = undefined;
-  let completionTokens: number | undefined = undefined;
-  const modelName = modelProvider.getModel(input) as SupportedModel;
 
   try {
     if (onStream) {
@@ -86,16 +81,6 @@ export async function getCompletion(
           autoRetry: false,
         };
       }
-      try {
-        promptTokens = countOpenAIChatTokens(modelName, input.messages);
-        completionTokens = countOpenAIChatTokens(
-          modelName,
-          finalCompletion.choices.map((c) => c.message).filter(truthyFilter),
-        );
-      } catch (err) {
-        // TODO handle this, library seems like maybe it doesn't work with function calls?
-        console.error(err);
-      }
     } else {
       const resp = await openai.chat.completions.create(
         { ...input, stream: false },
@@ -104,25 +89,14 @@ export async function getCompletion(
         },
       );
       finalCompletion = resp;
-      promptTokens = resp.usage?.prompt_tokens ?? 0;
-      completionTokens = resp.usage?.completion_tokens ?? 0;
     }
     const timeToComplete = Date.now() - start;
 
-    const { promptTokenPrice, completionTokenPrice } = frontendModelProvider.models[modelName];
-    let cost = undefined;
-    if (promptTokenPrice && completionTokenPrice && promptTokens && completionTokens) {
-      cost = promptTokens * promptTokenPrice + completionTokens * completionTokenPrice;
-    }
-
     return {
       type: "success",
       statusCode: 200,
       value: finalCompletion,
       timeToComplete,
-      promptTokens,
-      completionTokens,
-      cost,
     };
   } catch (error: unknown) {
     if (error instanceof APIError) {

app/src/modelProviders/openai-ChatCompletion/index.ts

@@ -4,6 +4,8 @@ import inputSchema from "./codegen/input.schema.json";
 import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
 import { getCompletion } from "./getCompletion";
 import frontendModelProvider from "./frontend";
+import { countOpenAIChatTokens } from "~/utils/countTokens";
+import { truthyFilter } from "~/utils/utils";
 
 const supportedModels = [
   "gpt-4-0613",
@@ -39,6 +41,41 @@ const modelProvider: OpenaiChatModelProvider = {
   inputSchema: inputSchema as JSONSchema4,
   canStream: true,
   getCompletion,
+  getUsage: (input, output) => {
+    if (output.choices.length === 0) return null;
+
+    const model = modelProvider.getModel(input);
+    if (!model) return null;
+
+    let inputTokens: number;
+    let outputTokens: number;
+
+    if (output.usage) {
+      inputTokens = output.usage.prompt_tokens;
+      outputTokens = output.usage.completion_tokens;
+    } else {
+      try {
+        inputTokens = countOpenAIChatTokens(model, input.messages);
+        outputTokens = countOpenAIChatTokens(
+          model,
+          output.choices.map((c) => c.message).filter(truthyFilter),
+        );
+      } catch (err) {
+        inputTokens = 0;
+        outputTokens = 0;
+        // TODO handle this, library seems like maybe it doesn't work with function calls?
+        console.error(err);
+      }
+    }
+
+    const { promptTokenPrice, completionTokenPrice } = frontendModelProvider.models[model];
+    let cost = undefined;
+    if (promptTokenPrice && completionTokenPrice && inputTokens && outputTokens) {
+      cost = inputTokens * promptTokenPrice + outputTokens * completionTokenPrice;
+    }
+
+    return { inputTokens: inputTokens, outputTokens: outputTokens, cost };
+  },
   ...frontendModelProvider,
 };
 

app/src/modelProviders/replicate-llama2/index.ts

@@ -75,6 +75,10 @@ const modelProvider: ReplicateLlama2Provider = {
   },
   canStream: true,
   getCompletion,
+  getUsage: (input, output) => {
+    // TODO: add usage logic
+    return null;
+  },
   ...frontendModelProvider,
 };
 

app/src/modelProviders/types.ts

@@ -43,9 +43,6 @@ export type CompletionResponse<T> =
       value: T;
       timeToComplete: number;
       statusCode: number;
-      promptTokens?: number;
-      completionTokens?: number;
-      cost?: number;
     };
 
 export type ModelProvider<SupportedModels extends string, InputSchema, OutputSchema> = {
@@ -56,6 +53,10 @@ export type ModelProvider<SupportedModels extends string, InputSchema, OutputSch
     input: InputSchema,
     onStream: ((partialOutput: OutputSchema) => void) | null,
   ) => Promise<CompletionResponse<OutputSchema>>;
+  getUsage: (
+    input: InputSchema,
+    output: OutputSchema,
+  ) => { gpuRuntime?: number; inputTokens?: number; outputTokens?: number; cost?: number } | null;
 
   // This is just a convenience for type inference, don't use it at runtime
   _outputSchema?: OutputSchema | null;

app/src/pages/data/[id].tsx

@@ -60,7 +60,7 @@ export default function Dataset() {
         <PageHeaderContainer>
           <Breadcrumb>
             <BreadcrumbItem>
-              <ProjectBreadcrumbContents orgName={dataset.data?.organization?.name} />
+              <ProjectBreadcrumbContents projectName={dataset.data?.project?.name} />
             </BreadcrumbItem>
             <BreadcrumbItem>
               <Link href="/data">

app/src/pages/experiments/[id].tsx

@@ -109,7 +109,7 @@ export default function Experiment() {
           <PageHeaderContainer>
             <Breadcrumb>
               <BreadcrumbItem>
-                <ProjectBreadcrumbContents orgName={experiment.data?.organization?.name} />
+                <ProjectBreadcrumbContents projectName={experiment.data?.project?.name} />
               </BreadcrumbItem>
               <BreadcrumbItem>
                 <Link href="/experiments">

app/src/pages/logged-calls/index.tsx

@@ -34,17 +34,17 @@ import { useMemo } from "react";
 import AppShell from "~/components/nav/AppShell";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
 import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
-import { useSelectedOrg } from "~/utils/hooks";
+import { useSelectedProject } from "~/utils/hooks";
 import dayjs from "~/utils/dayjs";
 import { api } from "~/utils/api";
 import LoggedCallTable from "~/components/dashboard/LoggedCallTable";
 
 export default function LoggedCalls() {
-  const { data: selectedOrg } = useSelectedOrg();
+  const { data: selectedProject } = useSelectedProject();
 
   const stats = api.dashboard.stats.useQuery(
-    { organizationId: selectedOrg?.id ?? "" },
-    { enabled: !!selectedOrg },
+    { projectId: selectedProject?.id ?? "" },
+    { enabled: !!selectedProject },
   );
 
   const data = useMemo(() => {
@@ -71,7 +71,7 @@ export default function LoggedCalls() {
       </PageHeaderContainer>
       <VStack px={8} pt={4} alignItems="flex-start" spacing={4}>
         <Text fontSize="2xl" fontWeight="bold">
-          {selectedOrg?.name}
+          {selectedProject?.name}
         </Text>
         <Divider />
         <VStack margin="auto" spacing={4} align="stretch" w="full">

app/src/pages/project/settings/index.tsx

@@ -5,7 +5,6 @@ import {
   type TextProps,
   VStack,
   HStack,
-  Input,
   Button,
   Divider,
   Icon,
@@ -17,33 +16,36 @@ import { BsTrash } from "react-icons/bs";
 import AppShell from "~/components/nav/AppShell";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
 import { api } from "~/utils/api";
-import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
 import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 import CopiableCode from "~/components/CopiableCode";
 import { DeleteProjectDialog } from "~/components/projectSettings/DeleteProjectDialog";
+import AutoResizeTextArea from "~/components/AutoResizeTextArea";
 
 export default function Settings() {
   const utils = api.useContext();
-  const { data: selectedOrg } = useSelectedOrg();
+  const { data: selectedProject } = useSelectedProject();
 
   const apiKey =
-    selectedOrg?.apiKeys?.length && selectedOrg?.apiKeys[0] ? selectedOrg?.apiKeys[0].apiKey : "";
+    selectedProject?.apiKeys?.length && selectedProject?.apiKeys[0]
+      ? selectedProject?.apiKeys[0].apiKey
+      : "";
 
-  const updateMutation = api.organizations.update.useMutation();
+  const updateMutation = api.projects.update.useMutation();
   const [onSaveName] = useHandledAsyncCallback(async () => {
-    if (name && name !== selectedOrg?.name && selectedOrg?.id) {
+    if (name && name !== selectedProject?.name && selectedProject?.id) {
       await updateMutation.mutateAsync({
-        id: selectedOrg.id,
+        id: selectedProject.id,
         updates: { name },
       });
-      await Promise.all([utils.organizations.get.invalidate({ id: selectedOrg.id })]);
+      await Promise.all([utils.projects.get.invalidate({ id: selectedProject.id })]);
     }
-  }, [updateMutation, selectedOrg]);
+  }, [updateMutation, selectedProject]);
 
-  const [name, setName] = useState(selectedOrg?.name);
+  const [name, setName] = useState(selectedProject?.name);
   useEffect(() => {
-    setName(selectedOrg?.name);
-  }, [selectedOrg?.name]);
+    setName(selectedProject?.name);
+  }, [selectedProject?.name]);
 
   const deleteProjectOpen = useDisclosure();
 
@@ -66,7 +68,7 @@ export default function Settings() {
               Project Settings
             </Text>
             <Text fontSize="sm">
-              Configure your project settings. These settings only apply to {selectedOrg?.name}.
+              Configure your project settings. These settings only apply to {selectedProject?.name}.
             </Text>
           </VStack>
           <VStack
@@ -82,7 +84,7 @@ export default function Settings() {
               <Text fontWeight="bold" fontSize="xl">
                 Display Name
               </Text>
-              <Input
+              <AutoResizeTextArea
                 w="full"
                 maxW={600}
                 value={name}
@@ -90,7 +92,7 @@ export default function Settings() {
                 borderColor="gray.300"
               />
               <Button
-                isDisabled={!name || name === selectedOrg?.name}
+                isDisabled={!name || name === selectedProject?.name}
                 colorScheme="orange"
                 borderRadius={4}
                 mt={2}
@@ -113,12 +115,12 @@ export default function Settings() {
             </VStack>
             <CopiableCode code={apiKey} />
             <Divider />
-            {selectedOrg?.personalOrgUserId ? (
+            {selectedProject?.personalProjectUserId ? (
               <VStack alignItems="flex-start">
                 <Subtitle>Personal Project</Subtitle>
                 <Text fontSize="sm">
-                  This project is {selectedOrg?.personalOrgUser?.name}'s personal project. It cannot
-                  be deleted.
+                  This project is {selectedProject?.personalProjectUser?.name}'s personal project.
+                  It cannot be deleted.
                 </Text>
               </VStack>
             ) : (
@@ -129,15 +131,18 @@ export default function Settings() {
                 </Text>
                 <HStack
                   as={Button}
-                  isDisabled={selectedOrg?.role !== "ADMIN"}
+                  isDisabled={selectedProject?.role !== "ADMIN"}
                   colorScheme="red"
                   variant="outline"
                   borderRadius={4}
                   mt={2}
+                  height="auto"
                   onClick={deleteProjectOpen.onOpen}
                 >
                   <Icon as={BsTrash} />
-                  <Text>Delete {selectedOrg?.name}</Text>
+                  <Text overflowWrap="break-word" whiteSpace="normal" py={2}>
+                    Delete {selectedProject?.name}
+                  </Text>
                 </HStack>
               </VStack>
             )}

app/src/server/api/root.router.ts

@@ -3,13 +3,13 @@ import { createTRPCRouter } from "~/server/api/trpc";
 import { experimentsRouter } from "./routers/experiments.router";
 import { scenariosRouter } from "./routers/scenarios.router";
 import { scenarioVariantCellsRouter } from "./routers/scenarioVariantCells.router";
-import { templateVarsRouter } from "./routers/templateVariables.router";
+import { scenarioVarsRouter } from "./routers/scenarioVariables.router";
 import { evaluationsRouter } from "./routers/evaluations.router";
 import { worldChampsRouter } from "./routers/worldChamps.router";
 import { datasetsRouter } from "./routers/datasets.router";
 import { datasetEntries } from "./routers/datasetEntries.router";
 import { externalApiRouter } from "./routers/externalApi.router";
-import { organizationsRouter } from "./routers/organizations.router";
+import { projectsRouter } from "./routers/projects.router";
 import { dashboardRouter } from "./routers/dashboard.router";
 
 /**
@@ -22,12 +22,12 @@ export const appRouter = createTRPCRouter({
   experiments: experimentsRouter,
   scenarios: scenariosRouter,
   scenarioVariantCells: scenarioVariantCellsRouter,
-  templateVars: templateVarsRouter,
+  scenarioVars: scenarioVarsRouter,
   evaluations: evaluationsRouter,
   worldChamps: worldChampsRouter,
   datasets: datasetsRouter,
   datasetEntries: datasetEntries,
-  organizations: organizationsRouter,
+  projects: projectsRouter,
   dashboard: dashboardRouter,
   externalApi: externalApiRouter,
 });

app/src/server/api/routers/dashboard.router.ts

@@ -10,7 +10,7 @@ export const dashboardRouter = createTRPCRouter({
       z.object({
         // TODO: actually take startDate into account
         startDate: z.string().optional(),
-        organizationId: z.string(),
+        projectId: z.string(),
       }),
     )
     .query(async ({ input }) => {
@@ -22,7 +22,7 @@ export const dashboardRouter = createTRPCRouter({
           "LoggedCall.id",
           "LoggedCallModelResponse.originalLoggedCallId",
         )
-        .where("organizationId", "=", input.organizationId)
+        .where("projectId", "=", input.projectId)
         .select(({ fn }) => [
           sql<Date>`date_trunc('day', "LoggedCallModelResponse"."startTime")`.as("period"),
           sql<number>`count("LoggedCall"."id")::int`.as("numQueries"),
@@ -70,7 +70,7 @@ export const dashboardRouter = createTRPCRouter({
           "LoggedCall.id",
           "LoggedCallModelResponse.originalLoggedCallId",
         )
-        .where("organizationId", "=", input.organizationId)
+        .where("projectId", "=", input.projectId)
         .select(({ fn }) => [
           fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
           fn.count("LoggedCall.id").as("numQueries"),
@@ -79,7 +79,7 @@ export const dashboardRouter = createTRPCRouter({
 
       const errors = await kysely
         .selectFrom("LoggedCall")
-        .where("organizationId", "=", input.organizationId)
+        .where("projectId", "=", input.projectId)
         .leftJoin(
           "LoggedCallModelResponse",
           "LoggedCall.id",

app/src/server/api/routers/datasets.router.ts

@@ -3,20 +3,20 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
 import { prisma } from "~/server/db";
 import {
   requireCanModifyDataset,
-  requireCanModifyOrganization,
+  requireCanModifyProject,
   requireCanViewDataset,
-  requireCanViewOrganization,
+  requireCanViewProject,
 } from "~/utils/accessControl";
 
 export const datasetsRouter = createTRPCRouter({
   list: protectedProcedure
-    .input(z.object({ organizationId: z.string() }))
+    .input(z.object({ projectId: z.string() }))
     .query(async ({ input, ctx }) => {
-      await requireCanViewOrganization(input.organizationId, ctx);
+      await requireCanViewProject(input.projectId, ctx);
 
       const datasets = await prisma.dataset.findMany({
         where: {
-          organizationId: input.organizationId,
+          projectId: input.projectId,
         },
         orderBy: {
           createdAt: "desc",
@@ -36,26 +36,26 @@ export const datasetsRouter = createTRPCRouter({
     return await prisma.dataset.findFirstOrThrow({
       where: { id: input.id },
       include: {
-        organization: true,
+        project: true,
       },
     });
   }),
 
   create: protectedProcedure
-    .input(z.object({ organizationId: z.string() }))
+    .input(z.object({ projectId: z.string() }))
     .mutation(async ({ input, ctx }) => {
-      await requireCanModifyOrganization(input.organizationId, ctx);
+      await requireCanModifyProject(input.projectId, ctx);
 
       const numDatasets = await prisma.dataset.count({
         where: {
-          organizationId: input.organizationId,
+          projectId: input.projectId,
         },
       });
 
       return await prisma.dataset.create({
         data: {
           name: `Dataset ${numDatasets + 1}`,
-          organizationId: input.organizationId,
+          projectId: input.projectId,
         },
       });
     }),

app/src/server/api/routers/experiments.router.ts

@@ -8,9 +8,9 @@ import { generateNewCell } from "~/server/utils/generateNewCell";
 import {
   canModifyExperiment,
   requireCanModifyExperiment,
-  requireCanModifyOrganization,
+  requireCanModifyProject,
   requireCanViewExperiment,
-  requireCanViewOrganization,
+  requireCanViewProject,
 } from "~/utils/accessControl";
 import generateTypes from "~/modelProviders/generateTypes";
 import { promptConstructorVersion } from "~/promptConstructor/version";
@@ -44,13 +44,13 @@ export const experimentsRouter = createTRPCRouter({
     };
   }),
   list: protectedProcedure
-    .input(z.object({ organizationId: z.string() }))
+    .input(z.object({ projectId: z.string() }))
     .query(async ({ input, ctx }) => {
-      await requireCanViewOrganization(input.organizationId, ctx);
+      await requireCanViewProject(input.projectId, ctx);
 
       const experiments = await prisma.experiment.findMany({
         where: {
-          organizationId: input.organizationId,
+          projectId: input.projectId,
         },
         orderBy: {
           sortIndex: "desc",
@@ -90,7 +90,7 @@ export const experimentsRouter = createTRPCRouter({
     const experiment = await prisma.experiment.findFirstOrThrow({
       where: { id: input.id },
       include: {
-        organization: true,
+        project: true,
       },
     });
 
@@ -108,10 +108,10 @@ export const experimentsRouter = createTRPCRouter({
   }),
 
   fork: protectedProcedure
-    .input(z.object({ id: z.string(), organizationId: z.string() }))
+    .input(z.object({ id: z.string(), projectId: z.string() }))
     .mutation(async ({ input, ctx }) => {
       await requireCanViewExperiment(input.id, ctx);
-      await requireCanModifyOrganization(input.organizationId, ctx);
+      await requireCanModifyProject(input.projectId, ctx);
 
       const [
         existingExp,
@@ -264,7 +264,7 @@ export const experimentsRouter = createTRPCRouter({
             id: newExperimentId,
             sortIndex: maxSortIndex + 1,
             label: `${existingExp.label} (forked)`,
-            organizationId: input.organizationId,
+            projectId: input.projectId,
           },
         }),
         prisma.promptVariant.createMany({
@@ -294,9 +294,9 @@ export const experimentsRouter = createTRPCRouter({
     }),
 
   create: protectedProcedure
-    .input(z.object({ organizationId: z.string() }))
+    .input(z.object({ projectId: z.string() }))
     .mutation(async ({ input, ctx }) => {
-      await requireCanModifyOrganization(input.organizationId, ctx);
+      await requireCanModifyProject(input.projectId, ctx);
 
       const maxSortIndex =
         (
@@ -304,7 +304,7 @@ export const experimentsRouter = createTRPCRouter({
             _max: {
               sortIndex: true,
             },
-            where: { organizationId: input.organizationId },
+            where: { projectId: input.projectId },
           })
         )._max?.sortIndex ?? 0;
 
@@ -312,7 +312,7 @@ export const experimentsRouter = createTRPCRouter({
         data: {
           sortIndex: maxSortIndex + 1,
           label: `Experiment ${maxSortIndex + 1}`,
-          organizationId: input.organizationId,
+          projectId: input.projectId,
         },
       });
 

app/src/server/api/routers/externalApi.router.ts

@@ -66,7 +66,7 @@ export const externalApiRouter = createTRPCRouter({
         throw new TRPCError({ code: "UNAUTHORIZED" });
       }
       const reqPayload = await reqValidator.spa(input.reqPayload);
-      const cacheKey = hashRequest(key.organizationId, reqPayload as JsonValue);
+      const cacheKey = hashRequest(key.projectId, reqPayload as JsonValue);
 
       const existingResponse = await prisma.loggedCallModelResponse.findFirst({
         where: {
@@ -84,7 +84,7 @@ export const externalApiRouter = createTRPCRouter({
 
       await prisma.loggedCall.create({
         data: {
-          organizationId: key.organizationId,
+          projectId: key.projectId,
           startTime: new Date(input.startTime),
           cacheHit: true,
           modelResponseId: existingResponse.id,
@@ -135,7 +135,7 @@ export const externalApiRouter = createTRPCRouter({
       const reqPayload = await reqValidator.spa(input.reqPayload);
       const respPayload = await respValidator.spa(input.respPayload);
 
-      const requestHash = hashRequest(key.organizationId, reqPayload as JsonValue);
+      const requestHash = hashRequest(key.projectId, reqPayload as JsonValue);
 
       const newLoggedCallId = uuidv4();
       const newModelResponseId = uuidv4();
@@ -146,7 +146,7 @@ export const externalApiRouter = createTRPCRouter({
         prisma.loggedCall.create({
           data: {
             id: newLoggedCallId,
-            organizationId: key.organizationId,
+            projectId: key.projectId,
             startTime: new Date(input.startTime),
             cacheHit: false,
           },

app/src/server/api/routers/projects.router.ts

@@ -5,15 +5,15 @@ import { z } from "zod";
 import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
 import { generateApiKey } from "~/server/utils/generateApiKey";
-import userOrg from "~/server/utils/userOrg";
+import userProject from "~/server/utils/userProject";
 import {
-  requireCanModifyOrganization,
-  requireCanViewOrganization,
-  requireIsOrgAdmin,
+  requireCanModifyProject,
+  requireCanViewProject,
+  requireIsProjectAdmin,
   requireNothing,
 } from "~/utils/accessControl";
 
-export const organizationsRouter = createTRPCRouter({
+export const projectsRouter = createTRPCRouter({
   list: protectedProcedure.query(async ({ ctx }) => {
     const userId = ctx.session.user.id;
     requireNothing(ctx);
@@ -22,9 +22,9 @@ export const organizationsRouter = createTRPCRouter({
       return null;
     }
 
-    const organizations = await prisma.organization.findMany({
+    const projects = await prisma.project.findMany({
       where: {
-        organizationUsers: {
+        projectUsers: {
           some: { userId: ctx.session.user.id },
         },
       },
@@ -33,30 +33,30 @@ export const organizationsRouter = createTRPCRouter({
       },
     });
 
-    if (!organizations.length) {
+    if (!projects.length) {
       // TODO: We should move this to a separate endpoint that is called on sign up
-      const personalOrg = await userOrg(userId);
-      organizations.push(personalOrg);
+      const personalProject = await userProject(userId);
+      projects.push(personalProject);
     }
 
-    return organizations;
+    return projects;
   }),
   get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
-    await requireCanViewOrganization(input.id, ctx);
-    const [org, userRole] = await prisma.$transaction([
-      prisma.organization.findUnique({
+    await requireCanViewProject(input.id, ctx);
+    const [proj, userRole] = await prisma.$transaction([
+      prisma.project.findUnique({
         where: {
           id: input.id,
         },
         include: {
           apiKeys: true,
-          personalOrgUser: true,
+          personalProjectUser: true,
         },
       }),
-      prisma.organizationUser.findFirst({
+      prisma.projectUser.findFirst({
         where: {
           userId: ctx.session.user.id,
-          organizationId: input.id,
+          projectId: input.id,
           role: {
             in: ["ADMIN", "MEMBER"],
           },
@@ -64,20 +64,20 @@ export const organizationsRouter = createTRPCRouter({
       }),
     ]);
 
-    if (!org) {
+    if (!proj) {
       throw new TRPCError({ code: "NOT_FOUND" });
     }
 
     return {
-      ...org,
+      ...proj,
       role: userRole?.role ?? null,
     };
   }),
   update: protectedProcedure
     .input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))
     .mutation(async ({ input, ctx }) => {
-      await requireCanModifyOrganization(input.id, ctx);
-      return await prisma.organization.update({
+      await requireCanModifyProject(input.id, ctx);
+      return await prisma.project.update({
         where: {
           id: input.id,
         },
@@ -90,36 +90,36 @@ export const organizationsRouter = createTRPCRouter({
     .input(z.object({ name: z.string() }))
     .mutation(async ({ input, ctx }) => {
       requireNothing(ctx);
-      const newOrgId = uuidv4();
-      const [newOrg] = await prisma.$transaction([
-        prisma.organization.create({
+      const newProjectId = uuidv4();
+      const [newProject] = await prisma.$transaction([
+        prisma.project.create({
           data: {
-            id: newOrgId,
+            id: newProjectId,
             name: input.name,
           },
         }),
-        prisma.organizationUser.create({
+        prisma.projectUser.create({
           data: {
             userId: ctx.session.user.id,
-            organizationId: newOrgId,
+            projectId: newProjectId,
             role: "ADMIN",
           },
         }),
         prisma.apiKey.create({
           data: {
             name: "Default API Key",
-            organizationId: newOrgId,
+            projectId: newProjectId,
             apiKey: generateApiKey(),
           },
         }),
       ]);
-      return newOrg;
+      return newProject;
     }),
   delete: protectedProcedure
     .input(z.object({ id: z.string() }))
     .mutation(async ({ input, ctx }) => {
-      await requireIsOrgAdmin(input.id, ctx);
-      return await prisma.organization.delete({
+      await requireIsProjectAdmin(input.id, ctx);
+      return await prisma.project.delete({
         where: {
           id: input.id,
         },

app/src/server/api/routers/promptVariants.router.ts

@@ -3,7 +3,7 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
 import { prisma } from "~/server/db";
 import { Prisma } from "@prisma/client";
 import { generateNewCell } from "~/server/utils/generateNewCell";
-import userError from "~/server/utils/error";
+import { error, success } from "~/utils/standardResponses";
 import { recordExperimentUpdated } from "~/server/utils/recordExperimentUpdated";
 import { reorderPromptVariants } from "~/server/utils/reorderPromptVariants";
 import { type PromptVariant } from "@prisma/client";
@@ -123,13 +123,13 @@ export const promptVariantsRouter = createTRPCRouter({
         },
         _sum: {
           cost: true,
-          promptTokens: true,
-          completionTokens: true,
+          inputTokens: true,
+          outputTokens: true,
         },
       });
 
-      const promptTokens = overallTokens._sum?.promptTokens ?? 0;
-      const completionTokens = overallTokens._sum?.completionTokens ?? 0;
+      const inputTokens = overallTokens._sum?.inputTokens ?? 0;
+      const outputTokens = overallTokens._sum?.outputTokens ?? 0;
 
       const awaitingEvals = !!evalResults.find(
         (result) => result.totalCount < scenarioCount * evals.length,
@@ -137,8 +137,8 @@ export const promptVariantsRouter = createTRPCRouter({
 
       return {
         evalResults,
-        promptTokens,
-        completionTokens,
+        inputTokens,
+        outputTokens,
         overallCost: overallTokens._sum?.cost ?? 0,
         scenarioCount,
         outputCount,
@@ -315,7 +315,7 @@ export const promptVariantsRouter = createTRPCRouter({
       const constructedPrompt = await parsePromptConstructor(existing.promptConstructor);
 
       if ("error" in constructedPrompt) {
-        return userError(constructedPrompt.error);
+        return error(constructedPrompt.error);
       }
 
       const model = input.newModel
@@ -353,7 +353,7 @@ export const promptVariantsRouter = createTRPCRouter({
       const parsedPrompt = await parsePromptConstructor(input.promptConstructor);
 
       if ("error" in parsedPrompt) {
-        return userError(parsedPrompt.error);
+        return error(parsedPrompt.error);
       }
 
       // Create a duplicate with only the config changed
@@ -398,7 +398,7 @@ export const promptVariantsRouter = createTRPCRouter({
         });
       }
 
-      return { status: "ok" } as const;
+      return success();
     }),
 
   reorder: protectedProcedure

app/src/server/api/routers/scenarioVariables.router.ts

@@ -0,0 +1,143 @@
+import { type TemplateVariable } from "@prisma/client";
+import { sql } from "kysely";
+import { z } from "zod";
+import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
+import { kysely, prisma } from "~/server/db";
+import { error, success } from "~/utils/standardResponses";
+import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl";
+
+export const scenarioVarsRouter = createTRPCRouter({
+  create: protectedProcedure
+    .input(z.object({ experimentId: z.string(), label: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyExperiment(input.experimentId, ctx);
+
+      // Make sure there isn't an existing variable with the same name
+      const existingVariable = await prisma.templateVariable.findFirst({
+        where: {
+          experimentId: input.experimentId,
+          label: input.label,
+        },
+      });
+      if (existingVariable) {
+        return error(`A variable named ${input.label} already exists.`);
+      }
+
+      await prisma.templateVariable.create({
+        data: {
+          experimentId: input.experimentId,
+          label: input.label,
+        },
+      });
+
+      return success();
+    }),
+
+  rename: protectedProcedure
+    .input(z.object({ id: z.string(), label: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const templateVariable = await prisma.templateVariable.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+      await requireCanModifyExperiment(templateVariable.experimentId, ctx);
+
+      // Make sure there isn't an existing variable with the same name
+      const existingVariable = await prisma.templateVariable.findFirst({
+        where: {
+          experimentId: templateVariable.experimentId,
+          label: input.label,
+        },
+      });
+      if (existingVariable) {
+        return error(`A variable named ${input.label} already exists.`);
+      }
+
+      await renameTemplateVariable(templateVariable, input.label);
+      return success();
+    }),
+
+  delete: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+
+      await requireCanModifyExperiment(experimentId, ctx);
+
+      await prisma.templateVariable.delete({ where: { id: input.id } });
+    }),
+
+  list: publicProcedure
+    .input(z.object({ experimentId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewExperiment(input.experimentId, ctx);
+      return await prisma.templateVariable.findMany({
+        where: {
+          experimentId: input.experimentId,
+        },
+        orderBy: {
+          createdAt: "asc",
+        },
+        select: {
+          id: true,
+          label: true,
+        },
+      });
+    }),
+});
+
+export const renameTemplateVariable = async (
+  templateVariable: TemplateVariable,
+  newLabel: string,
+) => {
+  const { experimentId } = templateVariable;
+
+  await kysely.transaction().execute(async (trx) => {
+    await trx
+      .updateTable("TemplateVariable")
+      .set({
+        label: newLabel,
+      })
+      .where("id", "=", templateVariable.id)
+      .execute();
+
+    await sql`
+      CREATE TEMP TABLE "TempTestScenario" AS
+      SELECT *
+      FROM "TestScenario"
+      WHERE "experimentId" = ${experimentId}
+
+      -- Only copy the rows that actually have a value for the variable, no reason to churn the rest and simplifies the update.
+      AND "variableValues"->${templateVariable.label} IS NOT NULL
+      `.execute(trx);
+
+    await sql`
+      UPDATE "TempTestScenario"
+      SET "variableValues" = jsonb_set(
+        "variableValues",
+        ${`{${newLabel}}`},
+        "variableValues"->${templateVariable.label}
+      ) - ${templateVariable.label},
+      "updatedAt" = NOW(),
+      "id" = uuid_generate_v4()
+    `.execute(trx);
+
+    // Print the contents of the temp table
+    const results = await sql`SELECT * FROM "TempTestScenario"`.execute(trx);
+    console.log(results.rows);
+
+    await trx
+      .updateTable("TestScenario")
+      .set({
+        visible: false,
+      })
+      .where("experimentId", "=", experimentId)
+      .execute();
+
+    await sql`
+      INSERT INTO "TestScenario" (id, "variableValues", "uiId", visible, "sortIndex", "experimentId", "createdAt", "updatedAt")
+      SELECT * FROM "TempTestScenario";
+    `.execute(trx);
+  });
+};

app/src/server/api/routers/templateVariables.router.test.ts

@@ -0,0 +1,110 @@
+import { expect, it } from "vitest";
+import { prisma } from "~/server/db";
+import { renameTemplateVariable } from "./scenarioVariables.router";
+
+const createExperiment = async () => {
+  return await prisma.experiment.create({
+    data: {
+      label: "Test Experiment",
+      project: {
+        create: {},
+      },
+    },
+  });
+};
+
+const createTemplateVar = async (experimentId: string, label: string) => {
+  return await prisma.templateVariable.create({
+    data: {
+      experimentId,
+      label,
+    },
+  });
+};
+
+it("renames templateVariables", async () => {
+  // Create experiments concurrently
+  const [exp1, exp2] = await Promise.all([createExperiment(), createExperiment()]);
+
+  // Create template variables concurrently
+  const [exp1Var, exp2Var1, exp2Var2] = await Promise.all([
+    createTemplateVar(exp1.id, "input1"),
+    createTemplateVar(exp2.id, "input1"),
+    createTemplateVar(exp2.id, "input2"),
+  ]);
+
+  // Create test scenarios concurrently
+  const [exp1Scenario, exp2Scenario, exp2HiddenScenario] = await Promise.all([
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp1.id,
+        visible: true,
+        variableValues: { input1: "test" },
+      },
+    }),
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp2.id,
+        visible: true,
+        variableValues: { input1: "test1", otherInput: "otherTest" },
+      },
+    }),
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp2.id,
+        visible: false,
+        variableValues: { otherInput: "otherTest2" },
+      },
+    }),
+  ]);
+
+  await renameTemplateVariable(exp2Var1, "input1-renamed");
+
+  expect(await prisma.templateVariable.findUnique({ where: { id: exp2Var1.id } })).toMatchObject({
+    label: "input1-renamed",
+  });
+
+  // It shouldn't mess with unrelated experiments
+  expect(await prisma.testScenario.findUnique({ where: { id: exp1Scenario.id } })).toMatchObject({
+    visible: true,
+    variableValues: { input1: "test" },
+  });
+
+  // Make sure there are a total of 4 scenarios for exp2
+  expect(
+    await prisma.testScenario.count({
+      where: {
+        experimentId: exp2.id,
+      },
+    }),
+  ).toBe(3);
+
+  // It shouldn't mess with the existing scenarios, except to hide them
+  expect(await prisma.testScenario.findUnique({ where: { id: exp2Scenario.id } })).toMatchObject({
+    visible: false,
+    variableValues: { input1: "test1", otherInput: "otherTest" },
+  });
+
+  // It should create a new scenario with the new variable name
+  const newScenario1 = await prisma.testScenario.findFirst({
+    where: {
+      experimentId: exp2.id,
+      variableValues: { equals: { "input1-renamed": "test1", otherInput: "otherTest" } },
+    },
+  });
+
+  expect(newScenario1).toMatchObject({
+    visible: true,
+  });
+
+  const newScenario2 = await prisma.testScenario.findFirst({
+    where: {
+      experimentId: exp2.id,
+      variableValues: { equals: { otherInput: "otherTest2" } },
+    },
+  });
+
+  expect(newScenario2).toMatchObject({
+    visible: false,
+  });
+});

app/src/server/api/routers/templateVariables.router.ts

@@ -1,49 +0,0 @@
-import { z } from "zod";
-import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
-import { prisma } from "~/server/db";
-import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl";
-
-export const templateVarsRouter = createTRPCRouter({
-  create: protectedProcedure
-    .input(z.object({ experimentId: z.string(), label: z.string() }))
-    .mutation(async ({ input, ctx }) => {
-      await requireCanModifyExperiment(input.experimentId, ctx);
-
-      await prisma.templateVariable.create({
-        data: {
-          experimentId: input.experimentId,
-          label: input.label,
-        },
-      });
-    }),
-
-  delete: protectedProcedure
-    .input(z.object({ id: z.string() }))
-    .mutation(async ({ input, ctx }) => {
-      const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({
-        where: { id: input.id },
-      });
-
-      await requireCanModifyExperiment(experimentId, ctx);
-
-      await prisma.templateVariable.delete({ where: { id: input.id } });
-    }),
-
-  list: publicProcedure
-    .input(z.object({ experimentId: z.string() }))
-    .query(async ({ input, ctx }) => {
-      await requireCanViewExperiment(input.experimentId, ctx);
-      return await prisma.templateVariable.findMany({
-        where: {
-          experimentId: input.experimentId,
-        },
-        orderBy: {
-          createdAt: "asc",
-        },
-        select: {
-          id: true,
-          label: true,
-        },
-      });
-    }),
-});

app/src/server/db.ts

@@ -9,8 +9,8 @@ import {
   type OutputEvaluation,
   type Dataset,
   type DatasetEntry,
-  type Organization,
-  type OrganizationUser,
+  type Project,
+  type ProjectUser,
   type WorldChampEntrant,
   type LoggedCall,
   type LoggedCallModelResponse,
@@ -43,8 +43,8 @@ interface DB {
   OutputEvaluation: OutputEvaluation;
   Dataset: Dataset;
   DatasetEntry: DatasetEntry;
-  Organization: Organization;
-  OrganizationUser: OrganizationUser;
+  Project: Project;
+  ProjectUser: ProjectUser;
   WorldChampEntrant: WorldChampEntrant;
   LoggedCall: LoggedCall;
   LoggedCallModelResponse: LoggedCallModelResponse;

app/src/server/scripts/backfillApiKeys.ts

@@ -4,21 +4,21 @@ import { generateApiKey } from "~/server/utils/generateApiKey";
 
 console.log("backfilling api keys");
 
-const organizations = await prisma.organization.findMany({
+const projects = await prisma.project.findMany({
   include: {
     apiKeys: true,
   },
 });
 
-console.log(`found ${organizations.length} organizations`);
+console.log(`found ${projects.length} projects`);
 
 const apiKeysToCreate: Prisma.ApiKeyCreateManyInput[] = [];
 
-for (const org of organizations) {
-  if (!org.apiKeys.length) {
+for (const proj of projects) {
+  if (!proj.apiKeys.length) {
     apiKeysToCreate.push({
       name: "Default API Key",
-      organizationId: org.id,
+      projectId: proj.id,
       apiKey: generateApiKey(),
     });
   }

app/src/server/scripts/test-queries.ts

@@ -1,63 +0,0 @@
-import dayjs from "dayjs";
-import { prisma } from "../db";
-
-const projectId = "1234";
-
-// Find all calls in the last 24 hours
-const responses = await prisma.loggedCall.findMany({
-  where: {
-    organizationId: projectId,
-    startTime: {
-      gt: dayjs()
-        .subtract(24 * 3600)
-        .toDate(),
-    },
-  },
-  include: {
-    modelResponse: true,
-  },
-  orderBy: {
-    startTime: "desc",
-  },
-});
-
-// Find all calls in the last 24 hours with promptId 'hello-world'
-const helloWorld = await prisma.loggedCall.findMany({
-  where: {
-    organizationId: projectId,
-    startTime: {
-      gt: dayjs()
-        .subtract(24 * 3600)
-        .toDate(),
-    },
-    tags: {
-      some: {
-        name: "promptId",
-        value: "hello-world",
-      },
-    },
-  },
-  include: {
-    modelResponse: true,
-  },
-  orderBy: {
-    startTime: "desc",
-  },
-});
-
-// Total spent on OpenAI in the last month
-const totalSpent = await prisma.loggedCallModelResponse.aggregate({
-  _sum: {
-    totalCost: true,
-  },
-  where: {
-    originalLoggedCall: {
-      organizationId: projectId,
-    },
-    startTime: {
-      gt: dayjs()
-        .subtract(30 * 24 * 3600)
-        .toDate(),
-    },
-  },
-});

app/src/server/tasks/queryModel.task.ts

@@ -110,15 +110,16 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
   });
   const response = await provider.getCompletion(prompt.modelInput, onStream);
   if (response.type === "success") {
+    const usage = provider.getUsage(prompt.modelInput, response.value);
     modelResponse = await prisma.modelResponse.update({
       where: { id: modelResponse.id },
       data: {
         output: response.value as Prisma.InputJsonObject,
         statusCode: response.statusCode,
         receivedAt: new Date(),
-        promptTokens: response.promptTokens,
-        completionTokens: response.completionTokens,
-        cost: response.cost,
+        inputTokens: usage?.inputTokens,
+        outputTokens: usage?.outputTokens,
+        cost: usage?.cost,
       },
     });
 

app/src/server/utils/error.ts

@@ -1,6 +0,0 @@
-export default function userError(message: string): { status: "error"; message: string } {
-  return {
-    status: "error",
-    message,
-  };
-}

app/src/server/utils/hashObject.ts

@@ -24,9 +24,9 @@ function sortKeys(obj: JsonValue): JsonValue {
   return sortedObj;
 }
 
-export function hashRequest(organizationId: string, reqPayload: JsonValue): string {
+export function hashRequest(projectId: string, reqPayload: JsonValue): string {
   const obj = {
-    organizationId,
+    projectId,
     reqPayload,
   };
   return hashObject(obj);

app/src/server/utils/userProject.ts

@@ -1,15 +1,15 @@
 import { prisma } from "~/server/db";
 import { generateApiKey } from "./generateApiKey";
 
-export default async function userOrg(userId: string) {
-  return await prisma.organization.upsert({
+export default async function userProject(userId: string) {
+  return await prisma.project.upsert({
     where: {
-      personalOrgUserId: userId,
+      personalProjectUserId: userId,
     },
     update: {},
     create: {
-      personalOrgUserId: userId,
-      organizationUsers: {
+      personalProjectUserId: userId,
+      projectUsers: {
         create: {
           userId: userId,
           role: "ADMIN",

app/src/state/persist.ts

@@ -0,0 +1,13 @@
+import { type PersistOptions } from "zustand/middleware/persist";
+import { type State } from "./store";
+
+export const stateToPersist = {
+  selectedProjectId: null as string | null,
+};
+
+export const persistOptions: PersistOptions<State, typeof stateToPersist> = {
+  name: "persisted-app-store",
+  partialize: (state) => ({
+    selectedProjectId: state.selectedProjectId,
+  }),
+};

app/src/state/sharedVariantEditor.slice.ts

@@ -8,9 +8,9 @@ export const editorBackground = "#fafafa";
 export type SharedVariantEditorSlice = {
   monaco: null | ReturnType<typeof loader.__getMonacoInstance>;
   loadMonaco: () => Promise<void>;
-  scenarios: RouterOutputs["scenarios"]["list"]["scenarios"];
+  scenarioVars: RouterOutputs["scenarioVars"]["list"];
   updateScenariosModel: () => void;
-  setScenarios: (scenarios: RouterOutputs["scenarios"]["list"]["scenarios"]) => void;
+  setScenarioVars: (scenarioVars: RouterOutputs["scenarioVars"]["list"]) => void;
 };
 
 export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> = (set, get) => ({
@@ -60,10 +60,10 @@ export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> =
     });
     get().sharedVariantEditor.updateScenariosModel();
   },
-  scenarios: [],
-  setScenarios: (scenarios) => {
+  scenarioVars: [],
+  setScenarioVars: (scenarios) => {
     set((state) => {
-      state.sharedVariantEditor.scenarios = scenarios;
+      state.sharedVariantEditor.scenarioVars = scenarios;
     });
 
     get().sharedVariantEditor.updateScenariosModel();
@@ -73,17 +73,16 @@ export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> =
     const monaco = get().sharedVariantEditor.monaco;
     if (!monaco) return;
 
-    const modelContents = `
-    const scenarios = ${JSON.stringify(
-      get().sharedVariantEditor.scenarios.map((s) => s.variableValues),
-      null,
-      2,
-    )} as const;
-    
-    type Scenario = typeof scenarios[number];
-    declare var scenario: Scenario | { [key: string]: string };
+    const modelContents = `    
+    declare var scenario: {
+      ${get()
+        .sharedVariantEditor.scenarioVars.map((s) => `${s.label}: string;`)
+        .join("\n")}
+    };
     `;
 
+    console.log(modelContents);
+
     const scenariosModel = monaco.editor.getModel(monaco.Uri.parse("file:///scenarios.ts"));
 
     if (scenariosModel) {

app/src/state/store.ts

@@ -1,11 +1,13 @@
 import { type StateCreator, create } from "zustand";
 import { immer } from "zustand/middleware/immer";
+import { persist } from "zustand/middleware";
 import { createSelectors } from "./createSelectors";
 import {
   type SharedVariantEditorSlice,
   createVariantEditorSlice,
 } from "./sharedVariantEditor.slice";
 import { type APIClient } from "~/utils/api";
+import { persistOptions, type stateToPersist } from "./persist";
 
 export type State = {
   drawerOpen: boolean;
@@ -14,8 +16,8 @@ export type State = {
   api: APIClient | null;
   setApi: (api: APIClient) => void;
   sharedVariantEditor: SharedVariantEditorSlice;
-  selectedOrgId: string | null;
-  setSelectedOrgId: (orgId: string) => void;
+  selectedProjectId: string | null;
+  setselectedProjectId: (id: string) => void;
 };
 
 export type SliceCreator<T> = StateCreator<State, [["zustand/immer", never]], [], T>;
@@ -23,30 +25,36 @@ export type SliceCreator<T> = StateCreator<State, [["zustand/immer", never]], []
 export type SetFn = Parameters<SliceCreator<unknown>>[0];
 export type GetFn = Parameters<SliceCreator<unknown>>[1];
 
-const useBaseStore = create<State, [["zustand/immer", never]]>(
-  immer((set, get, ...rest) => ({
-    api: null,
-    setApi: (api) =>
-      set((state) => {
-        state.api = api;
-      }),
+const useBaseStore = create<
+  State,
+  [["zustand/persist", typeof stateToPersist], ["zustand/immer", never]]
+>(
+  persist(
+    immer((set, get, ...rest) => ({
+      api: null,
+      setApi: (api) =>
+        set((state) => {
+          state.api = api;
+        }),
 
-    drawerOpen: false,
-    openDrawer: () =>
-      set((state) => {
-        state.drawerOpen = true;
-      }),
-    closeDrawer: () =>
-      set((state) => {
-        state.drawerOpen = false;
-      }),
-    sharedVariantEditor: createVariantEditorSlice(set, get, ...rest),
-    selectedOrgId: null,
-    setSelectedOrgId: (orgId: string) =>
-      set((state) => {
-        state.selectedOrgId = orgId;
-      }),
-  })),
+      drawerOpen: false,
+      openDrawer: () =>
+        set((state) => {
+          state.drawerOpen = true;
+        }),
+      closeDrawer: () =>
+        set((state) => {
+          state.drawerOpen = false;
+        }),
+      sharedVariantEditor: createVariantEditorSlice(set, get, ...rest),
+      selectedProjectId: null,
+      setselectedProjectId: (id: string) =>
+        set((state) => {
+          state.selectedProjectId = id;
+        }),
+    })),
+    persistOptions,
+  ),
 );
 
 export const useAppStore = createSelectors(useBaseStore);

app/src/state/sync.tsx

@@ -1,16 +1,16 @@
 import { useEffect } from "react";
 import { api } from "~/utils/api";
-import { useScenarios } from "~/utils/hooks";
+import { useScenarioVars } from "~/utils/hooks";
 import { useAppStore } from "./store";
 
 export function useSyncVariantEditor() {
-  const scenarios = useScenarios();
+  const scenarioVars = useScenarioVars();
 
   useEffect(() => {
-    if (scenarios.data) {
-      useAppStore.getState().sharedVariantEditor.setScenarios(scenarios.data.scenarios);
+    if (scenarioVars.data) {
+      useAppStore.getState().sharedVariantEditor.setScenarioVars(scenarioVars.data);
     }
-  }, [scenarios.data]);
+  }, [scenarioVars.data]);
 }
 
 export function SyncAppStore() {

app/src/tests/helpers/loadEnv.ts

@@ -0,0 +1,5 @@
+import { configDotenv } from "dotenv";
+
+configDotenv({
+  path: ".env.test",
+});

app/src/tests/helpers/setup.ts

@@ -0,0 +1,13 @@
+import "./loadEnv";
+import { sql } from "kysely";
+import { beforeEach } from "vitest";
+import { kysely } from "~/server/db";
+
+// Reset all Prisma data
+const resetDb = async () => {
+  await sql`truncate "Experiment" cascade;`.execute(kysely);
+};
+
+beforeEach(async () => {
+  await resetDb();
+});

app/src/theme/ChakraThemeProvider.tsx

@@ -1,4 +1,9 @@
-import { extendTheme, defineStyleConfig, ChakraProvider } from "@chakra-ui/react";
+import {
+  extendTheme,
+  defineStyleConfig,
+  ChakraProvider,
+  createStandaloneToast,
+} from "@chakra-ui/react";
 import "@fontsource/inconsolata";
 import { modalAnatomy } from "@chakra-ui/anatomy";
 import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system";
@@ -63,6 +68,15 @@ const theme = extendTheme({
   },
 });
 
+const { ToastContainer, toast } = createStandaloneToast(theme);
+
+export { toast };
+
 export const ChakraThemeProvider = ({ children }: { children: JSX.Element }) => {
-  return <ChakraProvider theme={theme}>{children}</ChakraProvider>;
+  return (
+    <ChakraProvider theme={theme}>
+      <ToastContainer />
+      {children}
+    </ChakraProvider>
+  );
 };

app/src/utils/accessControl.ts

@@ -1,4 +1,4 @@
-import { OrganizationUserRole } from "@prisma/client";
+import { ProjectUserRole } from "@prisma/client";
 import { TRPCError } from "@trpc/server";
 import { type TRPCContext } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
@@ -16,16 +16,16 @@ export const requireNothing = (ctx: TRPCContext) => {
   ctx.markAccessControlRun();
 };
 
-export const requireIsOrgAdmin = async (organizationId: string, ctx: TRPCContext) => {
+export const requireIsProjectAdmin = async (projectId: string, ctx: TRPCContext) => {
   const userId = ctx.session?.user.id;
   if (!userId) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
   }
 
-  const isAdmin = await prisma.organizationUser.findFirst({
+  const isAdmin = await prisma.projectUser.findFirst({
     where: {
       userId,
-      organizationId,
+      projectId,
       role: "ADMIN",
     },
   });
@@ -37,16 +37,16 @@ export const requireIsOrgAdmin = async (organizationId: string, ctx: TRPCContext
   ctx.markAccessControlRun();
 };
 
-export const requireCanViewOrganization = async (organizationId: string, ctx: TRPCContext) => {
+export const requireCanViewProject = async (projectId: string, ctx: TRPCContext) => {
   const userId = ctx.session?.user.id;
   if (!userId) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
   }
 
-  const canView = await prisma.organizationUser.findFirst({
+  const canView = await prisma.projectUser.findFirst({
     where: {
       userId,
-      organizationId,
+      projectId,
     },
   });
 
@@ -57,17 +57,17 @@ export const requireCanViewOrganization = async (organizationId: string, ctx: TR
   ctx.markAccessControlRun();
 };
 
-export const requireCanModifyOrganization = async (organizationId: string, ctx: TRPCContext) => {
+export const requireCanModifyProject = async (projectId: string, ctx: TRPCContext) => {
   const userId = ctx.session?.user.id;
   if (!userId) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
   }
 
-  const canModify = await prisma.organizationUser.findFirst({
+  const canModify = await prisma.projectUser.findFirst({
     where: {
       userId,
-      organizationId,
-      role: { in: [OrganizationUserRole.ADMIN, OrganizationUserRole.MEMBER] },
+      projectId,
+      role: { in: [ProjectUserRole.ADMIN, ProjectUserRole.MEMBER] },
     },
   });
 
@@ -82,10 +82,10 @@ export const requireCanViewDataset = async (datasetId: string, ctx: TRPCContext)
   const dataset = await prisma.dataset.findFirst({
     where: {
       id: datasetId,
-      organization: {
-        organizationUsers: {
+      project: {
+        projectUsers: {
           some: {
-            role: { in: [OrganizationUserRole.ADMIN, OrganizationUserRole.MEMBER] },
+            role: { in: [ProjectUserRole.ADMIN, ProjectUserRole.MEMBER] },
             userId: ctx.session?.user.id,
           },
         },
@@ -120,10 +120,10 @@ export const canModifyExperiment = async (experimentId: string, userId: string)
     prisma.experiment.findFirst({
       where: {
         id: experimentId,
-        organization: {
-          organizationUsers: {
+        project: {
+          projectUsers: {
             some: {
-              role: { in: [OrganizationUserRole.ADMIN, OrganizationUserRole.MEMBER] },
+              role: { in: [ProjectUserRole.ADMIN, ProjectUserRole.MEMBER] },
               userId,
             },
           },

app/src/utils/hooks.ts

@@ -5,10 +5,10 @@ import { NumberParam, useQueryParam, withDefault } from "use-query-params";
 import { useAppStore } from "~/state/store";
 
 export const useExperiments = () => {
-  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  const selectedProjectId = useAppStore((state) => state.selectedProjectId);
   return api.experiments.list.useQuery(
-    { organizationId: selectedOrgId ?? "" },
-    { enabled: !!selectedOrgId },
+    { projectId: selectedProjectId ?? "" },
+    { enabled: !!selectedProjectId },
   );
 };
 
@@ -27,10 +27,10 @@ export const useExperimentAccess = () => {
 };
 
 export const useDatasets = () => {
-  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  const selectedProjectId = useAppStore((state) => state.selectedProjectId);
   return api.datasets.list.useQuery(
-    { organizationId: selectedOrgId ?? "" },
-    { enabled: !!selectedOrgId },
+    { projectId: selectedProjectId ?? "" },
+    { enabled: !!selectedProjectId },
   );
 };
 
@@ -150,7 +150,19 @@ export const useScenario = (scenarioId: string) => {
 
 export const useVisibleScenarioIds = () => useScenarios().data?.scenarios.map((s) => s.id) ?? [];
 
-export const useSelectedOrg = () => {
-  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
-  return api.organizations.get.useQuery({ id: selectedOrgId ?? "" }, { enabled: !!selectedOrgId });
+export const useSelectedProject = () => {
+  const selectedProjectId = useAppStore((state) => state.selectedProjectId);
+  return api.projects.get.useQuery(
+    { id: selectedProjectId ?? "" },
+    { enabled: !!selectedProjectId },
+  );
+};
+
+export const useScenarioVars = () => {
+  const experiment = useExperiment();
+
+  return api.scenarioVars.list.useQuery(
+    { experimentId: experiment.data?.id ?? "" },
+    { enabled: experiment.data?.id != null },
+  );
 };

app/src/utils/standardResponses.ts

@@ -0,0 +1,31 @@
+import { toast } from "~/theme/ChakraThemeProvider";
+
+export function error(message: string): { status: "error"; message: string } {
+  return {
+    status: "error",
+    message,
+  };
+}
+export function success<T>(payload: T): { status: "success"; payload: T };
+export function success(payload?: undefined): { status: "success"; payload: undefined };
+export function success<T>(payload?: T) {
+  return { status: "success", payload };
+}
+
+type SuccessType<T> = ReturnType<typeof success<T>>;
+type ErrorType = ReturnType<typeof error>;
+
+// Used client-side to report generic errors
+export function maybeReportError<T>(response: SuccessType<T> | ErrorType): response is ErrorType {
+  if (response.status === "error") {
+    toast({
+      description: response.message,
+      status: "error",
+      duration: 5000,
+      isClosable: true,
+    });
+    return true;
+  }
+
+  return false;
+}

app/vitest.config.ts

@@ -4,6 +4,10 @@ import { configDefaults, defineConfig, type UserConfig } from "vitest/config";
 const config = defineConfig({
   test: {
     ...configDefaults, // Extending Vitest's default options
+    setupFiles: ["./src/tests/helpers/setup.ts"],
+
+    // Unfortunately using threads seems to cause issues with isolated-vm
+    threads: false,
   },
   plugins: [tsconfigPaths()],
 }) as UserConfig;