Add caching in Python

Still need it in JS
Add a python client library
2023-08-11 19:02:35 -07:00 · 2023-08-11 16:54:50 -07:00 · 2023-08-11 15:02:14 -07:00 · 2023-08-11 13:56:47 -07:00 · 2023-08-10 22:50:39 -07:00 · 2023-08-10 22:15:45 -07:00
104 changed files with 3835 additions and 1377 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.env
+.venv/
+*.pyc
--- a/README.md
+++ b/README.md
@@ -65,7 +65,14 @@ OpenPipe includes a tool to generate new test scenarios based on your existing p
 4. Clone this repository: `git clone https://github.com/openpipe/openpipe`
 5. Install the dependencies: `cd openpipe && pnpm install`
 6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`.
-7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma db push` to create the database.
+7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma migrate dev` to create the database.
 8. Create a [GitHub OAuth App](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app) and update the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` values. (Note: a PR to make auth optional when running locally would be a great contribution!)
 9. Start the app: `pnpm dev`.
 10. Navigate to [http://localhost:3000](http://localhost:3000)
+
+## Testing Locally
+
+1. Copy your `.env` file to `.env.test`.
+2. Update the `DATABASE_URL` to have a different database name than your development one
+3. Run `DATABASE_URL=[your new datatase url] pnpm prisma migrate dev --skip-seed --skip-generate`
+4. Run `pnpm test`
--- a/app/.gitignore
+++ b/app/.gitignore
@@ -34,6 +34,7 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env
 .env*.local
+.env.test

 # vercel
 .vercel
--- a/app/package.json
+++ b/app/package.json
@@ -19,7 +19,7 @@
    "codegen": "tsx src/server/scripts/client-codegen.ts",
    "seed": "tsx prisma/seed.ts",
    "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
-    "test": "pnpm vitest --no-threads"
+    "test": "pnpm vitest"
  },
  "dependencies": {
    "@anthropic-ai/sdk": "^0.5.8",
--- a/app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql
+++ b/app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql
@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
--- a/app/prisma/migrations/20230811200114_rename_model_response_fields/migration.sql
+++ b/app/prisma/migrations/20230811200114_rename_model_response_fields/migration.sql
@@ -0,0 +1,66 @@
+/*
+  Warnings:
+
+  - You are about to rename the column `completionTokens` to `outputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `promptTokens` to `inputTokens` on the `ModelResponse` table.
+  - You are about to rename the column `startTime` on the `LoggedCall` table to `requestedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `startTime` on the `LoggedCallModelResponse` table to `requestedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `endTime` on the `LoggedCallModelResponse` table to `receivedAt`. Ensure compatibility with application logic.
+  - You are about to rename the column `error` on the `LoggedCallModelResponse` table to `errorMessage`. Ensure compatibility with application logic.
+  - You are about to rename the column `respStatus` on the `LoggedCallModelResponse` table to `statusCode`. Ensure compatibility with application logic.
+  - You are about to rename the column `totalCost` on the `LoggedCallModelResponse` table to `cost`. Ensure compatibility with application logic.
+  - You are about to rename the column `inputHash` on the `ModelResponse` table to `cacheKey`. Ensure compatibility with application logic.
+  - You are about to rename the column `output` on the `ModelResponse` table to `respPayload`. Ensure compatibility with application logic.
+
+*/
+-- DropIndex
+DROP INDEX "LoggedCall_startTime_idx";
+
+-- DropIndex
+DROP INDEX "ModelResponse_inputHash_idx";
+
+-- Rename completionTokens to outputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "completionTokens" TO "outputTokens";
+
+-- Rename promptTokens to inputTokens
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "promptTokens" TO "inputTokens";
+
+-- AlterTable
+ALTER TABLE "LoggedCall"
+RENAME COLUMN "startTime" TO "requestedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "startTime" TO "requestedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "endTime" TO "receivedAt";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "error" TO "errorMessage";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "respStatus" TO "statusCode";
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse"
+RENAME COLUMN "totalCost" TO "cost";
+
+-- AlterTable
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "inputHash" TO "cacheKey";
+
+-- AlterTable
+ALTER TABLE "ModelResponse"
+RENAME COLUMN "output" TO "respPayload";
+
+-- CreateIndex
+CREATE INDEX "LoggedCall_requestedAt_idx" ON "LoggedCall"("requestedAt");
+
+-- CreateIndex
+CREATE INDEX "ModelResponse_cacheKey_idx" ON "ModelResponse"("cacheKey");
--- a/app/prisma/schema.prisma
+++ b/app/prisma/schema.prisma
@@ -112,13 +112,13 @@ model ScenarioVariantCell {
 model ModelResponse {
    id String @id @default(uuid()) @db.Uuid

-    inputHash        String
+    cacheKey        String
    requestedAt      DateTime?
    receivedAt       DateTime?
-    output           Json?
+    respPayload      Json?
    cost             Float?
-    promptTokens     Int?
-    completionTokens Int?
+    inputTokens      Int?
+    outputTokens     Int?
    statusCode       Int?
    errorMessage     String?
    retryTime        DateTime?
@@ -131,7 +131,7 @@ model ModelResponse {
    scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
    outputEvaluations     OutputEvaluation[]

-    @@index([inputHash])
+    @@index([cacheKey])
 }

 enum EvalType {
@@ -256,7 +256,7 @@ model WorldChampEntrant {
 model LoggedCall {
    id String @id @default(uuid()) @db.Uuid

-    startTime DateTime
+    requestedAt DateTime

    // True if this call was served from the cache, false otherwise
    cacheHit Boolean
@@ -278,7 +278,7 @@ model LoggedCall {
    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt

-    @@index([startTime])
+    @@index([requestedAt])
 }

 model LoggedCallModelResponse {
@@ -287,14 +287,14 @@ model LoggedCallModelResponse {
    reqPayload Json

    // The HTTP status returned by the model provider
-    respStatus  Int?
+    statusCode  Int?
    respPayload Json?

    // Should be null if the request was successful, and some string if the request failed.
-    error String?
+    errorMessage String?

-    startTime DateTime
-    endTime   DateTime
+    requestedAt DateTime
+    receivedAt   DateTime

    // Note: the function to calculate the cacheKey should include the project
    // ID so we don't share cached responses between projects, which could be an
@@ -308,7 +308,7 @@ model LoggedCallModelResponse {
    outputTokens Int?
    finishReason String?
    completionId String?
-    totalCost    Decimal? @db.Decimal(18, 12)
+    cost         Decimal? @db.Decimal(18, 12)

    // The LoggedCall that created this LoggedCallModelResponse
    originalLoggedCallId String     @unique @db.Uuid
--- a/app/prisma/seedDashboard.ts
+++ b/app/prisma/seedDashboard.ts
@@ -339,17 +339,17 @@ for (let i = 0; i < 1437; i++) {
    MODEL_RESPONSE_TEMPLATES[Math.floor(Math.random() * MODEL_RESPONSE_TEMPLATES.length)]!;
  const model = template.reqPayload.model;
  // choose random time in the last two weeks, with a bias towards the last few days
-  const startTime = new Date(Date.now() - Math.pow(Math.random(), 2) * 1000 * 60 * 60 * 24 * 14);
+  const requestedAt = new Date(Date.now() - Math.pow(Math.random(), 2) * 1000 * 60 * 60 * 24 * 14);
  // choose random delay anywhere from 2 to 10 seconds later for gpt-4, or 1 to 5 seconds for gpt-3.5
  const delay =
    model === "gpt-4" ? 1000 * 2 + Math.random() * 1000 * 8 : 1000 + Math.random() * 1000 * 4;
-  const endTime = new Date(startTime.getTime() + delay);
+  const receivedAt = new Date(requestedAt.getTime() + delay);
  loggedCallsToCreate.push({
    id: loggedCallId,
    cacheHit: false,
-    startTime,
+    requestedAt,
    projectId: project.id,
-    createdAt: startTime,
+    createdAt: requestedAt,
  });

  const { promptTokenPrice, completionTokenPrice } =
@@ -365,21 +365,20 @@ for (let i = 0; i < 1437; i++) {

  loggedCallModelResponsesToCreate.push({
    id: loggedCallModelResponseId,
-    startTime,
-    endTime,
+    requestedAt,
+    receivedAt,
    originalLoggedCallId: loggedCallId,
    reqPayload: template.reqPayload,
    respPayload: template.respPayload,
-    respStatus: template.respStatus,
-    error: template.error,
-    createdAt: startTime,
+    statusCode: template.respStatus,
+    errorMessage: template.error,
+    createdAt: requestedAt,
    cacheKey: hashRequest(project.id, template.reqPayload as JsonValue),
-    durationMs: endTime.getTime() - startTime.getTime(),
+    durationMs: receivedAt.getTime() - requestedAt.getTime(),
    inputTokens: template.inputTokens,
    outputTokens: template.outputTokens,
    finishReason: template.finishReason,
-    totalCost:
-      template.inputTokens * promptTokenPrice + template.outputTokens * completionTokenPrice,
+    cost: template.inputTokens * promptTokenPrice + template.outputTokens * completionTokenPrice,
  });
  loggedCallsToUpdate.push({
    where: {
--- a/app/public/favicon.ico
+++ b/app/public/favicon.ico
--- a/app/public/favicons/android-chrome-192x192.png
+++ b/app/public/favicons/android-chrome-192x192.png
--- a/app/public/favicons/android-chrome-512x512.png
+++ b/app/public/favicons/android-chrome-512x512.png
--- a/app/public/favicons/apple-touch-icon.png
+++ b/app/public/favicons/apple-touch-icon.png
--- a/app/public/favicons/favicon-16x16.png
+++ b/app/public/favicons/favicon-16x16.png
--- a/app/public/favicons/favicon-32x32.png
+++ b/app/public/favicons/favicon-32x32.png
--- a/app/public/favicons/favicon.ico
+++ b/app/public/favicons/favicon.ico
--- a/app/public/favicons/mstile-150x150.png
+++ b/app/public/favicons/mstile-150x150.png
--- a/app/public/favicons/safari-pinned-tab.svg
+++ b/app/public/favicons/safari-pinned-tab.svg
@@ -9,10 +9,9 @@ Created by potrace 1.14, written by Peter Selinger 2001-2017
 </metadata>
 <g transform="translate(0.000000,550.000000) scale(0.100000,-0.100000)"
 fill="#000000" stroke="none">
-<path d="M813 5478 c-18 -13 -37 -36 -43 -52 -6 -19 -10 -236 -10 -603 0 -638
-1 -626 65 -657 25 -12 67 -16 179 -16 l146 0 0 -2032 0 -2032 23 -33 c12 -18
-35 -37 51 -43 19 -7 539 -10 1528 -10 1663 0 1549 -5 1582 65 14 30 16 235 16
-2059 l0 2026 156 0 156 0 39 39 39 39 0 587 c0 651 1 638 -65 669 -30 14 -223
-16 -1932 16 l-1898 0 -32 -22z"/>
+<path d="M785 5474 l-25 -27 0 -622 0 -622 25 -27 24 -26 171 0 170 0 0 -2050
+0 -2051 25 -25 24 -24 1557 2 1556 3 19 24 c19 23 19 70 19 2072 l0 2049 169
+0 c165 0 169 1 195 25 l26 24 0 626 0 626 -26 24 -27 25 -1939 0 -1939 0 -24
+-26z"/>
 </g>
 </svg>
--- a/app/public/logo.svg
+++ b/app/public/logo.svg
@@ -1,5 +1,28 @@
-<svg width="380" height="320" viewBox="0 0 380 320" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M72 320L122.5 231L130.5 150.5L115 73L72 0H312L265 64.5L257 158.5L265 249L312 320H72Z" fill="#FF5733"/>
-<path d="M67.027 9.5C72.9909 9.5 79.5196 12.3449 86.3672 19.2588C93.2495 26.2075 99.8845 36.7468 105.66 50.5336C117.194 78.0671 124.554 116.764 124.554 160C124.554 203.236 117.194 241.933 105.66 269.466C99.8845 283.253 93.2495 293.793 86.3672 300.741C79.5196 307.655 72.9909 310.5 67.027 310.5C61.0632 310.5 54.5345 307.655 47.6868 300.741C40.8045 293.793 34.1695 283.253 28.394 269.466C16.8596 241.933 9.5 203.236 9.5 160C9.5 116.764 16.8596 78.0671 28.394 50.5336C34.1695 36.7468 40.8045 26.2075 47.6868 19.2588C54.5345 12.3449 61.0632 9.5 67.027 9.5Z" stroke="#FF5733" stroke-width="19"/>
-<path d="M312.027 9.5C317.991 9.5 324.52 12.3449 331.367 19.2588C338.25 26.2075 344.885 36.7468 350.66 50.5336C362.194 78.0671 369.554 116.764 369.554 160C369.554 203.236 362.194 241.933 350.66 269.466C344.885 283.253 338.25 293.793 331.367 300.741C324.52 307.655 317.991 310.5 312.027 310.5C306.063 310.5 299.534 307.655 292.687 300.741C285.805 293.793 279.17 283.253 273.394 269.466C261.86 241.933 254.5 203.236 254.5 160C254.5 116.764 261.86 78.0671 273.394 50.5336C279.17 36.7468 285.805 26.2075 292.687 19.2588C299.534 12.3449 306.063 9.5 312.027 9.5Z" stroke="#FF5733" stroke-width="19"/>
+<svg width="398" height="550" viewBox="0 0 398 550" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M39 125H359V542C359 546.418 355.418 550 351 550H47C42.5817 550 39 546.418 39 542V125Z" fill="black"/>
+<path d="M0 8C0 3.58172 3.58172 0 8 0H390C394.418 0 398 3.58172 398 8V127C398 131.418 394.418 135 390 135H7.99999C3.58171 135 0 131.418 0 127V8Z" fill="black"/>
+<path d="M50 135H348V535C348 537.209 346.209 539 344 539H54C51.7909 539 50 537.209 50 535V135Z" fill="#FF5733"/>
+<path d="M11 14.0001C11 11.791 12.7909 10.0001 15 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H15C12.7909 124 11 122.209 11 120V14.0001Z" fill="#FF5733"/>
+<path d="M11 14.0001C11 11.791 12.7909 10.0001 15 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H15C12.7909 124 11 122.209 11 120V14.0001Z" fill="url(#paint0_linear_102_49)"/>
+<path d="M50 134H348V535C348 537.209 346.209 539 344 539H54C51.7909 539 50 537.209 50 535V134Z" fill="url(#paint1_linear_102_49)"/>
+<path d="M108 142H156V535H108V142Z" fill="white"/>
+<path d="M300 135H348V535C348 537.209 346.209 539 344 539H300V135Z" fill="white" fill-opacity="0.25"/>
+<path d="M96 142H108V535H96V142Z" fill="white" fill-opacity="0.5"/>
+<path d="M84 10.0001H133V120H84V10.0001Z" fill="white"/>
+<path d="M339 10.0001H384C386.209 10.0001 388 11.791 388 14.0001V120C388 122.209 386.209 124 384 124H339V10.0001Z" fill="white" fill-opacity="0.25"/>
+<path d="M71.9995 10.0001H83.9995V120H71.9995V10.0001Z" fill="white" fill-opacity="0.5"/>
+<path d="M108 534.529H156V539.019H108V534.529Z" fill="#AAAAAA"/>
+<path opacity="0.5" d="M95.9927 534.529H107.982V539.019H95.9927V534.529Z" fill="#AAAAAA"/>
+<path d="M84.0029 119.887H133.007V124.027H84.0029V119.887Z" fill="#AAAAAA"/>
+<path opacity="0.5" d="M71.9883 119.887H83.978V124.027H71.9883V119.887Z" fill="#AAAAAA"/>
+<defs>
+<linearGradient id="paint0_linear_102_49" x1="335" y1="67.0001" x2="137" y2="67.0001" gradientUnits="userSpaceOnUse">
+<stop stop-color="#D62600"/>
+<stop offset="1" stop-color="#FF5733" stop-opacity="0"/>
+</linearGradient>
+<linearGradient id="paint1_linear_102_49" x1="306.106" y1="336.5" x2="149.597" y2="336.5" gradientUnits="userSpaceOnUse">
+<stop stop-color="#D62600"/>
+<stop offset="1" stop-color="#FF5733" stop-opacity="0"/>
+</linearGradient>
+</defs>
 </svg>
--- a/app/public/og.png
+++ b/app/public/og.png
--- a/app/src/components/CopiableCode.tsx
+++ b/app/src/components/CopiableCode.tsx
@@ -19,7 +19,7 @@ const CopiableCode = ({ code }: { code: string }) => {
      w="full"
      justifyContent="space-between"
    >
-      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5}>
+      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5} overflowX="auto">
        {code}
      </Text>
      <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
--- a/app/src/components/Favicon.tsx
+++ b/app/src/components/Favicon.tsx
@@ -8,8 +8,8 @@ export default function Favicon() {
      <link rel="icon" type="image/png" sizes="16x16" href="/favicons/favicon-16x16.png" />
      <link rel="manifest" href="/favicons/site.webmanifest" />
      <link rel="shortcut icon" href="/favicons/favicon.ico" />
+      <link rel="mask-icon" href="/favicons/safari-pinned-tab.svg" color="#5bbad5" />
      <meta name="msapplication-TileColor" content="#da532c" />
-      <meta name="msapplication-config" content="/favicons/browserconfig.xml" />
      <meta name="theme-color" content="#ffffff" />
    </Head>
  );
--- a/app/src/components/OutputsTable/EditEvaluations.tsx
+++ b/app/src/components/OutputsTable/EditEvaluations.tsx
@@ -12,6 +12,7 @@ import {
  Select,
  FormHelperText,
  Code,
+  IconButton,
 } from "@chakra-ui/react";
 import { type Evaluation, EvalType } from "@prisma/client";
 import { useCallback, useState } from "react";
@@ -183,46 +184,37 @@ export default function EditEvaluations() {
                <Text flex={1}>
                  {evaluation.evalType}: &quot;{evaluation.value}&quot;
                </Text>
-                <Button
+
+                <IconButton
+                  aria-label="Edit"
                  variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                  minW="unset"
+                  color="gray.400"
                  onClick={() => setEditingId(evaluation.id)}
-                  _hover={{
-                    color: "gray.800",
-                    cursor: "pointer",
-                  }}
-                >
-                  <Icon as={BsPencil} boxSize={4} />
-                </Button>
-                <Button
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
+                  icon={<Icon as={BsPencil} />}
+                />
+                <IconButton
+                  aria-label="Delete"
                  variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                  minW="unset"
+                  color="gray.400"
                  onClick={() => onDelete(evaluation.id)}
-                  _hover={{
-                    color: "gray.800",
-                    cursor: "pointer",
-                  }}
-                >
-                  <Icon as={BsX} boxSize={6} />
-                </Button>
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
+                  icon={<Icon as={BsX} boxSize={6} />}
+                />
              </HStack>
            ),
          )}
          {editingId == null && (
            <Button
              onClick={() => setEditingId("new")}
-              alignSelf="flex-start"
+              alignSelf="end"
              size="sm"
              mt={4}
              colorScheme="blue"
            >
-              Add Evaluation
+              New Evaluation
            </Button>
          )}
          {editingId == "new" && (
--- a/app/src/components/OutputsTable/EditScenarioVars.tsx
+++ b/app/src/components/OutputsTable/EditScenarioVars.tsx
@@ -1,103 +1,185 @@
-import { Text, Button, HStack, Heading, Icon, Input, Stack } from "@chakra-ui/react";
-import { useState } from "react";
-import { BsCheck, BsX } from "react-icons/bs";
+import { Text, Button, HStack, Heading, Icon, IconButton, Stack, VStack } from "@chakra-ui/react";
+import { type TemplateVariable } from "@prisma/client";
+import { useEffect, useState } from "react";
+import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+
+export const ScenarioVar = ({
+  variable,
+  isEditing,
+  setIsEditing,
+}: {
+  variable: Pick<TemplateVariable, "id" | "label">;
+  isEditing: boolean;
+  setIsEditing: (isEditing: boolean) => void;
+}) => {
+  const utils = api.useContext();
+
+  const [label, setLabel] = useState(variable.label);
+
+  useEffect(() => {
+    setLabel(variable.label);
+  }, [variable.label]);
+
+  const renameVarMutation = api.scenarioVars.rename.useMutation();
+  const [onRename] = useHandledAsyncCallback(async () => {
+    const resp = await renameVarMutation.mutateAsync({ id: variable.id, label });
+    if (maybeReportError(resp)) return;
+
+    setIsEditing(false);
+    await utils.scenarioVars.list.invalidate();
+    await utils.scenarios.list.invalidate();
+  }, [label, variable.id]);
+
+  const deleteMutation = api.scenarioVars.delete.useMutation();
+  const [onDeleteVar] = useHandledAsyncCallback(async () => {
+    await deleteMutation.mutateAsync({ id: variable.id });
+    await utils.scenarioVars.list.invalidate();
+  }, [variable.id]);
+
+  if (isEditing) {
+    return (
+      <HStack w="full">
+        <FloatingLabelInput
+          flex={1}
+          label="Renamed Variable"
+          value={label}
+          onChange={(e) => setLabel(e.target.value)}
+          onKeyDown={(e) => {
+            if (e.key === "Enter") {
+              e.preventDefault();
+              onRename();
+            }
+            // If the user types a space, replace it with an underscore
+            if (e.key === " ") {
+              e.preventDefault();
+              setLabel((label) => label && `${label}_`);
+            }
+          }}
+        />
+        <Button size="sm" onClick={() => setIsEditing(false)}>
+          Cancel
+        </Button>
+        <Button size="sm" colorScheme="blue" onClick={onRename}>
+          Save
+        </Button>
+      </HStack>
+    );
+  } else {
+    return (
+      <HStack w="full" borderTopWidth={1} borderColor="gray.200">
+        <Text flex={1}>{variable.label}</Text>
+        <IconButton
+          aria-label="Edit"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={() => setIsEditing(true)}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsPencil} />}
+        />
+        <IconButton
+          aria-label="Delete"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={onDeleteVar}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsX} boxSize={6} />}
+        />
+      </HStack>
+    );
+  }
+};

 export default function EditScenarioVars() {
  const experiment = useExperiment();
-  const vars =
-    api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+  const vars = useScenarioVars();
+
+  const [currentlyEditingId, setCurrentlyEditingId] = useState<string | null>(null);

  const [newVariable, setNewVariable] = useState<string>("");
-  const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
+  const newVarIsValid = newVariable?.length ?? 0 > 0;

  const utils = api.useContext();
-  const addVarMutation = api.templateVars.create.useMutation();
+  const addVarMutation = api.scenarioVars.create.useMutation();
  const [onAddVar] = useHandledAsyncCallback(async () => {
    if (!experiment.data?.id) return;
-    if (!newVarIsValid) return;
-    await addVarMutation.mutateAsync({
+    if (!newVariable) return;
+    const resp = await addVarMutation.mutateAsync({
      experimentId: experiment.data.id,
      label: newVariable,
    });
-    await utils.templateVars.list.invalidate();
+    if (maybeReportError(resp)) return;
+
+    await utils.scenarioVars.list.invalidate();
    setNewVariable("");
  }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);

-  const deleteMutation = api.templateVars.delete.useMutation();
-  const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
-    await deleteMutation.mutateAsync({ id });
-    await utils.templateVars.list.invalidate();
-  }, []);
-
  return (
    <Stack>
      <Heading size="sm">Scenario Variables</Heading>
-      <Stack spacing={2}>
+      <VStack spacing={4}>
        <Text fontSize="sm">
          Scenario variables can be used in your prompt variants as well as evaluations.
        </Text>
-        <HStack spacing={0}>
-          <Input
-            placeholder="Add Scenario Variable"
-            size="sm"
-            borderTopRadius={0}
-            borderRightRadius={0}
-            value={newVariable}
-            onChange={(e) => setNewVariable(e.target.value)}
-            onKeyDown={(e) => {
-              if (e.key === "Enter") {
-                e.preventDefault();
-                onAddVar();
-              }
-              // If the user types a space, replace it with an underscore
-              if (e.key === " ") {
-                e.preventDefault();
-                setNewVariable((v) => v + "_");
-              }
-            }}
-          />
-          <Button
-            size="xs"
-            height="100%"
-            borderLeftRadius={0}
-            isDisabled={!newVarIsValid}
-            onClick={onAddVar}
-          >
-            <Icon as={BsCheck} boxSize={8} />
-          </Button>
-        </HStack>
-
-        <HStack spacing={2} py={4} wrap="wrap">
-          {vars.map((variable) => (
-            <HStack
+        <VStack spacing={0} w="full">
+          {vars.data?.map((variable) => (
+            <ScenarioVar
+              variable={variable}
              key={variable.id}
-              spacing={0}
-              bgColor="blue.100"
-              color="blue.600"
-              pl={2}
-              pr={0}
-              fontWeight="bold"
-            >
-              <Text fontSize="sm" flex={1}>
-                {variable.label}
-              </Text>
-              <Button
-                size="xs"
-                variant="ghost"
-                colorScheme="blue"
-                p="unset"
-                minW="unset"
-                px="unset"
-                onClick={() => onDeleteVar(variable.id)}
-              >
-                <Icon as={BsX} boxSize={6} color="blue.800" />
-              </Button>
-            </HStack>
+              isEditing={currentlyEditingId === variable.id}
+              setIsEditing={(isEditing) => {
+                if (isEditing) {
+                  setCurrentlyEditingId(variable.id);
+                } else {
+                  setCurrentlyEditingId(null);
+                }
+              }}
+            />
          ))}
-        </HStack>
-      </Stack>
+        </VStack>
+        {currentlyEditingId !== "new" && (
+          <Button
+            colorScheme="blue"
+            size="sm"
+            onClick={() => setCurrentlyEditingId("new")}
+            alignSelf="end"
+          >
+            New Variable
+          </Button>
+        )}
+        {currentlyEditingId === "new" && (
+          <HStack w="full">
+            <FloatingLabelInput
+              flex={1}
+              label="New Variable"
+              value={newVariable}
+              onChange={(e) => setNewVariable(e.target.value)}
+              onKeyDown={(e) => {
+                if (e.key === "Enter") {
+                  e.preventDefault();
+                  onAddVar();
+                }
+                // If the user types a space, replace it with an underscore
+                if (e.key === " ") {
+                  e.preventDefault();
+                  setNewVariable((v) => v && `${v}_`);
+                }
+              }}
+            />
+            <Button size="sm" onClick={() => setCurrentlyEditingId(null)}>
+              Cancel
+            </Button>
+            <Button size="sm" colorScheme="blue" onClick={onAddVar}>
+              Save
+            </Button>
+          </HStack>
+        )}
+      </VStack>
    </Stack>
  );
 }
--- a/app/src/components/OutputsTable/OutputCell/OutputCell.tsx
+++ b/app/src/components/OutputsTable/OutputCell/OutputCell.tsx
@@ -1,7 +1,7 @@
 import { api } from "~/utils/api";
 import { type PromptVariant, type Scenario } from "../types";
 import { type StackProps, Text, VStack } from "@chakra-ui/react";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
 import stringify from "json-stringify-pretty-compact";
@@ -23,10 +23,7 @@ export default function OutputCell({
  variant: PromptVariant;
 }): ReactElement | null {
  const utils = api.useContext();
-  const experiment = useExperiment();
-  const vars = api.templateVars.list.useQuery({
-    experimentId: experiment.data?.id ?? "",
-  }).data;
+  const vars = useScenarioVars().data;

  const scenarioVariables = scenario.variableValues as Record<string, string>;
  const templateHasVariables =
@@ -110,7 +107,7 @@ export default function OutputCell({

  if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;

-  const showLogs = !streamedMessage && !mostRecentResponse?.output;
+  const showLogs = !streamedMessage && !mostRecentResponse?.respPayload;

  if (showLogs)
    return (
@@ -163,13 +160,13 @@ export default function OutputCell({
      </CellWrapper>
    );

-  const normalizedOutput = mostRecentResponse?.output
-    ? provider.normalizeOutput(mostRecentResponse?.output)
+  const normalizedOutput = mostRecentResponse?.respPayload
+    ? provider.normalizeOutput(mostRecentResponse?.respPayload)
    : streamedMessage
    ? provider.normalizeOutput(streamedMessage)
    : null;

-  if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
+  if (mostRecentResponse?.respPayload && normalizedOutput?.type === "json") {
    return (
      <CellWrapper>
        <SyntaxHighlighter
@@ -191,7 +188,7 @@ export default function OutputCell({

  return (
    <CellWrapper>
-      <Text>{contentToDisplay}</Text>
+      <Text whiteSpace="pre-wrap">{contentToDisplay}</Text>
    </CellWrapper>
  );
 }
--- a/app/src/components/OutputsTable/OutputCell/OutputStats.tsx
+++ b/app/src/components/OutputsTable/OutputCell/OutputStats.tsx
@@ -19,8 +19,8 @@ export const OutputStats = ({
      ? modelResponse.receivedAt.getTime() - modelResponse.requestedAt.getTime()
      : 0;

-  const promptTokens = modelResponse.promptTokens;
-  const completionTokens = modelResponse.completionTokens;
+  const inputTokens = modelResponse.inputTokens;
+  const outputTokens = modelResponse.outputTokens;

  return (
    <HStack
@@ -55,8 +55,8 @@ export const OutputStats = ({
      </HStack>
      {modelResponse.cost && (
        <CostTooltip
-          promptTokens={promptTokens}
-          completionTokens={completionTokens}
+          inputTokens={inputTokens}
+          outputTokens={outputTokens}
          cost={modelResponse.cost}
        >
          <HStack spacing={0}>
--- a/app/src/components/OutputsTable/ScenarioEditor.tsx
+++ b/app/src/components/OutputsTable/ScenarioEditor.tsx
@@ -1,7 +1,7 @@
 import { isEqual } from "lodash-es";
 import { useEffect, useState, type DragEvent } from "react";
 import { api } from "~/utils/api";
-import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperimentAccess, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
 import { type Scenario } from "./types";

 import {
@@ -41,8 +41,7 @@ export default function ScenarioEditor({
    if (savedValues) setValues(savedValues);
  }, [savedValues]);

-  const experiment = useExperiment();
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = useScenarioVars();

  const variableLabels = vars.data?.map((v) => v.label) ?? [];

--- a/app/src/components/OutputsTable/ScenarioEditorModal.tsx
+++ b/app/src/components/OutputsTable/ScenarioEditorModal.tsx
@@ -58,7 +58,7 @@ export const ScenarioEditorModal = ({
    await utils.scenarios.list.invalidate();
  }, [mutation, values]);

-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = api.scenarioVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
  const variableLabels = vars.data?.map((v) => v.label) ?? [];

  return (
--- a/app/src/components/OutputsTable/ScenariosHeader.tsx
+++ b/app/src/components/OutputsTable/ScenariosHeader.tsx
@@ -72,7 +72,7 @@ export const ScenariosHeader = () => {
              Autogenerate Scenario
            </MenuItem>
            <MenuItem icon={<BsPencil />} onClick={openDrawer}>
-              Edit Vars
+              Add or Remove Variables
            </MenuItem>
          </MenuList>
        </Menu>
--- a/app/src/components/OutputsTable/VariantStats.tsx
+++ b/app/src/components/OutputsTable/VariantStats.tsx
@@ -17,8 +17,8 @@ export default function VariantStats(props: { variant: PromptVariant }) {
      initialData: {
        evalResults: [],
        overallCost: 0,
-        promptTokens: 0,
-        completionTokens: 0,
+        inputTokens: 0,
+        outputTokens: 0,
        scenarioCount: 0,
        outputCount: 0,
        awaitingEvals: false,
@@ -68,8 +68,8 @@ export default function VariantStats(props: { variant: PromptVariant }) {
      </HStack>
      {data.overallCost && (
        <CostTooltip
-          promptTokens={data.promptTokens}
-          completionTokens={data.completionTokens}
+          inputTokens={data.inputTokens}
+          outputTokens={data.outputTokens}
          cost={data.overallCost}
        >
          <HStack spacing={0} align="center" color="gray.500">
--- a/app/src/components/dashboard/LoggedCallTable.tsx
+++ b/app/src/components/dashboard/LoggedCallTable.tsx
@@ -90,15 +90,23 @@ function TableRow({
  isExpanded: boolean;
  onToggle: () => void;
 }) {
-  const isError = loggedCall.modelResponse?.respStatus !== 200;
-  const timeAgo = dayjs(loggedCall.startTime).fromNow();
-  const fullTime = dayjs(loggedCall.startTime).toString();
+  const isError = loggedCall.modelResponse?.statusCode !== 200;
+  const timeAgo = dayjs(loggedCall.requestedAt).fromNow();
+  const fullTime = dayjs(loggedCall.requestedAt).toString();

  const model = useMemo(
    () => loggedCall.tags.find((tag) => tag.name.startsWith("$model"))?.value,
    [loggedCall.tags],
  );

+  const durationCell = (
+    <Td isNumeric>
+      {loggedCall.cacheHit
+        ? "Cache hit"
+        : ((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2) + "s"}
+    </Td>
+  );
+
  return (
    <>
      <Tr
@@ -120,11 +128,11 @@ function TableRow({
          </Tooltip>
        </Td>
        <Td width="100%">{model}</Td>
-        <Td isNumeric>{((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2)}s</Td>
+        {durationCell}
        <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
        <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
        <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
-          {loggedCall.modelResponse?.respStatus ?? "No response"}
+          {loggedCall.modelResponse?.statusCode ?? "No response"}
        </Td>
      </Tr>
      <Tr>
--- a/app/src/components/dashboard/UsageGraph.tsx
+++ b/app/src/components/dashboard/UsageGraph.tsx
@@ -0,0 +1,61 @@
+import {
+  ResponsiveContainer,
+  LineChart,
+  Line,
+  XAxis,
+  YAxis,
+  CartesianGrid,
+  Tooltip,
+  Legend,
+} from "recharts";
+import { useMemo } from "react";
+
+import { useSelectedProject } from "~/utils/hooks";
+import dayjs from "~/utils/dayjs";
+import { api } from "~/utils/api";
+
+export default function UsageGraph() {
+  const { data: selectedProject } = useSelectedProject();
+
+  const stats = api.dashboard.stats.useQuery(
+    { projectId: selectedProject?.id ?? "" },
+    { enabled: !!selectedProject },
+  );
+
+  const data = useMemo(() => {
+    return (
+      stats.data?.periods.map(({ period, numQueries, cost }) => ({
+        period,
+        Requests: numQueries,
+        "Total Spent (USD)": parseFloat(cost.toString()),
+      })) || []
+    );
+  }, [stats.data]);
+
+  return (
+    <ResponsiveContainer width="100%" height={400}>
+      <LineChart data={data} margin={{ top: 5, right: 20, left: 10, bottom: 5 }}>
+        <XAxis dataKey="period" tickFormatter={(str: string) => dayjs(str).format("MMM D")} />
+        <YAxis yAxisId="left" dataKey="Requests" orientation="left" stroke="#8884d8" />
+        <YAxis
+          yAxisId="right"
+          dataKey="Total Spent (USD)"
+          orientation="right"
+          unit="$"
+          stroke="#82ca9d"
+        />
+        <Tooltip />
+        <Legend />
+        <CartesianGrid stroke="#f5f5f5" />
+        <Line dataKey="Requests" stroke="#8884d8" yAxisId="left" dot={false} strokeWidth={2} />
+        <Line
+          dataKey="Total Spent (USD)"
+          stroke="#82ca9d"
+          yAxisId="right"
+          dot={false}
+          strokeWidth={2}
+        />
+      </LineChart>
+    </ResponsiveContainer>
+  );
+}
--- a/app/src/components/nav/AppShell.tsx
+++ b/app/src/components/nav/AppShell.tsx
@@ -40,8 +40,15 @@ const NavSidebar = () => {
      borderRightWidth={1}
      borderColor="gray.300"
    >
-      <HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={2} py={2}>
-        <Image src="/logo.svg" alt="" boxSize={6} mr={4} />
+      <HStack
+        as={Link}
+        href="/"
+        _hover={{ textDecoration: "none" }}
+        spacing={{ base: 1, md: 0 }}
+        mx={2}
+        py={{ base: 1, md: 2 }}
+      >
+        <Image src="/logo.svg" alt="" boxSize={6} mr={4} ml={{ base: 0.5, md: 0 }} />
        <Heading size="md" fontFamily="inconsolata, monospace">
          OpenPipe
        </Heading>
--- a/app/src/components/nav/NavSidebarOption.tsx
+++ b/app/src/components/nav/NavSidebarOption.tsx
@@ -1,11 +1,10 @@
-import { Box, type BoxProps } from "@chakra-ui/react";
+import { Box, type BoxProps, forwardRef } from "@chakra-ui/react";
 import { useRouter } from "next/router";

-const NavSidebarOption = ({
-  activeHrefPattern,
-  disableHoverEffect,
-  ...props
-}: { activeHrefPattern?: string; disableHoverEffect?: boolean } & BoxProps) => {
+const NavSidebarOption = forwardRef<
+  { activeHrefPattern?: string; disableHoverEffect?: boolean } & BoxProps,
+  "div"
+>(({ activeHrefPattern, disableHoverEffect, ...props }, ref) => {
  const router = useRouter();
  const isActive = activeHrefPattern && router.pathname.startsWith(activeHrefPattern);
  return (
@@ -18,10 +17,13 @@ const NavSidebarOption = ({
      cursor="pointer"
      borderRadius={4}
      {...props}
+      ref={ref}
    >
      {props.children}
    </Box>
  );
-};
+});
+
+NavSidebarOption.displayName = "NavSidebarOption";

 export default NavSidebarOption;
--- a/app/src/components/nav/ProjectMenu.tsx
+++ b/app/src/components/nav/ProjectMenu.tsx
@@ -15,8 +15,7 @@ import {
 } from "@chakra-ui/react";
 import React, { useEffect, useState } from "react";
 import Link from "next/link";
-import { AiFillCaretDown } from "react-icons/ai";
-import { BsGear, BsPlus } from "react-icons/bs";
+import { BsChevronRight, BsGear, BsPlus } from "react-icons/bs";
 import { type Project } from "@prisma/client";

 import { useAppStore } from "~/state/store";
@@ -68,14 +67,9 @@ export default function ProjectMenu() {
      >
        PROJECT
      </Text>
-      <NavSidebarOption>
-        <Popover
-          placement="bottom-start"
-          isOpen={popover.isOpen}
-          onClose={popover.onClose}
-          closeOnBlur
-        >
-          <PopoverTrigger>
+      <Popover placement="right-end" isOpen={popover.isOpen} onClose={popover.onClose} closeOnBlur>
+        <PopoverTrigger>
+          <NavSidebarOption>
            <HStack w="full" onClick={popover.onToggle}>
              <Flex
                p={1}
@@ -92,46 +86,41 @@ export default function ProjectMenu() {
              <Text fontSize="sm" display={{ base: "none", md: "block" }} py={1} flex={1}>
                {selectedProject?.name}
              </Text>
-              <Icon as={AiFillCaretDown} boxSize={3} size="xs" color="gray.500" mr={2} />
+              <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
            </HStack>
-          </PopoverTrigger>
-          <PopoverContent
-            _focusVisible={{ boxShadow: "unset" }}
-            minW={0}
-            borderColor="blue.400"
-            w="full"
-          >
-            <VStack alignItems="flex-start" spacing={2} py={4} px={2}>
-              <Text color="gray.500" fontSize="xs" fontWeight="bold" pb={1}>
-                PROJECTS
-              </Text>
-              <Divider />
-              <VStack spacing={0} w="full">
-                {projects?.map((proj) => (
-                  <ProjectOption
-                    key={proj.id}
-                    proj={proj}
-                    isActive={proj.id === selectedProjectId}
-                    onClose={popover.onClose}
-                  />
-                ))}
-              </VStack>
-              <HStack
-                as={Button}
-                variant="ghost"
-                colorScheme="blue"
-                color="blue.400"
-                pr={8}
-                w="full"
-                onClick={createProject}
-              >
-                <Icon as={isLoading ? Spinner : BsPlus} boxSize={6} />
-                <Text>New project</Text>
-              </HStack>
+          </NavSidebarOption>
+        </PopoverTrigger>
+        <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} w="auto" minW={100} maxW={280}>
+          <VStack alignItems="flex-start" spacing={2} py={4} px={2}>
+            <Text color="gray.500" fontSize="xs" fontWeight="bold" pb={1}>
+              PROJECTS
+            </Text>
+            <Divider />
+            <VStack spacing={0} w="full">
+              {projects?.map((proj) => (
+                <ProjectOption
+                  key={proj.id}
+                  proj={proj}
+                  isActive={proj.id === selectedProjectId}
+                  onClose={popover.onClose}
+                />
+              ))}
            </VStack>
-          </PopoverContent>
-        </Popover>
-      </NavSidebarOption>
+            <HStack
+              as={Button}
+              variant="ghost"
+              colorScheme="blue"
+              color="blue.400"
+              pr={8}
+              w="full"
+              onClick={createProject}
+            >
+              <Icon as={isLoading ? Spinner : BsPlus} boxSize={6} />
+              <Text>New project</Text>
+            </HStack>
+          </VStack>
+        </PopoverContent>
+      </Popover>
    </VStack>
  );
 }
@@ -160,6 +149,8 @@ const ProjectOption = ({
      bgColor={isActive ? "gray.100" : "transparent"}
      _hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
      p={2}
+      borderRadius={4}
+      spacing={4}
    >
      <Text>{proj.name}</Text>
      <IconButton
--- a/app/src/components/nav/UserMenu.tsx
+++ b/app/src/components/nav/UserMenu.tsx
@@ -9,7 +9,6 @@ import {
  PopoverContent,
  Link,
  type StackProps,
-  Box,
 } from "@chakra-ui/react";
 import { type Session } from "next-auth";
 import { signOut } from "next-auth/react";
@@ -27,30 +26,28 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
    <>
      <Popover placement="right">
        <PopoverTrigger>
-          <Box>
-            <NavSidebarOption>
-              <HStack
-                // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
-                py={2}
-                px={1}
-                spacing={3}
-                {...rest}
-              >
-                {profileImage}
-                <VStack spacing={0} align="start" flex={1} flexShrink={1}>
-                  <Text fontWeight="bold" fontSize="sm">
-                    {user.user.name}
-                  </Text>
-                  <Text color="gray.500" fontSize="xs">
-                    {/* {user.user.email} */}
-                  </Text>
-                </VStack>
-                <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
-              </HStack>
-            </NavSidebarOption>
-          </Box>
+          <NavSidebarOption>
+            <HStack
+              // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
+              py={2}
+              px={1}
+              spacing={3}
+              {...rest}
+            >
+              {profileImage}
+              <VStack spacing={0} align="start" flex={1} flexShrink={1}>
+                <Text fontWeight="bold" fontSize="sm">
+                  {user.user.name}
+                </Text>
+                <Text color="gray.500" fontSize="xs">
+                  {/* {user.user.email} */}
+                </Text>
+              </VStack>
+              <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+            </HStack>
+          </NavSidebarOption>
        </PopoverTrigger>
-        <PopoverContent _focusVisible={{ boxShadow: "unset", outline: "unset" }} maxW="200px">
+        <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} minW={48} w="full">
          <VStack align="stretch" spacing={0}>
            {/* sign out */}
            <HStack
--- a/app/src/components/tooltip/CostTooltip.tsx
+++ b/app/src/components/tooltip/CostTooltip.tsx
@@ -2,14 +2,14 @@ import { HStack, Icon, Text, Tooltip, type TooltipProps, VStack, Divider } from
 import { BsCurrencyDollar } from "react-icons/bs";

 type CostTooltipProps = {
-  promptTokens: number | null;
-  completionTokens: number | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
  cost: number;
 } & TooltipProps;

 export const CostTooltip = ({
-  promptTokens,
-  completionTokens,
+  inputTokens,
+  outputTokens,
  cost,
  children,
  ...props
@@ -36,12 +36,12 @@ export const CostTooltip = ({
            <HStack>
              <VStack w="28" spacing={1}>
                <Text>Prompt</Text>
-                <Text>{promptTokens ?? 0}</Text>
+                <Text>{inputTokens ?? 0}</Text>
              </VStack>
              <Divider borderColor="gray.200" h={8} orientation="vertical" />
              <VStack w="28" spacing={1}>
                <Text whiteSpace="nowrap">Completion</Text>
-                <Text>{completionTokens ?? 0}</Text>
+                <Text>{outputTokens ?? 0}</Text>
              </VStack>
            </HStack>
          </VStack>
--- a/app/src/modelProviders/anthropic-completion/index.ts
+++ b/app/src/modelProviders/anthropic-completion/index.ts
@@ -28,6 +28,10 @@ const modelProvider: AnthropicProvider = {
  inputSchema: inputSchema as JSONSchema4,
  canStream: true,
  getCompletion,
+  getUsage: (input, output) => {
+    // TODO: add usage logic
+    return null;
+  },
  ...frontendModelProvider,
 };

--- a/app/src/modelProviders/openai-ChatCompletion/getCompletion.ts
+++ b/app/src/modelProviders/openai-ChatCompletion/getCompletion.ts
@@ -4,14 +4,10 @@ import {
  type ChatCompletion,
  type CompletionCreateParams,
 } from "openai/resources/chat";
-import { countOpenAIChatTokens } from "~/utils/countTokens";
 import { type CompletionResponse } from "../types";
 import { isArray, isString, omit } from "lodash-es";
 import { openai } from "~/server/utils/openai";
-import { truthyFilter } from "~/utils/utils";
 import { APIError } from "openai";
-import frontendModelProvider from "./frontend";
-import modelProvider, { type SupportedModel } from ".";

 const mergeStreamedChunks = (
  base: ChatCompletion | null,
@@ -60,9 +56,6 @@ export async function getCompletion(
 ): Promise<CompletionResponse<ChatCompletion>> {
  const start = Date.now();
  let finalCompletion: ChatCompletion | null = null;
-  let promptTokens: number | undefined = undefined;
-  let completionTokens: number | undefined = undefined;
-  const modelName = modelProvider.getModel(input) as SupportedModel;

  try {
    if (onStream) {
@@ -86,16 +79,6 @@ export async function getCompletion(
          autoRetry: false,
        };
      }
-      try {
-        promptTokens = countOpenAIChatTokens(modelName, input.messages);
-        completionTokens = countOpenAIChatTokens(
-          modelName,
-          finalCompletion.choices.map((c) => c.message).filter(truthyFilter),
-        );
-      } catch (err) {
-        // TODO handle this, library seems like maybe it doesn't work with function calls?
-        console.error(err);
-      }
    } else {
      const resp = await openai.chat.completions.create(
        { ...input, stream: false },
@@ -104,25 +87,14 @@ export async function getCompletion(
        },
      );
      finalCompletion = resp;
-      promptTokens = resp.usage?.prompt_tokens ?? 0;
-      completionTokens = resp.usage?.completion_tokens ?? 0;
    }
    const timeToComplete = Date.now() - start;

-    const { promptTokenPrice, completionTokenPrice } = frontendModelProvider.models[modelName];
-    let cost = undefined;
-    if (promptTokenPrice && completionTokenPrice && promptTokens && completionTokens) {
-      cost = promptTokens * promptTokenPrice + completionTokens * completionTokenPrice;
-    }
-
    return {
      type: "success",
      statusCode: 200,
      value: finalCompletion,
      timeToComplete,
-      promptTokens,
-      completionTokens,
-      cost,
    };
  } catch (error: unknown) {
    if (error instanceof APIError) {
--- a/app/src/modelProviders/openai-ChatCompletion/index.ts
+++ b/app/src/modelProviders/openai-ChatCompletion/index.ts
@@ -4,6 +4,8 @@ import inputSchema from "./codegen/input.schema.json";
 import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
 import { getCompletion } from "./getCompletion";
 import frontendModelProvider from "./frontend";
+import { countOpenAIChatTokens } from "~/utils/countTokens";
+import { truthyFilter } from "~/utils/utils";

 const supportedModels = [
  "gpt-4-0613",
@@ -39,6 +41,41 @@ const modelProvider: OpenaiChatModelProvider = {
  inputSchema: inputSchema as JSONSchema4,
  canStream: true,
  getCompletion,
+  getUsage: (input, output) => {
+    if (output.choices.length === 0) return null;
+
+    const model = modelProvider.getModel(input);
+    if (!model) return null;
+
+    let inputTokens: number;
+    let outputTokens: number;
+
+    if (output.usage) {
+      inputTokens = output.usage.prompt_tokens;
+      outputTokens = output.usage.completion_tokens;
+    } else {
+      try {
+        inputTokens = countOpenAIChatTokens(model, input.messages);
+        outputTokens = countOpenAIChatTokens(
+          model,
+          output.choices.map((c) => c.message).filter(truthyFilter),
+        );
+      } catch (err) {
+        inputTokens = 0;
+        outputTokens = 0;
+        // TODO handle this, library seems like maybe it doesn't work with function calls?
+        console.error(err);
+      }
+    }
+
+    const { promptTokenPrice, completionTokenPrice } = frontendModelProvider.models[model];
+    let cost = undefined;
+    if (promptTokenPrice && completionTokenPrice && inputTokens && outputTokens) {
+      cost = inputTokens * promptTokenPrice + outputTokens * completionTokenPrice;
+    }
+
+    return { inputTokens: inputTokens, outputTokens: outputTokens, cost };
+  },
  ...frontendModelProvider,
 };

--- a/app/src/modelProviders/replicate-llama2/index.ts
+++ b/app/src/modelProviders/replicate-llama2/index.ts
@@ -75,6 +75,10 @@ const modelProvider: ReplicateLlama2Provider = {
  },
  canStream: true,
  getCompletion,
+  getUsage: (input, output) => {
+    // TODO: add usage logic
+    return null;
+  },
  ...frontendModelProvider,
 };

--- a/app/src/modelProviders/types.ts
+++ b/app/src/modelProviders/types.ts
@@ -43,9 +43,6 @@ export type CompletionResponse<T> =
      value: T;
      timeToComplete: number;
      statusCode: number;
-      promptTokens?: number;
-      completionTokens?: number;
-      cost?: number;
    };

 export type ModelProvider<SupportedModels extends string, InputSchema, OutputSchema> = {
@@ -56,6 +53,10 @@ export type ModelProvider<SupportedModels extends string, InputSchema, OutputSch
    input: InputSchema,
    onStream: ((partialOutput: OutputSchema) => void) | null,
  ) => Promise<CompletionResponse<OutputSchema>>;
+  getUsage: (
+    input: InputSchema,
+    output: OutputSchema,
+  ) => { gpuRuntime?: number; inputTokens?: number; outputTokens?: number; cost?: number } | null;

  // This is just a convenience for type inference, don't use it at runtime
  _outputSchema?: OutputSchema | null;
--- a/app/src/pages/logged-calls/index.tsx
+++ b/app/src/pages/logged-calls/index.tsx
@@ -18,26 +18,15 @@ import {
  Breadcrumb,
  BreadcrumbItem,
 } from "@chakra-ui/react";
-import {
-  LineChart,
-  Line,
-  XAxis,
-  YAxis,
-  CartesianGrid,
-  Tooltip,
-  Legend,
-  ResponsiveContainer,
-} from "recharts";
 import { Ban, DollarSign, Hash } from "lucide-react";
-import { useMemo } from "react";

 import AppShell from "~/components/nav/AppShell";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
 import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 import { useSelectedProject } from "~/utils/hooks";
-import dayjs from "~/utils/dayjs";
 import { api } from "~/utils/api";
 import LoggedCallTable from "~/components/dashboard/LoggedCallTable";
+import UsageGraph from "~/components/dashboard/UsageGraph";

 export default function LoggedCalls() {
  const { data: selectedProject } = useSelectedProject();
@@ -47,18 +36,8 @@ export default function LoggedCalls() {
    { enabled: !!selectedProject },
  );

-  const data = useMemo(() => {
-    return (
-      stats.data?.periods.map(({ period, numQueries, totalCost }) => ({
-        period,
-        Requests: numQueries,
-        "Total Spent (USD)": parseFloat(totalCost.toString()),
-      })) || []
-    );
-  }, [stats.data]);
-
  return (
-    <AppShell requireAuth>
+    <AppShell title="Logged Calls" requireAuth>
      <PageHeaderContainer>
        <Breadcrumb>
          <BreadcrumbItem>
@@ -83,39 +62,7 @@ export default function LoggedCalls() {
                </Heading>
              </CardHeader>
              <CardBody>
-                <ResponsiveContainer width="100%" height={400}>
-                  <LineChart data={data} margin={{ top: 5, right: 20, left: 10, bottom: 5 }}>
-                    <XAxis
-                      dataKey="period"
-                      tickFormatter={(str: string) => dayjs(str).format("MMM D")}
-                    />
-                    <YAxis yAxisId="left" dataKey="Requests" orientation="left" stroke="#8884d8" />
-                    <YAxis
-                      yAxisId="right"
-                      dataKey="Total Spent (USD)"
-                      orientation="right"
-                      unit="$"
-                      stroke="#82ca9d"
-                    />
-                    <Tooltip />
-                    <Legend />
-                    <CartesianGrid stroke="#f5f5f5" />
-                    <Line
-                      dataKey="Requests"
-                      stroke="#8884d8"
-                      yAxisId="left"
-                      dot={false}
-                      strokeWidth={2}
-                    />
-                    <Line
-                      dataKey="Total Spent (USD)"
-                      stroke="#82ca9d"
-                      yAxisId="right"
-                      dot={false}
-                      strokeWidth={2}
-                    />
-                  </LineChart>
-                </ResponsiveContainer>
+                <UsageGraph />
              </CardBody>
            </Card>
            <VStack spacing="4" width="300px" align="stretch">
@@ -127,7 +74,7 @@ export default function LoggedCalls() {
                      <Icon as={DollarSign} boxSize={4} color="gray.500" />
                    </HStack>
                    <StatNumber>
-                      ${parseFloat(stats.data?.totals?.totalCost?.toString() ?? "0").toFixed(2)}
+                      ${parseFloat(stats.data?.totals?.cost?.toString() ?? "0").toFixed(3)}
                    </StatNumber>
                  </Stat>
                </CardBody>
--- a/app/src/pages/project/settings/index.tsx
+++ b/app/src/pages/project/settings/index.tsx
@@ -5,7 +5,6 @@ import {
  type TextProps,
  VStack,
  HStack,
-  Input,
  Button,
  Divider,
  Icon,
@@ -21,6 +20,7 @@ import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
 import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 import CopiableCode from "~/components/CopiableCode";
 import { DeleteProjectDialog } from "~/components/projectSettings/DeleteProjectDialog";
+import AutoResizeTextArea from "~/components/AutoResizeTextArea";

 export default function Settings() {
  const utils = api.useContext();
@@ -38,7 +38,10 @@ export default function Settings() {
        id: selectedProject.id,
        updates: { name },
      });
-      await Promise.all([utils.projects.get.invalidate({ id: selectedProject.id })]);
+      await Promise.all([
+        utils.projects.get.invalidate({ id: selectedProject.id }),
+        utils.projects.list.invalidate(),
+      ]);
    }
  }, [updateMutation, selectedProject]);

@@ -84,7 +87,7 @@ export default function Settings() {
              <Text fontWeight="bold" fontSize="xl">
                Display Name
              </Text>
-              <Input
+              <AutoResizeTextArea
                w="full"
                maxW={600}
                value={name}
@@ -136,10 +139,13 @@ export default function Settings() {
                  variant="outline"
                  borderRadius={4}
                  mt={2}
+                  height="auto"
                  onClick={deleteProjectOpen.onOpen}
                >
                  <Icon as={BsTrash} />
-                  <Text>Delete {selectedProject?.name}</Text>
+                  <Text overflowWrap="break-word" whiteSpace="normal" py={2}>
+                    Delete {selectedProject?.name}
+                  </Text>
                </HStack>
              </VStack>
            )}
--- a/app/src/server/api/root.router.ts
+++ b/app/src/server/api/root.router.ts
@@ -3,7 +3,7 @@ import { createTRPCRouter } from "~/server/api/trpc";
 import { experimentsRouter } from "./routers/experiments.router";
 import { scenariosRouter } from "./routers/scenarios.router";
 import { scenarioVariantCellsRouter } from "./routers/scenarioVariantCells.router";
-import { templateVarsRouter } from "./routers/templateVariables.router";
+import { scenarioVarsRouter } from "./routers/scenarioVariables.router";
 import { evaluationsRouter } from "./routers/evaluations.router";
 import { worldChampsRouter } from "./routers/worldChamps.router";
 import { datasetsRouter } from "./routers/datasets.router";
@@ -22,7 +22,7 @@ export const appRouter = createTRPCRouter({
  experiments: experimentsRouter,
  scenarios: scenariosRouter,
  scenarioVariantCells: scenarioVariantCellsRouter,
-  templateVars: templateVarsRouter,
+  scenarioVars: scenarioVarsRouter,
  evaluations: evaluationsRouter,
  worldChamps: worldChampsRouter,
  datasets: datasetsRouter,
--- a/app/src/server/api/routers/dashboard.router.ts
+++ b/app/src/server/api/routers/dashboard.router.ts
@@ -24,9 +24,9 @@ export const dashboardRouter = createTRPCRouter({
        )
        .where("projectId", "=", input.projectId)
        .select(({ fn }) => [
-          sql<Date>`date_trunc('day', "LoggedCallModelResponse"."startTime")`.as("period"),
+          sql<Date>`date_trunc('day', "LoggedCallModelResponse"."requestedAt")`.as("period"),
          sql<number>`count("LoggedCall"."id")::int`.as("numQueries"),
-          fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
+          fn.sum(fn.coalesce("LoggedCallModelResponse.cost", sql<number>`0`)).as("cost"),
        ])
        .groupBy("period")
        .orderBy("period")
@@ -57,7 +57,7 @@ export const dashboardRouter = createTRPCRouter({
          backfilledPeriods.unshift({
            period: dayjs(dayToMatch).toDate(),
            numQueries: 0,
-            totalCost: 0,
+            cost: 0,
          });
        }
        dayToMatch = dayToMatch.subtract(1, "day");
@@ -72,7 +72,7 @@ export const dashboardRouter = createTRPCRouter({
        )
        .where("projectId", "=", input.projectId)
        .select(({ fn }) => [
-          fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
+          fn.sum(fn.coalesce("LoggedCallModelResponse.cost", sql<number>`0`)).as("cost"),
          fn.count("LoggedCall.id").as("numQueries"),
        ])
        .executeTakeFirst();
@@ -85,8 +85,8 @@ export const dashboardRouter = createTRPCRouter({
          "LoggedCall.id",
          "LoggedCallModelResponse.originalLoggedCallId",
        )
-        .select(({ fn }) => [fn.count("LoggedCall.id").as("count"), "respStatus as code"])
-        .where("respStatus", ">", 200)
+        .select(({ fn }) => [fn.count("LoggedCall.id").as("count"), "statusCode as code"])
+        .where("statusCode", ">", 200)
        .groupBy("code")
        .orderBy("count", "desc")
        .execute();
@@ -108,7 +108,7 @@ export const dashboardRouter = createTRPCRouter({
  // https://discord.com/channels/966627436387266600/1122258443886153758/1122258443886153758
  loggedCalls: publicProcedure.input(z.object({})).query(async ({ input }) => {
    const loggedCalls = await prisma.loggedCall.findMany({
-      orderBy: { startTime: "desc" },
+      orderBy: { requestedAt: "desc" },
      include: { tags: true, modelResponse: true },
      take: 20,
    });
--- a/app/src/server/api/routers/experiments.router.ts
+++ b/app/src/server/api/routers/experiments.router.ts
@@ -227,7 +227,7 @@ export const experimentsRouter = createTRPCRouter({
            ...modelResponseData,
            id: newModelResponseId,
            scenarioVariantCellId: newCellId,
-            output: (modelResponse.output as Prisma.InputJsonValue) ?? undefined,
+            respPayload: (modelResponse.respPayload as Prisma.InputJsonValue) ?? undefined,
          });
          for (const evaluation of outputEvaluations) {
            outputEvaluationsToCreate.push({
--- a/app/src/server/api/routers/externalApi.router.ts
+++ b/app/src/server/api/routers/externalApi.router.ts
@@ -7,6 +7,11 @@ import { TRPCError } from "@trpc/server";
 import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
 import { hashRequest } from "~/server/utils/hashObject";
+import modelProvider from "~/modelProviders/openai-ChatCompletion";
+import {
+  type ChatCompletion,
+  type CompletionCreateParams,
+} from "openai/resources/chat/completions";

 const reqValidator = z.object({
  model: z.string(),
@@ -16,11 +21,6 @@ const reqValidator = z.object({
 const respValidator = z.object({
  id: z.string(),
  model: z.string(),
-  usage: z.object({
-    total_tokens: z.number(),
-    prompt_tokens: z.number(),
-    completion_tokens: z.number(),
-  }),
  choices: z.array(
    z.object({
      finish_reason: z.string(),
@@ -35,11 +35,12 @@ export const externalApiRouter = createTRPCRouter({
        method: "POST",
        path: "/v1/check-cache",
        description: "Check if a prompt is cached",
+        protect: true,
      },
    })
    .input(
      z.object({
-        startTime: z.number().describe("Unix timestamp in milliseconds"),
+        requestedAt: z.number().describe("Unix timestamp in milliseconds"),
        reqPayload: z.unknown().describe("JSON-encoded request payload"),
        tags: z
          .record(z.string())
@@ -69,15 +70,9 @@ export const externalApiRouter = createTRPCRouter({
      const cacheKey = hashRequest(key.projectId, reqPayload as JsonValue);

      const existingResponse = await prisma.loggedCallModelResponse.findFirst({
-        where: {
-          cacheKey,
-        },
-        include: {
-          originalLoggedCall: true,
-        },
-        orderBy: {
-          startTime: "desc",
-        },
+        where: { cacheKey },
+        include: { originalLoggedCall: true },
+        orderBy: { requestedAt: "desc" },
      });

      if (!existingResponse) return { respPayload: null };
@@ -85,7 +80,7 @@ export const externalApiRouter = createTRPCRouter({
      await prisma.loggedCall.create({
        data: {
          projectId: key.projectId,
-          startTime: new Date(input.startTime),
+          requestedAt: new Date(input.requestedAt),
          cacheHit: true,
          modelResponseId: existingResponse.id,
        },
@@ -102,16 +97,17 @@ export const externalApiRouter = createTRPCRouter({
        method: "POST",
        path: "/v1/report",
        description: "Report an API call",
+        protect: true,
      },
    })
    .input(
      z.object({
-        startTime: z.number().describe("Unix timestamp in milliseconds"),
-        endTime: z.number().describe("Unix timestamp in milliseconds"),
+        requestedAt: z.number().describe("Unix timestamp in milliseconds"),
+        receivedAt: z.number().describe("Unix timestamp in milliseconds"),
        reqPayload: z.unknown().describe("JSON-encoded request payload"),
        respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
-        respStatus: z.number().optional().describe("HTTP status code of response"),
-        error: z.string().optional().describe("User-friendly error message"),
+        statusCode: z.number().optional().describe("HTTP status code of response"),
+        errorMessage: z.string().optional().describe("User-friendly error message"),
        tags: z
          .record(z.string())
          .optional()
@@ -122,6 +118,7 @@ export const externalApiRouter = createTRPCRouter({
    )
    .output(z.void())
    .mutation(async ({ input, ctx }) => {
+      console.log("GOT TAGS", input.tags);
      const apiKey = ctx.apiKey;
      if (!apiKey) {
        throw new TRPCError({ code: "UNAUTHORIZED" });
@@ -140,14 +137,20 @@ export const externalApiRouter = createTRPCRouter({
      const newLoggedCallId = uuidv4();
      const newModelResponseId = uuidv4();

-      const usage = respPayload.success ? respPayload.data.usage : undefined;
+      let usage;
+      if (reqPayload.success && respPayload.success) {
+        usage = modelProvider.getUsage(
+          input.reqPayload as CompletionCreateParams,
+          input.respPayload as ChatCompletion,
+        );
+      }

      await prisma.$transaction([
        prisma.loggedCall.create({
          data: {
            id: newLoggedCallId,
            projectId: key.projectId,
-            startTime: new Date(input.startTime),
+            requestedAt: new Date(input.requestedAt),
            cacheHit: false,
          },
        }),
@@ -155,20 +158,17 @@ export const externalApiRouter = createTRPCRouter({
          data: {
            id: newModelResponseId,
            originalLoggedCallId: newLoggedCallId,
-            startTime: new Date(input.startTime),
-            endTime: new Date(input.endTime),
+            requestedAt: new Date(input.requestedAt),
+            receivedAt: new Date(input.receivedAt),
            reqPayload: input.reqPayload as Prisma.InputJsonValue,
            respPayload: input.respPayload as Prisma.InputJsonValue,
-            respStatus: input.respStatus,
-            error: input.error,
-            durationMs: input.endTime - input.startTime,
-            ...(respPayload.success
-              ? {
-                  cacheKey: requestHash,
-                  inputTokens: usage ? usage.prompt_tokens : undefined,
-                  outputTokens: usage ? usage.completion_tokens : undefined,
-                }
-              : null),
+            statusCode: input.statusCode,
+            errorMessage: input.errorMessage,
+            durationMs: input.receivedAt - input.requestedAt,
+            cacheKey: respPayload.success ? requestHash : null,
+            inputTokens: usage?.inputTokens,
+            outputTokens: usage?.outputTokens,
+            cost: usage?.cost,
          },
        }),
        // Avoid foreign key constraint error by updating the logged call after the model response is created
@@ -182,24 +182,22 @@ export const externalApiRouter = createTRPCRouter({
        }),
      ]);

-      if (input.tags) {
-        const tagsToCreate = Object.entries(input.tags).map(([name, value]) => ({
-          loggedCallId: newLoggedCallId,
-          // sanitize tags
-          name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
-          value,
-        }));
+      const tagsToCreate = Object.entries(input.tags ?? {}).map(([name, value]) => ({
+        loggedCallId: newLoggedCallId,
+        // sanitize tags
+        name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
+        value,
+      }));

-        if (reqPayload.success) {
-          tagsToCreate.push({
-            loggedCallId: newLoggedCallId,
-            name: "$model",
-            value: reqPayload.data.model,
-          });
-        }
-        await prisma.loggedCallTag.createMany({
-          data: tagsToCreate,
+      if (reqPayload.success) {
+        tagsToCreate.push({
+          loggedCallId: newLoggedCallId,
+          name: "$model",
+          value: reqPayload.data.model,
        });
      }
+      await prisma.loggedCallTag.createMany({
+        data: tagsToCreate,
+      });
    }),
 });
--- a/app/src/server/api/routers/promptVariants.router.ts
+++ b/app/src/server/api/routers/promptVariants.router.ts
@@ -3,7 +3,7 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
 import { prisma } from "~/server/db";
 import { Prisma } from "@prisma/client";
 import { generateNewCell } from "~/server/utils/generateNewCell";
-import userError from "~/server/utils/error";
+import { error, success } from "~/utils/errorHandling/standardResponses";
 import { recordExperimentUpdated } from "~/server/utils/recordExperimentUpdated";
 import { reorderPromptVariants } from "~/server/utils/reorderPromptVariants";
 import { type PromptVariant } from "@prisma/client";
@@ -55,7 +55,7 @@ export const promptVariantsRouter = createTRPCRouter({
        where: {
          modelResponse: {
            outdated: false,
-            output: { not: Prisma.AnyNull },
+            respPayload: { not: Prisma.AnyNull },
            scenarioVariantCell: {
              promptVariant: {
                id: input.variantId,
@@ -100,7 +100,7 @@ export const promptVariantsRouter = createTRPCRouter({
          modelResponses: {
            some: {
              outdated: false,
-              output: {
+              respPayload: {
                not: Prisma.AnyNull,
              },
            },
@@ -111,7 +111,7 @@ export const promptVariantsRouter = createTRPCRouter({
      const overallTokens = await prisma.modelResponse.aggregate({
        where: {
          outdated: false,
-          output: {
+          respPayload: {
            not: Prisma.AnyNull,
          },
          scenarioVariantCell: {
@@ -123,13 +123,13 @@ export const promptVariantsRouter = createTRPCRouter({
        },
        _sum: {
          cost: true,
-          promptTokens: true,
-          completionTokens: true,
+          inputTokens: true,
+          outputTokens: true,
        },
      });

-      const promptTokens = overallTokens._sum?.promptTokens ?? 0;
-      const completionTokens = overallTokens._sum?.completionTokens ?? 0;
+      const inputTokens = overallTokens._sum?.inputTokens ?? 0;
+      const outputTokens = overallTokens._sum?.outputTokens ?? 0;

      const awaitingEvals = !!evalResults.find(
        (result) => result.totalCount < scenarioCount * evals.length,
@@ -137,8 +137,8 @@ export const promptVariantsRouter = createTRPCRouter({

      return {
        evalResults,
-        promptTokens,
-        completionTokens,
+        inputTokens,
+        outputTokens,
        overallCost: overallTokens._sum?.cost ?? 0,
        scenarioCount,
        outputCount,
@@ -315,7 +315,7 @@ export const promptVariantsRouter = createTRPCRouter({
      const constructedPrompt = await parsePromptConstructor(existing.promptConstructor);

      if ("error" in constructedPrompt) {
-        return userError(constructedPrompt.error);
+        return error(constructedPrompt.error);
      }

      const model = input.newModel
@@ -353,7 +353,7 @@ export const promptVariantsRouter = createTRPCRouter({
      const parsedPrompt = await parsePromptConstructor(input.promptConstructor);

      if ("error" in parsedPrompt) {
-        return userError(parsedPrompt.error);
+        return error(parsedPrompt.error);
      }

      // Create a duplicate with only the config changed
@@ -398,7 +398,7 @@ export const promptVariantsRouter = createTRPCRouter({
        });
      }

-      return { status: "ok" } as const;
+      return success();
    }),

  reorder: protectedProcedure
--- a/app/src/server/api/routers/scenarioVariables.router.ts
+++ b/app/src/server/api/routers/scenarioVariables.router.ts
@@ -0,0 +1,143 @@
+import { type TemplateVariable } from "@prisma/client";
+import { sql } from "kysely";
+import { z } from "zod";
+import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
+import { kysely, prisma } from "~/server/db";
+import { error, success } from "~/utils/errorHandling/standardResponses";
+import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl";
+
+export const scenarioVarsRouter = createTRPCRouter({
+  create: protectedProcedure
+    .input(z.object({ experimentId: z.string(), label: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyExperiment(input.experimentId, ctx);
+
+      // Make sure there isn't an existing variable with the same name
+      const existingVariable = await prisma.templateVariable.findFirst({
+        where: {
+          experimentId: input.experimentId,
+          label: input.label,
+        },
+      });
+      if (existingVariable) {
+        return error(`A variable named ${input.label} already exists.`);
+      }
+
+      await prisma.templateVariable.create({
+        data: {
+          experimentId: input.experimentId,
+          label: input.label,
+        },
+      });
+
+      return success();
+    }),
+
+  rename: protectedProcedure
+    .input(z.object({ id: z.string(), label: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const templateVariable = await prisma.templateVariable.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+      await requireCanModifyExperiment(templateVariable.experimentId, ctx);
+
+      // Make sure there isn't an existing variable with the same name
+      const existingVariable = await prisma.templateVariable.findFirst({
+        where: {
+          experimentId: templateVariable.experimentId,
+          label: input.label,
+        },
+      });
+      if (existingVariable) {
+        return error(`A variable named ${input.label} already exists.`);
+      }
+
+      await renameTemplateVariable(templateVariable, input.label);
+      return success();
+    }),
+
+  delete: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+
+      await requireCanModifyExperiment(experimentId, ctx);
+
+      await prisma.templateVariable.delete({ where: { id: input.id } });
+    }),
+
+  list: publicProcedure
+    .input(z.object({ experimentId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewExperiment(input.experimentId, ctx);
+      return await prisma.templateVariable.findMany({
+        where: {
+          experimentId: input.experimentId,
+        },
+        orderBy: {
+          createdAt: "asc",
+        },
+        select: {
+          id: true,
+          label: true,
+        },
+      });
+    }),
+});
+
+export const renameTemplateVariable = async (
+  templateVariable: TemplateVariable,
+  newLabel: string,
+) => {
+  const { experimentId } = templateVariable;
+
+  await kysely.transaction().execute(async (trx) => {
+    await trx
+      .updateTable("TemplateVariable")
+      .set({
+        label: newLabel,
+      })
+      .where("id", "=", templateVariable.id)
+      .execute();
+
+    await sql`
+      CREATE TEMP TABLE "TempTestScenario" AS
+      SELECT *
+      FROM "TestScenario"
+      WHERE "experimentId" = ${experimentId}
+
+      -- Only copy the rows that actually have a value for the variable, no reason to churn the rest and simplifies the update.
+      AND "variableValues"->${templateVariable.label} IS NOT NULL
+      `.execute(trx);
+
+    await sql`
+      UPDATE "TempTestScenario"
+      SET "variableValues" = jsonb_set(
+        "variableValues",
+        ${`{${newLabel}}`},
+        "variableValues"->${templateVariable.label}
+      ) - ${templateVariable.label},
+      "updatedAt" = NOW(),
+      "id" = uuid_generate_v4()
+    `.execute(trx);
+
+    // Print the contents of the temp table
+    const results = await sql`SELECT * FROM "TempTestScenario"`.execute(trx);
+    console.log(results.rows);
+
+    await trx
+      .updateTable("TestScenario")
+      .set({
+        visible: false,
+      })
+      .where("experimentId", "=", experimentId)
+      .execute();
+
+    await sql`
+      INSERT INTO "TestScenario" (id, "variableValues", "uiId", visible, "sortIndex", "experimentId", "createdAt", "updatedAt")
+      SELECT * FROM "TempTestScenario";
+    `.execute(trx);
+  });
+};
--- a/app/src/server/api/routers/templateVariables.router.test.ts
+++ b/app/src/server/api/routers/templateVariables.router.test.ts
@@ -0,0 +1,110 @@
+import { expect, it } from "vitest";
+import { prisma } from "~/server/db";
+import { renameTemplateVariable } from "./scenarioVariables.router";
+
+const createExperiment = async () => {
+  return await prisma.experiment.create({
+    data: {
+      label: "Test Experiment",
+      project: {
+        create: {},
+      },
+    },
+  });
+};
+
+const createTemplateVar = async (experimentId: string, label: string) => {
+  return await prisma.templateVariable.create({
+    data: {
+      experimentId,
+      label,
+    },
+  });
+};
+
+it("renames templateVariables", async () => {
+  // Create experiments concurrently
+  const [exp1, exp2] = await Promise.all([createExperiment(), createExperiment()]);
+
+  // Create template variables concurrently
+  const [exp1Var, exp2Var1, exp2Var2] = await Promise.all([
+    createTemplateVar(exp1.id, "input1"),
+    createTemplateVar(exp2.id, "input1"),
+    createTemplateVar(exp2.id, "input2"),
+  ]);
+
+  // Create test scenarios concurrently
+  const [exp1Scenario, exp2Scenario, exp2HiddenScenario] = await Promise.all([
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp1.id,
+        visible: true,
+        variableValues: { input1: "test" },
+      },
+    }),
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp2.id,
+        visible: true,
+        variableValues: { input1: "test1", otherInput: "otherTest" },
+      },
+    }),
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp2.id,
+        visible: false,
+        variableValues: { otherInput: "otherTest2" },
+      },
+    }),
+  ]);
+
+  await renameTemplateVariable(exp2Var1, "input1-renamed");
+
+  expect(await prisma.templateVariable.findUnique({ where: { id: exp2Var1.id } })).toMatchObject({
+    label: "input1-renamed",
+  });
+
+  // It shouldn't mess with unrelated experiments
+  expect(await prisma.testScenario.findUnique({ where: { id: exp1Scenario.id } })).toMatchObject({
+    visible: true,
+    variableValues: { input1: "test" },
+  });
+
+  // Make sure there are a total of 4 scenarios for exp2
+  expect(
+    await prisma.testScenario.count({
+      where: {
+        experimentId: exp2.id,
+      },
+    }),
+  ).toBe(3);
+
+  // It shouldn't mess with the existing scenarios, except to hide them
+  expect(await prisma.testScenario.findUnique({ where: { id: exp2Scenario.id } })).toMatchObject({
+    visible: false,
+    variableValues: { input1: "test1", otherInput: "otherTest" },
+  });
+
+  // It should create a new scenario with the new variable name
+  const newScenario1 = await prisma.testScenario.findFirst({
+    where: {
+      experimentId: exp2.id,
+      variableValues: { equals: { "input1-renamed": "test1", otherInput: "otherTest" } },
+    },
+  });
+
+  expect(newScenario1).toMatchObject({
+    visible: true,
+  });
+
+  const newScenario2 = await prisma.testScenario.findFirst({
+    where: {
+      experimentId: exp2.id,
+      variableValues: { equals: { otherInput: "otherTest2" } },
+    },
+  });
+
+  expect(newScenario2).toMatchObject({
+    visible: false,
+  });
+});
--- a/app/src/server/api/routers/templateVariables.router.ts
+++ b/app/src/server/api/routers/templateVariables.router.ts
@@ -1,49 +0,0 @@
-import { z } from "zod";
-import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
-import { prisma } from "~/server/db";
-import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl";
-
-export const templateVarsRouter = createTRPCRouter({
-  create: protectedProcedure
-    .input(z.object({ experimentId: z.string(), label: z.string() }))
-    .mutation(async ({ input, ctx }) => {
-      await requireCanModifyExperiment(input.experimentId, ctx);
-
-      await prisma.templateVariable.create({
-        data: {
-          experimentId: input.experimentId,
-          label: input.label,
-        },
-      });
-    }),
-
-  delete: protectedProcedure
-    .input(z.object({ id: z.string() }))
-    .mutation(async ({ input, ctx }) => {
-      const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({
-        where: { id: input.id },
-      });
-
-      await requireCanModifyExperiment(experimentId, ctx);
-
-      await prisma.templateVariable.delete({ where: { id: input.id } });
-    }),
-
-  list: publicProcedure
-    .input(z.object({ experimentId: z.string() }))
-    .query(async ({ input, ctx }) => {
-      await requireCanViewExperiment(input.experimentId, ctx);
-      return await prisma.templateVariable.findMany({
-        where: {
-          experimentId: input.experimentId,
-        },
-        orderBy: {
-          createdAt: "asc",
-        },
-        select: {
-          id: true,
-          label: true,
-        },
-      });
-    }),
-});
--- a/app/src/server/api/trpc.ts
+++ b/app/src/server/api/trpc.ts
@@ -64,7 +64,7 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
  // Get the session from the server using the getServerSession wrapper function
  const session = await getServerAuthSession({ req, res });

-  const apiKey = req.headers["x-openpipe-api-key"] as string | null;
+  const apiKey = req.headers.authorization?.split(" ")[1] as string | null;

  return createInnerTRPCContext({
    session,
--- a/app/src/server/scripts/test-queries.ts
+++ b/app/src/server/scripts/test-queries.ts
@@ -1,63 +0,0 @@
-import dayjs from "dayjs";
-import { prisma } from "../db";
-
-const projectId = "1234";
-
-// Find all calls in the last 24 hours
-const responses = await prisma.loggedCall.findMany({
-  where: {
-    projectId: projectId,
-    startTime: {
-      gt: dayjs()
-        .subtract(24 * 3600)
-        .toDate(),
-    },
-  },
-  include: {
-    modelResponse: true,
-  },
-  orderBy: {
-    startTime: "desc",
-  },
-});
-
-// Find all calls in the last 24 hours with promptId 'hello-world'
-const helloWorld = await prisma.loggedCall.findMany({
-  where: {
-    projectId: projectId,
-    startTime: {
-      gt: dayjs()
-        .subtract(24 * 3600)
-        .toDate(),
-    },
-    tags: {
-      some: {
-        name: "promptId",
-        value: "hello-world",
-      },
-    },
-  },
-  include: {
-    modelResponse: true,
-  },
-  orderBy: {
-    startTime: "desc",
-  },
-});
-
-// Total spent on OpenAI in the last month
-const totalSpent = await prisma.loggedCallModelResponse.aggregate({
-  _sum: {
-    totalCost: true,
-  },
-  where: {
-    originalLoggedCall: {
-      projectId: projectId,
-    },
-    startTime: {
-      gt: dayjs()
-        .subtract(30 * 24 * 3600)
-        .toDate(),
-    },
-  },
-});
--- a/app/src/server/tasks/queryModel.task.ts
+++ b/app/src/server/tasks/queryModel.task.ts
@@ -99,26 +99,27 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
      }
    : null;

-  const inputHash = hashObject(prompt as JsonValue);
+  const cacheKey = hashObject(prompt as JsonValue);

  let modelResponse = await prisma.modelResponse.create({
    data: {
-      inputHash,
+      cacheKey,
      scenarioVariantCellId: cellId,
      requestedAt: new Date(),
    },
  });
  const response = await provider.getCompletion(prompt.modelInput, onStream);
  if (response.type === "success") {
+    const usage = provider.getUsage(prompt.modelInput, response.value);
    modelResponse = await prisma.modelResponse.update({
      where: { id: modelResponse.id },
      data: {
-        output: response.value as Prisma.InputJsonObject,
+        respPayload: response.value as Prisma.InputJsonObject,
        statusCode: response.statusCode,
        receivedAt: new Date(),
-        promptTokens: response.promptTokens,
-        completionTokens: response.completionTokens,
-        cost: response.cost,
+        inputTokens: usage?.inputTokens,
+        outputTokens: usage?.outputTokens,
+        cost: usage?.cost,
      },
    });

--- a/app/src/server/utils/error.ts
+++ b/app/src/server/utils/error.ts
@@ -1,6 +0,0 @@
-export default function userError(message: string): { status: "error"; message: string } {
-  return {
-    status: "error",
-    message,
-  };
-}
--- a/app/src/server/utils/evaluations.ts
+++ b/app/src/server/utils/evaluations.ts
@@ -51,7 +51,7 @@ export const runAllEvals = async (experimentId: string) => {
  const outputs = await prisma.modelResponse.findMany({
    where: {
      outdated: false,
-      output: {
+      respPayload: {
        not: Prisma.AnyNull,
      },
      scenarioVariantCell: {
--- a/app/src/server/utils/generateNewCell.ts
+++ b/app/src/server/utils/generateNewCell.ts
@@ -57,7 +57,7 @@ export const generateNewCell = async (
    return;
  }

-  const inputHash = hashObject(parsedConstructFn);
+  const cacheKey = hashObject(parsedConstructFn);

  cell = await prisma.scenarioVariantCell.create({
    data: {
@@ -73,8 +73,8 @@ export const generateNewCell = async (

  const matchingModelResponse = await prisma.modelResponse.findFirst({
    where: {
-      inputHash,
-      output: {
+      cacheKey,
+      respPayload: {
        not: Prisma.AnyNull,
      },
    },
@@ -92,7 +92,7 @@ export const generateNewCell = async (
      data: {
        ...omit(matchingModelResponse, ["id", "scenarioVariantCell"]),
        scenarioVariantCellId: cell.id,
-        output: matchingModelResponse.output as Prisma.InputJsonValue,
+        respPayload: matchingModelResponse.respPayload as Prisma.InputJsonValue,
      },
    });

--- a/app/src/server/utils/runOneEval.ts
+++ b/app/src/server/utils/runOneEval.ts
@@ -71,7 +71,7 @@ export const runOneEval = async (
  provider: SupportedProvider,
 ): Promise<{ result: number; details?: string }> => {
  const modelProvider = modelProviders[provider];
-  const message = modelProvider.normalizeOutput(modelResponse.output);
+  const message = modelProvider.normalizeOutput(modelResponse.respPayload);

  if (!message) return { result: 0 };

--- a/app/src/state/persist.ts
+++ b/app/src/state/persist.ts
@@ -0,0 +1,13 @@
+import { type PersistOptions } from "zustand/middleware/persist";
+import { type State } from "./store";
+
+export const stateToPersist = {
+  selectedProjectId: null as string | null,
+};
+
+export const persistOptions: PersistOptions<State, typeof stateToPersist> = {
+  name: "persisted-app-store",
+  partialize: (state) => ({
+    selectedProjectId: state.selectedProjectId,
+  }),
+};
--- a/app/src/state/sharedVariantEditor.slice.ts
+++ b/app/src/state/sharedVariantEditor.slice.ts
@@ -8,9 +8,9 @@ export const editorBackground = "#fafafa";
 export type SharedVariantEditorSlice = {
  monaco: null | ReturnType<typeof loader.__getMonacoInstance>;
  loadMonaco: () => Promise<void>;
-  scenarios: RouterOutputs["scenarios"]["list"]["scenarios"];
+  scenarioVars: RouterOutputs["scenarioVars"]["list"];
  updateScenariosModel: () => void;
-  setScenarios: (scenarios: RouterOutputs["scenarios"]["list"]["scenarios"]) => void;
+  setScenarioVars: (scenarioVars: RouterOutputs["scenarioVars"]["list"]) => void;
 };

 export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> = (set, get) => ({
@@ -60,10 +60,10 @@ export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> =
    });
    get().sharedVariantEditor.updateScenariosModel();
  },
-  scenarios: [],
-  setScenarios: (scenarios) => {
+  scenarioVars: [],
+  setScenarioVars: (scenarios) => {
    set((state) => {
-      state.sharedVariantEditor.scenarios = scenarios;
+      state.sharedVariantEditor.scenarioVars = scenarios;
    });

    get().sharedVariantEditor.updateScenariosModel();
@@ -74,16 +74,15 @@ export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> =
    if (!monaco) return;

    const modelContents = `    
-    const scenarios = ${JSON.stringify(
-      get().sharedVariantEditor.scenarios.map((s) => s.variableValues),
-      null,
-      2,
-    )} as const;
-    
-    type Scenario = typeof scenarios[number];
-    declare var scenario: Scenario | { [key: string]: string };
+    declare var scenario: {
+      ${get()
+        .sharedVariantEditor.scenarioVars.map((s) => `${s.label}: string;`)
+        .join("\n")}
+    };
    `;

+    console.log(modelContents);
+
    const scenariosModel = monaco.editor.getModel(monaco.Uri.parse("file:///scenarios.ts"));

    if (scenariosModel) {
--- a/app/src/state/store.ts
+++ b/app/src/state/store.ts
@@ -1,11 +1,13 @@
 import { type StateCreator, create } from "zustand";
 import { immer } from "zustand/middleware/immer";
+import { persist } from "zustand/middleware";
 import { createSelectors } from "./createSelectors";
 import {
  type SharedVariantEditorSlice,
  createVariantEditorSlice,
 } from "./sharedVariantEditor.slice";
 import { type APIClient } from "~/utils/api";
+import { persistOptions, type stateToPersist } from "./persist";

 export type State = {
  drawerOpen: boolean;
@@ -23,30 +25,36 @@ export type SliceCreator<T> = StateCreator<State, [["zustand/immer", never]], []
 export type SetFn = Parameters<SliceCreator<unknown>>[0];
 export type GetFn = Parameters<SliceCreator<unknown>>[1];

-const useBaseStore = create<State, [["zustand/immer", never]]>(
-  immer((set, get, ...rest) => ({
-    api: null,
-    setApi: (api) =>
-      set((state) => {
-        state.api = api;
-      }),
+const useBaseStore = create<
+  State,
+  [["zustand/persist", typeof stateToPersist], ["zustand/immer", never]]
+>(
+  persist(
+    immer((set, get, ...rest) => ({
+      api: null,
+      setApi: (api) =>
+        set((state) => {
+          state.api = api;
+        }),

-    drawerOpen: false,
-    openDrawer: () =>
-      set((state) => {
-        state.drawerOpen = true;
-      }),
-    closeDrawer: () =>
-      set((state) => {
-        state.drawerOpen = false;
-      }),
-    sharedVariantEditor: createVariantEditorSlice(set, get, ...rest),
-    selectedProjectId: null,
-    setselectedProjectId: (id: string) =>
-      set((state) => {
-        state.selectedProjectId = id;
-      }),
-  })),
+      drawerOpen: false,
+      openDrawer: () =>
+        set((state) => {
+          state.drawerOpen = true;
+        }),
+      closeDrawer: () =>
+        set((state) => {
+          state.drawerOpen = false;
+        }),
+      sharedVariantEditor: createVariantEditorSlice(set, get, ...rest),
+      selectedProjectId: null,
+      setselectedProjectId: (id: string) =>
+        set((state) => {
+          state.selectedProjectId = id;
+        }),
+    })),
+    persistOptions,
+  ),
 );

 export const useAppStore = createSelectors(useBaseStore);
--- a/app/src/state/sync.tsx
+++ b/app/src/state/sync.tsx
@@ -1,16 +1,16 @@
 import { useEffect } from "react";
 import { api } from "~/utils/api";
-import { useScenarios } from "~/utils/hooks";
+import { useScenarioVars } from "~/utils/hooks";
 import { useAppStore } from "./store";

 export function useSyncVariantEditor() {
-  const scenarios = useScenarios();
+  const scenarioVars = useScenarioVars();

  useEffect(() => {
-    if (scenarios.data) {
-      useAppStore.getState().sharedVariantEditor.setScenarios(scenarios.data.scenarios);
+    if (scenarioVars.data) {
+      useAppStore.getState().sharedVariantEditor.setScenarioVars(scenarioVars.data);
    }
-  }, [scenarios.data]);
+  }, [scenarioVars.data]);
 }

 export function SyncAppStore() {
--- a/app/src/tests/helpers/loadEnv.ts
+++ b/app/src/tests/helpers/loadEnv.ts
@@ -0,0 +1,5 @@
+import { configDotenv } from "dotenv";
+
+configDotenv({
+  path: ".env.test",
+});
--- a/app/src/tests/helpers/setup.ts
+++ b/app/src/tests/helpers/setup.ts
@@ -0,0 +1,13 @@
+import "./loadEnv";
+import { sql } from "kysely";
+import { beforeEach } from "vitest";
+import { kysely } from "~/server/db";
+
+// Reset all Prisma data
+const resetDb = async () => {
+  await sql`truncate "Experiment" cascade;`.execute(kysely);
+};
+
+beforeEach(async () => {
+  await resetDb();
+});
--- a/app/src/theme/ChakraThemeProvider.tsx
+++ b/app/src/theme/ChakraThemeProvider.tsx
@@ -1,4 +1,9 @@
-import { extendTheme, defineStyleConfig, ChakraProvider } from "@chakra-ui/react";
+import {
+  extendTheme,
+  defineStyleConfig,
+  ChakraProvider,
+  createStandaloneToast,
+} from "@chakra-ui/react";
 import "@fontsource/inconsolata";
 import { modalAnatomy } from "@chakra-ui/anatomy";
 import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system";
@@ -63,6 +68,15 @@ const theme = extendTheme({
  },
 });

+const { ToastContainer, toast } = createStandaloneToast(theme);
+
+export { toast };
+
 export const ChakraThemeProvider = ({ children }: { children: JSX.Element }) => {
-  return <ChakraProvider theme={theme}>{children}</ChakraProvider>;
+  return (
+    <ChakraProvider theme={theme}>
+      <ToastContainer />
+      {children}
+    </ChakraProvider>
+  );
 };
--- a/app/src/utils/errorHandling/maybeReportError.ts
+++ b/app/src/utils/errorHandling/maybeReportError.ts
@@ -0,0 +1,20 @@
+import { toast } from "~/theme/ChakraThemeProvider";
+import { type error, type success } from "./standardResponses";
+
+type SuccessType<T> = ReturnType<typeof success<T>>;
+type ErrorType = ReturnType<typeof error>;
+
+// Used client-side to report generic errors
+export function maybeReportError<T>(response: SuccessType<T> | ErrorType): response is ErrorType {
+  if (response.status === "error") {
+    toast({
+      description: response.message,
+      status: "error",
+      duration: 5000,
+      isClosable: true,
+    });
+    return true;
+  }
+
+  return false;
+}
--- a/app/src/utils/errorHandling/standardResponses.ts
+++ b/app/src/utils/errorHandling/standardResponses.ts
@@ -0,0 +1,11 @@
+export function error(message: string): { status: "error"; message: string } {
+  return {
+    status: "error",
+    message,
+  };
+}
+export function success<T>(payload: T): { status: "success"; payload: T };
+export function success(payload?: undefined): { status: "success"; payload: undefined };
+export function success<T>(payload?: T) {
+  return { status: "success", payload };
+}
--- a/app/src/utils/hooks.ts
+++ b/app/src/utils/hooks.ts
@@ -157,3 +157,12 @@ export const useSelectedProject = () => {
    { enabled: !!selectedProjectId },
  );
 };
+
+export const useScenarioVars = () => {
+  const experiment = useExperiment();
+
+  return api.scenarioVars.list.useQuery(
+    { experimentId: experiment.data?.id ?? "" },
+    { enabled: experiment.data?.id != null },
+  );
+};
--- a/app/vitest.config.ts
+++ b/app/vitest.config.ts
@@ -4,6 +4,10 @@ import { configDefaults, defineConfig, type UserConfig } from "vitest/config";
 const config = defineConfig({
  test: {
    ...configDefaults, // Extending Vitest's default options
+    setupFiles: ["./src/tests/helpers/setup.ts"],
+
+    // Unfortunately using threads seems to cause issues with isolated-vm
+    threads: false,
  },
  plugins: [tsconfigPaths()],
 }) as UserConfig;
--- a/client-libs/python/.testmondata
+++ b/client-libs/python/.testmondata
--- a/client-libs/python/.testmondata-shm
+++ b/client-libs/python/.testmondata-shm
--- a/client-libs/python/.testmondata-wal
+++ b/client-libs/python/.testmondata-wal
--- a/client-libs/python/codegen.sh
+++ b/client-libs/python/codegen.sh
@@ -0,0 +1,11 @@
+#! /bin/bash
+
+set -e
+
+cd "$(dirname "$0")"
+
+poetry run openapi-python-client generate --url http://localhost:3000/api/openapi.json
+
+rm -rf openpipe/api_client
+mv open-pipe-api-client/open_pipe_api_client openpipe/api_client
+rm -rf open-pipe-api-client
--- a/client-libs/python/openpipe/init.py
+++ b/client-libs/python/openpipe/init.py
@@ -0,0 +1,10 @@
+from .openai import OpenAIWrapper
+from .shared import configured_client
+
+openai = OpenAIWrapper()
+
+def configure_openpipe(base_url=None, api_key=None):
+  if base_url is not None:
+    configured_client._base_url = base_url
+  if api_key is not None:
+    configured_client.token = api_key
--- a/client-libs/python/openpipe/api_client/init.py
+++ b/client-libs/python/openpipe/api_client/init.py
@@ -0,0 +1,7 @@
+""" A client library for accessing OpenPipe API """
+from .client import AuthenticatedClient, Client
+
+__all__ = (
+    "AuthenticatedClient",
+    "Client",
+)
--- a/client-libs/python/openpipe/api_client/api/init.py
+++ b/client-libs/python/openpipe/api_client/api/init.py
@@ -0,0 +1 @@
+""" Contains methods for accessing the API """
--- a/client-libs/python/openpipe/api_client/api/default/init.py
+++ b/client-libs/python/openpipe/api_client/api/default/init.py
--- a/client-libs/python/openpipe/api_client/api/default/external_api_check_cache.py
+++ b/client-libs/python/openpipe/api_client/api/default/external_api_check_cache.py
@@ -0,0 +1,155 @@
+from http import HTTPStatus
+from typing import Any, Dict, Optional, Union
+
+import httpx
+
+from ... import errors
+from ...client import AuthenticatedClient, Client
+from ...models.external_api_check_cache_json_body import ExternalApiCheckCacheJsonBody
+from ...models.external_api_check_cache_response_200 import ExternalApiCheckCacheResponse200
+from ...types import Response
+
+
+def _get_kwargs(
+    *,
+    json_body: ExternalApiCheckCacheJsonBody,
+) -> Dict[str, Any]:
+    pass
+
+    json_json_body = json_body.to_dict()
+
+    return {
+        "method": "post",
+        "url": "/v1/check-cache",
+        "json": json_json_body,
+    }
+
+
+def _parse_response(
+    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
+) -> Optional[ExternalApiCheckCacheResponse200]:
+    if response.status_code == HTTPStatus.OK:
+        response_200 = ExternalApiCheckCacheResponse200.from_dict(response.json())
+
+        return response_200
+    if client.raise_on_unexpected_status:
+        raise errors.UnexpectedStatus(response.status_code, response.content)
+    else:
+        return None
+
+
+def _build_response(
+    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
+) -> Response[ExternalApiCheckCacheResponse200]:
+    return Response(
+        status_code=HTTPStatus(response.status_code),
+        content=response.content,
+        headers=response.headers,
+        parsed=_parse_response(client=client, response=response),
+    )
+
+
+def sync_detailed(
+    *,
+    client: AuthenticatedClient,
+    json_body: ExternalApiCheckCacheJsonBody,
+) -> Response[ExternalApiCheckCacheResponse200]:
+    """Check if a prompt is cached
+
+    Args:
+        json_body (ExternalApiCheckCacheJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[ExternalApiCheckCacheResponse200]
+    """
+
+    kwargs = _get_kwargs(
+        json_body=json_body,
+    )
+
+    response = client.get_httpx_client().request(
+        **kwargs,
+    )
+
+    return _build_response(client=client, response=response)
+
+
+def sync(
+    *,
+    client: AuthenticatedClient,
+    json_body: ExternalApiCheckCacheJsonBody,
+) -> Optional[ExternalApiCheckCacheResponse200]:
+    """Check if a prompt is cached
+
+    Args:
+        json_body (ExternalApiCheckCacheJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        ExternalApiCheckCacheResponse200
+    """
+
+    return sync_detailed(
+        client=client,
+        json_body=json_body,
+    ).parsed
+
+
+async def asyncio_detailed(
+    *,
+    client: AuthenticatedClient,
+    json_body: ExternalApiCheckCacheJsonBody,
+) -> Response[ExternalApiCheckCacheResponse200]:
+    """Check if a prompt is cached
+
+    Args:
+        json_body (ExternalApiCheckCacheJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[ExternalApiCheckCacheResponse200]
+    """
+
+    kwargs = _get_kwargs(
+        json_body=json_body,
+    )
+
+    response = await client.get_async_httpx_client().request(**kwargs)
+
+    return _build_response(client=client, response=response)
+
+
+async def asyncio(
+    *,
+    client: AuthenticatedClient,
+    json_body: ExternalApiCheckCacheJsonBody,
+) -> Optional[ExternalApiCheckCacheResponse200]:
+    """Check if a prompt is cached
+
+    Args:
+        json_body (ExternalApiCheckCacheJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        ExternalApiCheckCacheResponse200
+    """
+
+    return (
+        await asyncio_detailed(
+            client=client,
+            json_body=json_body,
+        )
+    ).parsed
--- a/client-libs/python/openpipe/api_client/api/default/external_api_report.py
+++ b/client-libs/python/openpipe/api_client/api/default/external_api_report.py
@@ -0,0 +1,98 @@
+from http import HTTPStatus
+from typing import Any, Dict, Optional, Union
+
+import httpx
+
+from ... import errors
+from ...client import AuthenticatedClient, Client
+from ...models.external_api_report_json_body import ExternalApiReportJsonBody
+from ...types import Response
+
+
+def _get_kwargs(
+    *,
+    json_body: ExternalApiReportJsonBody,
+) -> Dict[str, Any]:
+    pass
+
+    json_json_body = json_body.to_dict()
+
+    return {
+        "method": "post",
+        "url": "/v1/report",
+        "json": json_json_body,
+    }
+
+
+def _parse_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Optional[Any]:
+    if response.status_code == HTTPStatus.OK:
+        return None
+    if client.raise_on_unexpected_status:
+        raise errors.UnexpectedStatus(response.status_code, response.content)
+    else:
+        return None
+
+
+def _build_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Response[Any]:
+    return Response(
+        status_code=HTTPStatus(response.status_code),
+        content=response.content,
+        headers=response.headers,
+        parsed=_parse_response(client=client, response=response),
+    )
+
+
+def sync_detailed(
+    *,
+    client: AuthenticatedClient,
+    json_body: ExternalApiReportJsonBody,
+) -> Response[Any]:
+    """Report an API call
+
+    Args:
+        json_body (ExternalApiReportJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[Any]
+    """
+
+    kwargs = _get_kwargs(
+        json_body=json_body,
+    )
+
+    response = client.get_httpx_client().request(
+        **kwargs,
+    )
+
+    return _build_response(client=client, response=response)
+
+
+async def asyncio_detailed(
+    *,
+    client: AuthenticatedClient,
+    json_body: ExternalApiReportJsonBody,
+) -> Response[Any]:
+    """Report an API call
+
+    Args:
+        json_body (ExternalApiReportJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[Any]
+    """
+
+    kwargs = _get_kwargs(
+        json_body=json_body,
+    )
+
+    response = await client.get_async_httpx_client().request(**kwargs)
+
+    return _build_response(client=client, response=response)
--- a/client-libs/python/openpipe/api_client/client.py
+++ b/client-libs/python/openpipe/api_client/client.py
@@ -0,0 +1,268 @@
+import ssl
+from typing import Any, Dict, Optional, Union
+
+import httpx
+from attrs import define, evolve, field
+
+
+@define
+class Client:
+    """A class for keeping track of data related to the API
+
+    The following are accepted as keyword arguments and will be used to construct httpx Clients internally:
+
+        ``base_url``: The base URL for the API, all requests are made to a relative path to this URL
+
+        ``cookies``: A dictionary of cookies to be sent with every request
+
+        ``headers``: A dictionary of headers to be sent with every request
+
+        ``timeout``: The maximum amount of a time a request can take. API functions will raise
+        httpx.TimeoutException if this is exceeded.
+
+        ``verify_ssl``: Whether or not to verify the SSL certificate of the API server. This should be True in production,
+        but can be set to False for testing purposes.
+
+        ``follow_redirects``: Whether or not to follow redirects. Default value is False.
+
+        ``httpx_args``: A dictionary of additional arguments to be passed to the ``httpx.Client`` and ``httpx.AsyncClient`` constructor.
+
+
+    Attributes:
+        raise_on_unexpected_status: Whether or not to raise an errors.UnexpectedStatus if the API returns a
+            status code that was not documented in the source OpenAPI document. Can also be provided as a keyword
+            argument to the constructor.
+    """
+
+    raise_on_unexpected_status: bool = field(default=False, kw_only=True)
+    _base_url: str
+    _cookies: Dict[str, str] = field(factory=dict, kw_only=True)
+    _headers: Dict[str, str] = field(factory=dict, kw_only=True)
+    _timeout: Optional[httpx.Timeout] = field(default=None, kw_only=True)
+    _verify_ssl: Union[str, bool, ssl.SSLContext] = field(default=True, kw_only=True)
+    _follow_redirects: bool = field(default=False, kw_only=True)
+    _httpx_args: Dict[str, Any] = field(factory=dict, kw_only=True)
+    _client: Optional[httpx.Client] = field(default=None, init=False)
+    _async_client: Optional[httpx.AsyncClient] = field(default=None, init=False)
+
+    def with_headers(self, headers: Dict[str, str]) -> "Client":
+        """Get a new client matching this one with additional headers"""
+        if self._client is not None:
+            self._client.headers.update(headers)
+        if self._async_client is not None:
+            self._async_client.headers.update(headers)
+        return evolve(self, headers={**self._headers, **headers})
+
+    def with_cookies(self, cookies: Dict[str, str]) -> "Client":
+        """Get a new client matching this one with additional cookies"""
+        if self._client is not None:
+            self._client.cookies.update(cookies)
+        if self._async_client is not None:
+            self._async_client.cookies.update(cookies)
+        return evolve(self, cookies={**self._cookies, **cookies})
+
+    def with_timeout(self, timeout: httpx.Timeout) -> "Client":
+        """Get a new client matching this one with a new timeout (in seconds)"""
+        if self._client is not None:
+            self._client.timeout = timeout
+        if self._async_client is not None:
+            self._async_client.timeout = timeout
+        return evolve(self, timeout=timeout)
+
+    def set_httpx_client(self, client: httpx.Client) -> "Client":
+        """Manually the underlying httpx.Client
+
+        **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout.
+        """
+        self._client = client
+        return self
+
+    def get_httpx_client(self) -> httpx.Client:
+        """Get the underlying httpx.Client, constructing a new one if not previously set"""
+        if self._client is None:
+            self._client = httpx.Client(
+                base_url=self._base_url,
+                cookies=self._cookies,
+                headers=self._headers,
+                timeout=self._timeout,
+                verify=self._verify_ssl,
+                follow_redirects=self._follow_redirects,
+                **self._httpx_args,
+            )
+        return self._client
+
+    def __enter__(self) -> "Client":
+        """Enter a context manager for self.client—you cannot enter twice (see httpx docs)"""
+        self.get_httpx_client().__enter__()
+        return self
+
+    def __exit__(self, *args: Any, **kwargs: Any) -> None:
+        """Exit a context manager for internal httpx.Client (see httpx docs)"""
+        self.get_httpx_client().__exit__(*args, **kwargs)
+
+    def set_async_httpx_client(self, async_client: httpx.AsyncClient) -> "Client":
+        """Manually the underlying httpx.AsyncClient
+
+        **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout.
+        """
+        self._async_client = async_client
+        return self
+
+    def get_async_httpx_client(self) -> httpx.AsyncClient:
+        """Get the underlying httpx.AsyncClient, constructing a new one if not previously set"""
+        if self._async_client is None:
+            self._async_client = httpx.AsyncClient(
+                base_url=self._base_url,
+                cookies=self._cookies,
+                headers=self._headers,
+                timeout=self._timeout,
+                verify=self._verify_ssl,
+                follow_redirects=self._follow_redirects,
+                **self._httpx_args,
+            )
+        return self._async_client
+
+    async def __aenter__(self) -> "Client":
+        """Enter a context manager for underlying httpx.AsyncClient—you cannot enter twice (see httpx docs)"""
+        await self.get_async_httpx_client().__aenter__()
+        return self
+
+    async def __aexit__(self, *args: Any, **kwargs: Any) -> None:
+        """Exit a context manager for underlying httpx.AsyncClient (see httpx docs)"""
+        await self.get_async_httpx_client().__aexit__(*args, **kwargs)
+
+
+@define
+class AuthenticatedClient:
+    """A Client which has been authenticated for use on secured endpoints
+
+    The following are accepted as keyword arguments and will be used to construct httpx Clients internally:
+
+        ``base_url``: The base URL for the API, all requests are made to a relative path to this URL
+
+        ``cookies``: A dictionary of cookies to be sent with every request
+
+        ``headers``: A dictionary of headers to be sent with every request
+
+        ``timeout``: The maximum amount of a time a request can take. API functions will raise
+        httpx.TimeoutException if this is exceeded.
+
+        ``verify_ssl``: Whether or not to verify the SSL certificate of the API server. This should be True in production,
+        but can be set to False for testing purposes.
+
+        ``follow_redirects``: Whether or not to follow redirects. Default value is False.
+
+        ``httpx_args``: A dictionary of additional arguments to be passed to the ``httpx.Client`` and ``httpx.AsyncClient`` constructor.
+
+
+    Attributes:
+        raise_on_unexpected_status: Whether or not to raise an errors.UnexpectedStatus if the API returns a
+            status code that was not documented in the source OpenAPI document. Can also be provided as a keyword
+            argument to the constructor.
+        token: The token to use for authentication
+        prefix: The prefix to use for the Authorization header
+        auth_header_name: The name of the Authorization header
+    """
+
+    raise_on_unexpected_status: bool = field(default=False, kw_only=True)
+    _base_url: str
+    _cookies: Dict[str, str] = field(factory=dict, kw_only=True)
+    _headers: Dict[str, str] = field(factory=dict, kw_only=True)
+    _timeout: Optional[httpx.Timeout] = field(default=None, kw_only=True)
+    _verify_ssl: Union[str, bool, ssl.SSLContext] = field(default=True, kw_only=True)
+    _follow_redirects: bool = field(default=False, kw_only=True)
+    _httpx_args: Dict[str, Any] = field(factory=dict, kw_only=True)
+    _client: Optional[httpx.Client] = field(default=None, init=False)
+    _async_client: Optional[httpx.AsyncClient] = field(default=None, init=False)
+
+    token: str
+    prefix: str = "Bearer"
+    auth_header_name: str = "Authorization"
+
+    def with_headers(self, headers: Dict[str, str]) -> "AuthenticatedClient":
+        """Get a new client matching this one with additional headers"""
+        if self._client is not None:
+            self._client.headers.update(headers)
+        if self._async_client is not None:
+            self._async_client.headers.update(headers)
+        return evolve(self, headers={**self._headers, **headers})
+
+    def with_cookies(self, cookies: Dict[str, str]) -> "AuthenticatedClient":
+        """Get a new client matching this one with additional cookies"""
+        if self._client is not None:
+            self._client.cookies.update(cookies)
+        if self._async_client is not None:
+            self._async_client.cookies.update(cookies)
+        return evolve(self, cookies={**self._cookies, **cookies})
+
+    def with_timeout(self, timeout: httpx.Timeout) -> "AuthenticatedClient":
+        """Get a new client matching this one with a new timeout (in seconds)"""
+        if self._client is not None:
+            self._client.timeout = timeout
+        if self._async_client is not None:
+            self._async_client.timeout = timeout
+        return evolve(self, timeout=timeout)
+
+    def set_httpx_client(self, client: httpx.Client) -> "AuthenticatedClient":
+        """Manually the underlying httpx.Client
+
+        **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout.
+        """
+        self._client = client
+        return self
+
+    def get_httpx_client(self) -> httpx.Client:
+        """Get the underlying httpx.Client, constructing a new one if not previously set"""
+        if self._client is None:
+            self._headers[self.auth_header_name] = f"{self.prefix} {self.token}" if self.prefix else self.token
+            self._client = httpx.Client(
+                base_url=self._base_url,
+                cookies=self._cookies,
+                headers=self._headers,
+                timeout=self._timeout,
+                verify=self._verify_ssl,
+                follow_redirects=self._follow_redirects,
+                **self._httpx_args,
+            )
+        return self._client
+
+    def __enter__(self) -> "AuthenticatedClient":
+        """Enter a context manager for self.client—you cannot enter twice (see httpx docs)"""
+        self.get_httpx_client().__enter__()
+        return self
+
+    def __exit__(self, *args: Any, **kwargs: Any) -> None:
+        """Exit a context manager for internal httpx.Client (see httpx docs)"""
+        self.get_httpx_client().__exit__(*args, **kwargs)
+
+    def set_async_httpx_client(self, async_client: httpx.AsyncClient) -> "AuthenticatedClient":
+        """Manually the underlying httpx.AsyncClient
+
+        **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout.
+        """
+        self._async_client = async_client
+        return self
+
+    def get_async_httpx_client(self) -> httpx.AsyncClient:
+        """Get the underlying httpx.AsyncClient, constructing a new one if not previously set"""
+        if self._async_client is None:
+            self._headers[self.auth_header_name] = f"{self.prefix} {self.token}" if self.prefix else self.token
+            self._async_client = httpx.AsyncClient(
+                base_url=self._base_url,
+                cookies=self._cookies,
+                headers=self._headers,
+                timeout=self._timeout,
+                verify=self._verify_ssl,
+                follow_redirects=self._follow_redirects,
+                **self._httpx_args,
+            )
+        return self._async_client
+
+    async def __aenter__(self) -> "AuthenticatedClient":
+        """Enter a context manager for underlying httpx.AsyncClient—you cannot enter twice (see httpx docs)"""
+        await self.get_async_httpx_client().__aenter__()
+        return self
+
+    async def __aexit__(self, *args: Any, **kwargs: Any) -> None:
+        """Exit a context manager for underlying httpx.AsyncClient (see httpx docs)"""
+        await self.get_async_httpx_client().__aexit__(*args, **kwargs)
--- a/client-libs/python/openpipe/api_client/errors.py
+++ b/client-libs/python/openpipe/api_client/errors.py
@@ -0,0 +1,14 @@
+""" Contains shared errors types that can be raised from API functions """
+
+
+class UnexpectedStatus(Exception):
+    """Raised by api functions when the response status an undocumented status and Client.raise_on_unexpected_status is True"""
+
+    def __init__(self, status_code: int, content: bytes):
+        self.status_code = status_code
+        self.content = content
+
+        super().__init__(f"Unexpected status code: {status_code}")
+
+
+__all__ = ["UnexpectedStatus"]
--- a/client-libs/python/openpipe/api_client/models/init.py
+++ b/client-libs/python/openpipe/api_client/models/init.py
@@ -0,0 +1,15 @@
+""" Contains all the data models used in inputs/outputs """
+
+from .external_api_check_cache_json_body import ExternalApiCheckCacheJsonBody
+from .external_api_check_cache_json_body_tags import ExternalApiCheckCacheJsonBodyTags
+from .external_api_check_cache_response_200 import ExternalApiCheckCacheResponse200
+from .external_api_report_json_body import ExternalApiReportJsonBody
+from .external_api_report_json_body_tags import ExternalApiReportJsonBodyTags
+
+__all__ = (
+    "ExternalApiCheckCacheJsonBody",
+    "ExternalApiCheckCacheJsonBodyTags",
+    "ExternalApiCheckCacheResponse200",
+    "ExternalApiReportJsonBody",
+    "ExternalApiReportJsonBodyTags",
+)
--- a/client-libs/python/openpipe/api_client/models/external_api_check_cache_json_body.py
+++ b/client-libs/python/openpipe/api_client/models/external_api_check_cache_json_body.py
@@ -0,0 +1,70 @@
+from typing import TYPE_CHECKING, Any, Dict, Type, TypeVar, Union
+
+from attrs import define
+
+from ..types import UNSET, Unset
+
+if TYPE_CHECKING:
+    from ..models.external_api_check_cache_json_body_tags import ExternalApiCheckCacheJsonBodyTags
+
+
+T = TypeVar("T", bound="ExternalApiCheckCacheJsonBody")
+
+
+@define
+class ExternalApiCheckCacheJsonBody:
+    """
+    Attributes:
+        requested_at (float): Unix timestamp in milliseconds
+        req_payload (Union[Unset, Any]): JSON-encoded request payload
+        tags (Union[Unset, ExternalApiCheckCacheJsonBodyTags]): Extra tags to attach to the call for filtering. Eg {
+            "userId": "123", "promptId": "populate-title" }
+    """
+
+    requested_at: float
+    req_payload: Union[Unset, Any] = UNSET
+    tags: Union[Unset, "ExternalApiCheckCacheJsonBodyTags"] = UNSET
+
+    def to_dict(self) -> Dict[str, Any]:
+        requested_at = self.requested_at
+        req_payload = self.req_payload
+        tags: Union[Unset, Dict[str, Any]] = UNSET
+        if not isinstance(self.tags, Unset):
+            tags = self.tags.to_dict()
+
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(
+            {
+                "requestedAt": requested_at,
+            }
+        )
+        if req_payload is not UNSET:
+            field_dict["reqPayload"] = req_payload
+        if tags is not UNSET:
+            field_dict["tags"] = tags
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        from ..models.external_api_check_cache_json_body_tags import ExternalApiCheckCacheJsonBodyTags
+
+        d = src_dict.copy()
+        requested_at = d.pop("requestedAt")
+
+        req_payload = d.pop("reqPayload", UNSET)
+
+        _tags = d.pop("tags", UNSET)
+        tags: Union[Unset, ExternalApiCheckCacheJsonBodyTags]
+        if isinstance(_tags, Unset):
+            tags = UNSET
+        else:
+            tags = ExternalApiCheckCacheJsonBodyTags.from_dict(_tags)
+
+        external_api_check_cache_json_body = cls(
+            requested_at=requested_at,
+            req_payload=req_payload,
+            tags=tags,
+        )
+
+        return external_api_check_cache_json_body
--- a/client-libs/python/openpipe/api_client/models/external_api_check_cache_json_body_tags.py
+++ b/client-libs/python/openpipe/api_client/models/external_api_check_cache_json_body_tags.py
@@ -0,0 +1,43 @@
+from typing import Any, Dict, List, Type, TypeVar
+
+from attrs import define, field
+
+T = TypeVar("T", bound="ExternalApiCheckCacheJsonBodyTags")
+
+
+@define
+class ExternalApiCheckCacheJsonBodyTags:
+    """Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }"""
+
+    additional_properties: Dict[str, str] = field(init=False, factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update({})
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        d = src_dict.copy()
+        external_api_check_cache_json_body_tags = cls()
+
+        external_api_check_cache_json_body_tags.additional_properties = d
+        return external_api_check_cache_json_body_tags
+
+    @property
+    def additional_keys(self) -> List[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> str:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: str) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties
--- a/client-libs/python/openpipe/api_client/models/external_api_check_cache_response_200.py
+++ b/client-libs/python/openpipe/api_client/models/external_api_check_cache_response_200.py
@@ -0,0 +1,38 @@
+from typing import Any, Dict, Type, TypeVar, Union
+
+from attrs import define
+
+from ..types import UNSET, Unset
+
+T = TypeVar("T", bound="ExternalApiCheckCacheResponse200")
+
+
+@define
+class ExternalApiCheckCacheResponse200:
+    """
+    Attributes:
+        resp_payload (Union[Unset, Any]): JSON-encoded response payload
+    """
+
+    resp_payload: Union[Unset, Any] = UNSET
+
+    def to_dict(self) -> Dict[str, Any]:
+        resp_payload = self.resp_payload
+
+        field_dict: Dict[str, Any] = {}
+        field_dict.update({})
+        if resp_payload is not UNSET:
+            field_dict["respPayload"] = resp_payload
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        d = src_dict.copy()
+        resp_payload = d.pop("respPayload", UNSET)
+
+        external_api_check_cache_response_200 = cls(
+            resp_payload=resp_payload,
+        )
+
+        return external_api_check_cache_response_200
--- a/client-libs/python/openpipe/api_client/models/external_api_report_json_body.py
+++ b/client-libs/python/openpipe/api_client/models/external_api_report_json_body.py
@@ -0,0 +1,101 @@
+from typing import TYPE_CHECKING, Any, Dict, Type, TypeVar, Union
+
+from attrs import define
+
+from ..types import UNSET, Unset
+
+if TYPE_CHECKING:
+    from ..models.external_api_report_json_body_tags import ExternalApiReportJsonBodyTags
+
+
+T = TypeVar("T", bound="ExternalApiReportJsonBody")
+
+
+@define
+class ExternalApiReportJsonBody:
+    """
+    Attributes:
+        requested_at (float): Unix timestamp in milliseconds
+        received_at (float): Unix timestamp in milliseconds
+        req_payload (Union[Unset, Any]): JSON-encoded request payload
+        resp_payload (Union[Unset, Any]): JSON-encoded response payload
+        status_code (Union[Unset, float]): HTTP status code of response
+        error_message (Union[Unset, str]): User-friendly error message
+        tags (Union[Unset, ExternalApiReportJsonBodyTags]): Extra tags to attach to the call for filtering. Eg {
+            "userId": "123", "promptId": "populate-title" }
+    """
+
+    requested_at: float
+    received_at: float
+    req_payload: Union[Unset, Any] = UNSET
+    resp_payload: Union[Unset, Any] = UNSET
+    status_code: Union[Unset, float] = UNSET
+    error_message: Union[Unset, str] = UNSET
+    tags: Union[Unset, "ExternalApiReportJsonBodyTags"] = UNSET
+
+    def to_dict(self) -> Dict[str, Any]:
+        requested_at = self.requested_at
+        received_at = self.received_at
+        req_payload = self.req_payload
+        resp_payload = self.resp_payload
+        status_code = self.status_code
+        error_message = self.error_message
+        tags: Union[Unset, Dict[str, Any]] = UNSET
+        if not isinstance(self.tags, Unset):
+            tags = self.tags.to_dict()
+
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(
+            {
+                "requestedAt": requested_at,
+                "receivedAt": received_at,
+            }
+        )
+        if req_payload is not UNSET:
+            field_dict["reqPayload"] = req_payload
+        if resp_payload is not UNSET:
+            field_dict["respPayload"] = resp_payload
+        if status_code is not UNSET:
+            field_dict["statusCode"] = status_code
+        if error_message is not UNSET:
+            field_dict["errorMessage"] = error_message
+        if tags is not UNSET:
+            field_dict["tags"] = tags
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        from ..models.external_api_report_json_body_tags import ExternalApiReportJsonBodyTags
+
+        d = src_dict.copy()
+        requested_at = d.pop("requestedAt")
+
+        received_at = d.pop("receivedAt")
+
+        req_payload = d.pop("reqPayload", UNSET)
+
+        resp_payload = d.pop("respPayload", UNSET)
+
+        status_code = d.pop("statusCode", UNSET)
+
+        error_message = d.pop("errorMessage", UNSET)
+
+        _tags = d.pop("tags", UNSET)
+        tags: Union[Unset, ExternalApiReportJsonBodyTags]
+        if isinstance(_tags, Unset):
+            tags = UNSET
+        else:
+            tags = ExternalApiReportJsonBodyTags.from_dict(_tags)
+
+        external_api_report_json_body = cls(
+            requested_at=requested_at,
+            received_at=received_at,
+            req_payload=req_payload,
+            resp_payload=resp_payload,
+            status_code=status_code,
+            error_message=error_message,
+            tags=tags,
+        )
+
+        return external_api_report_json_body
--- a/client-libs/python/openpipe/api_client/models/external_api_report_json_body_tags.py
+++ b/client-libs/python/openpipe/api_client/models/external_api_report_json_body_tags.py
@@ -0,0 +1,43 @@
+from typing import Any, Dict, List, Type, TypeVar
+
+from attrs import define, field
+
+T = TypeVar("T", bound="ExternalApiReportJsonBodyTags")
+
+
+@define
+class ExternalApiReportJsonBodyTags:
+    """Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }"""
+
+    additional_properties: Dict[str, str] = field(init=False, factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update({})
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        d = src_dict.copy()
+        external_api_report_json_body_tags = cls()
+
+        external_api_report_json_body_tags.additional_properties = d
+        return external_api_report_json_body_tags
+
+    @property
+    def additional_keys(self) -> List[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> str:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: str) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties
--- a/client-libs/python/openpipe/api_client/py.typed
+++ b/client-libs/python/openpipe/api_client/py.typed
@@ -0,0 +1 @@
+# Marker file for PEP 561
--- a/client-libs/python/openpipe/api_client/types.py
+++ b/client-libs/python/openpipe/api_client/types.py
@@ -0,0 +1,44 @@
+""" Contains some shared types for properties """
+from http import HTTPStatus
+from typing import BinaryIO, Generic, Literal, MutableMapping, Optional, Tuple, TypeVar
+
+from attrs import define
+
+
+class Unset:
+    def __bool__(self) -> Literal[False]:
+        return False
+
+
+UNSET: Unset = Unset()
+
+FileJsonType = Tuple[Optional[str], BinaryIO, Optional[str]]
+
+
+@define
+class File:
+    """Contains information for file uploads"""
+
+    payload: BinaryIO
+    file_name: Optional[str] = None
+    mime_type: Optional[str] = None
+
+    def to_tuple(self) -> FileJsonType:
+        """Return a tuple representation that httpx will accept for multipart/form-data"""
+        return self.file_name, self.payload, self.mime_type
+
+
+T = TypeVar("T")
+
+
+@define
+class Response(Generic[T]):
+    """A response from an endpoint"""
+
+    status_code: HTTPStatus
+    content: bytes
+    headers: MutableMapping[str, str]
+    parsed: Optional[T]
+
+
+__all__ = ["File", "Response", "FileJsonType"]
--- a/client-libs/python/openpipe/merge_openai_chunks.py
+++ b/client-libs/python/openpipe/merge_openai_chunks.py
@@ -0,0 +1,42 @@
+from typing import Any, Optional
+
+
+def merge_streamed_chunks(base: Optional[Any], chunk: Any) -> Any:
+    if base is None:
+        return merge_streamed_chunks({**chunk, "choices": []}, chunk)
+
+    choices = base["choices"].copy()
+    for choice in chunk["choices"]:
+        base_choice = next((c for c in choices if c["index"] == choice["index"]), None)
+
+        if base_choice:
+            base_choice["finish_reason"] = (
+                choice.get("finish_reason") or base_choice["finish_reason"]
+            )
+            base_choice["message"] = base_choice.get("message") or {"role": "assistant"}
+
+            if choice.get("delta") and choice["delta"].get("content"):
+                base_choice["message"]["content"] = (
+                    base_choice["message"].get("content") or ""
+                ) + (choice["delta"].get("content") or "")
+            if choice.get("delta") and choice["delta"].get("function_call"):
+                fn_call = base_choice["message"].get("function_call") or {}
+                fn_call["name"] = (fn_call.get("name") or "") + (
+                    choice["delta"]["function_call"].get("name") or ""
+                )
+                fn_call["arguments"] = (fn_call.get("arguments") or "") + (
+                    choice["delta"]["function_call"].get("arguments") or ""
+                )
+        else:
+            # Here, we'll have to handle the omitted property "delta" manually
+            new_choice = {k: v for k, v in choice.items() if k != "delta"}
+            choices.append(
+                {**new_choice, "message": {"role": "assistant", **choice["delta"]}}
+            )
+
+    merged = {
+        **base,
+        "choices": choices,
+    }
+
+    return merged
--- a/client-libs/python/openpipe/openai.py
+++ b/client-libs/python/openpipe/openai.py
@@ -0,0 +1,168 @@
+import openai as original_openai
+from openai.openai_object import OpenAIObject
+import time
+import inspect
+
+from openpipe.merge_openai_chunks import merge_streamed_chunks
+
+from .shared import maybe_check_cache, maybe_check_cache_async, report_async, report
+
+
+class WrappedChatCompletion(original_openai.ChatCompletion):
+    @classmethod
+    def create(cls, *args, **kwargs):
+        openpipe_options = kwargs.pop("openpipe", {})
+
+        cached_response = maybe_check_cache(
+            openpipe_options=openpipe_options, req_payload=kwargs
+        )
+        if cached_response:
+            return OpenAIObject.construct_from(cached_response, api_key=None)
+
+        requested_at = int(time.time() * 1000)
+
+        try:
+            chat_completion = original_openai.ChatCompletion.create(*args, **kwargs)
+
+            if inspect.isgenerator(chat_completion):
+
+                def _gen():
+                    assembled_completion = None
+                    for chunk in chat_completion:
+                        assembled_completion = merge_streamed_chunks(
+                            assembled_completion, chunk
+                        )
+                        yield chunk
+
+                    received_at = int(time.time() * 1000)
+
+                    report(
+                        openpipe_options=openpipe_options,
+                        requested_at=requested_at,
+                        received_at=received_at,
+                        req_payload=kwargs,
+                        resp_payload=assembled_completion,
+                        status_code=200,
+                    )
+
+                return _gen()
+            else:
+                received_at = int(time.time() * 1000)
+
+                report(
+                    openpipe_options=openpipe_options,
+                    requested_at=requested_at,
+                    received_at=received_at,
+                    req_payload=kwargs,
+                    resp_payload=chat_completion,
+                    status_code=200,
+                )
+
+            return chat_completion
+        except Exception as e:
+            received_at = int(time.time() * 1000)
+
+            if isinstance(e, original_openai.OpenAIError):
+                report(
+                    openpipe_options=openpipe_options,
+                    requested_at=requested_at,
+                    received_at=received_at,
+                    req_payload=kwargs,
+                    resp_payload=e.json_body,
+                    error_message=str(e),
+                    status_code=e.http_status,
+                )
+            else:
+                report(
+                    openpipe_options=openpipe_options,
+                    requested_at=requested_at,
+                    received_at=received_at,
+                    req_payload=kwargs,
+                    error_message=str(e),
+                )
+
+            raise e
+
+    @classmethod
+    async def acreate(cls, *args, **kwargs):
+        openpipe_options = kwargs.pop("openpipe", {})
+
+        cached_response = await maybe_check_cache_async(
+            openpipe_options=openpipe_options, req_payload=kwargs
+        )
+        if cached_response:
+            return OpenAIObject.construct_from(cached_response, api_key=None)
+
+        requested_at = int(time.time() * 1000)
+
+        try:
+            chat_completion = original_openai.ChatCompletion.acreate(*args, **kwargs)
+
+            if inspect.isgenerator(chat_completion):
+
+                def _gen():
+                    assembled_completion = None
+                    for chunk in chat_completion:
+                        assembled_completion = merge_streamed_chunks(
+                            assembled_completion, chunk
+                        )
+                        yield chunk
+
+                    received_at = int(time.time() * 1000)
+
+                    report_async(
+                        openpipe_options=openpipe_options,
+                        requested_at=requested_at,
+                        received_at=received_at,
+                        req_payload=kwargs,
+                        resp_payload=assembled_completion,
+                        status_code=200,
+                    )
+
+                return _gen()
+            else:
+                received_at = int(time.time() * 1000)
+
+                report_async(
+                    openpipe_options=openpipe_options,
+                    requested_at=requested_at,
+                    received_at=received_at,
+                    req_payload=kwargs,
+                    resp_payload=chat_completion,
+                    status_code=200,
+                )
+
+            return chat_completion
+        except Exception as e:
+            received_at = int(time.time() * 1000)
+
+            if isinstance(e, original_openai.OpenAIError):
+                report_async(
+                    openpipe_options=openpipe_options,
+                    requested_at=requested_at,
+                    received_at=received_at,
+                    req_payload=kwargs,
+                    resp_payload=e.json_body,
+                    error_message=str(e),
+                    status_code=e.http_status,
+                )
+            else:
+                report_async(
+                    openpipe_options=openpipe_options,
+                    requested_at=requested_at,
+                    received_at=received_at,
+                    req_payload=kwargs,
+                    error_message=str(e),
+                )
+
+            raise e
+
+
+class OpenAIWrapper:
+    ChatCompletion = WrappedChatCompletion()
+
+    def __getattr__(self, name):
+        return getattr(original_openai, name)
+
+    def __setattr__(self, name, value):
+        return setattr(original_openai, name, value)
--- a/client-libs/python/openpipe/shared.py
+++ b/client-libs/python/openpipe/shared.py
@@ -0,0 +1,125 @@
+from openpipe.api_client.api.default import (
+    external_api_report,
+    external_api_check_cache,
+)
+from openpipe.api_client.client import AuthenticatedClient
+from openpipe.api_client.models.external_api_report_json_body_tags import (
+    ExternalApiReportJsonBodyTags,
+)
+import toml
+import time
+
+version = toml.load("pyproject.toml")["tool"]["poetry"]["version"]
+
+configured_client = AuthenticatedClient(
+    base_url="https://app.openpipe.ai/api/v1", token=""
+)
+
+
+def _get_tags(openpipe_options):
+    tags = openpipe_options.get("tags") or {}
+    tags["$sdk"] = "python"
+    tags["$sdk_version"] = version
+
+    return ExternalApiReportJsonBodyTags.from_dict(tags)
+
+
+def _should_check_cache(openpipe_options):
+    if configured_client.token == "":
+        return False
+    return openpipe_options.get("cache", False)
+
+
+def _process_cache_payload(
+    payload: external_api_check_cache.ExternalApiCheckCacheResponse200,
+):
+    if not payload or not payload.resp_payload:
+        return None
+    payload.resp_payload["openpipe"] = {"cache_status": "HIT"}
+
+    return payload.resp_payload
+
+
+def maybe_check_cache(
+    openpipe_options={},
+    req_payload={},
+):
+    if not _should_check_cache(openpipe_options):
+        return None
+    try:
+        payload = external_api_check_cache.sync(
+            client=configured_client,
+            json_body=external_api_check_cache.ExternalApiCheckCacheJsonBody(
+                req_payload=req_payload,
+                requested_at=int(time.time() * 1000),
+                tags=_get_tags(openpipe_options),
+            ),
+        )
+        return _process_cache_payload(payload)
+
+    except Exception as e:
+        # We don't want to break client apps if our API is down for some reason
+        print(f"Error reporting to OpenPipe: {e}")
+        print(e)
+        return None
+
+
+async def maybe_check_cache_async(
+    openpipe_options={},
+    req_payload={},
+):
+    if not _should_check_cache(openpipe_options):
+        return None
+
+    try:
+        payload = await external_api_check_cache.asyncio(
+            client=configured_client,
+            json_body=external_api_check_cache.ExternalApiCheckCacheJsonBody(
+                req_payload=req_payload,
+                requested_at=int(time.time() * 1000),
+                tags=_get_tags(openpipe_options),
+            ),
+        )
+        return _process_cache_payload(payload)
+
+    except Exception as e:
+        # We don't want to break client apps if our API is down for some reason
+        print(f"Error reporting to OpenPipe: {e}")
+        print(e)
+        return None
+
+
+def report(
+    openpipe_options={},
+    **kwargs,
+):
+    try:
+        external_api_report.sync_detailed(
+            client=configured_client,
+            json_body=external_api_report.ExternalApiReportJsonBody(
+                **kwargs,
+                tags=_get_tags(openpipe_options),
+            ),
+        )
+    except Exception as e:
+        # We don't want to break client apps if our API is down for some reason
+        print(f"Error reporting to OpenPipe: {e}")
+        print(e)
+
+
+async def report_async(
+    openpipe_options={},
+    **kwargs,
+):
+    try:
+        await external_api_report.asyncio_detailed(
+            client=configured_client,
+            json_body=external_api_report.ExternalApiReportJsonBody(
+                **kwargs,
+                tags=_get_tags(openpipe_options),
+            ),
+        )
+    except Exception as e:
+        # We don't want to break client apps if our API is down for some reason
+        print(f"Error reporting to OpenPipe: {e}")
+        print(e)
--- a/client-libs/python/openpipe/test_client.py
+++ b/client-libs/python/openpipe/test_client.py
@@ -0,0 +1,88 @@
+from dotenv import load_dotenv
+from . import openai, configure_openpipe
+import os
+import pytest
+
+load_dotenv()
+
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+configure_openpipe(
+    base_url="http://localhost:3000/api", api_key=os.getenv("OPENPIPE_API_KEY")
+)
+
+
+def test_sync():
+    completion = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "system", "content": "count to 10"}],
+    )
+
+    print(completion.choices[0].message.content)
+
+
+def test_streaming():
+    completion = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "system", "content": "count to 10"}],
+        stream=True,
+    )
+
+    for chunk in completion:
+        print(chunk)
+
+
+async def test_async():
+    acompletion = await openai.ChatCompletion.acreate(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "count down from 5"}],
+    )
+
+    print(acompletion.choices[0].message.content)
+
+
+async def test_async_streaming():
+    acompletion = await openai.ChatCompletion.acreate(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "count down from 5"}],
+        stream=True,
+    )
+
+    async for chunk in acompletion:
+        print(chunk)
+
+
+def test_sync_with_tags():
+    completion = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "system", "content": "count to 10"}],
+        openpipe={"tags": {"promptId": "testprompt"}},
+    )
+    print("finished")
+
+    print(completion.choices[0].message.content)
+
+
+def test_bad_call():
+    completion = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo-blaster",
+        messages=[{"role": "system", "content": "count to 10"}],
+        stream=True,
+    )
+
+
+@pytest.mark.focus
+async def test_caching():
+    completion = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "system", "content": "count to 10"}],
+        openpipe={"cache": True},
+    )
+
+    completion2 = await openai.ChatCompletion.acreate(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "system", "content": "count to 10"}],
+        openpipe={"cache": True},
+    )
+
+    print(completion2)
--- a/client-libs/python/poetry.lock
+++ b/client-libs/python/poetry.lock
--- a/client-libs/python/pyproject.toml
+++ b/client-libs/python/pyproject.toml
@@ -0,0 +1,35 @@
+[tool.poetry]
+name = "openpipe"
+version = "0.1.0"
+description = ""
+authors = ["Kyle Corbitt <kyle@corbt.com>"]
+license = "Apache-2.0"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+openai = "^0.27.8"
+httpx = "^0.24.1"
+attrs = "^23.1.0"
+python-dateutil = "^2.8.2"
+toml = "^0.10.2"
+
+[tool.poetry.dev-dependencies]
+
+[tool.poetry.group.dev.dependencies]
+openapi-python-client = "^0.15.0"
+black = "^23.7.0"
+isort = "^5.12.0"
+autoflake = "^2.2.0"
+pytest = "^7.4.0"
+python-dotenv = "^1.0.0"
+pytest-asyncio = "^0.21.1"
+pytest-watch = "^4.2.0"
+pytest-testmon = "^2.0.12"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+markers = "focus"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
--- a/client-libs/schema.json
+++ b/client-libs/schema.json
@@ -5,16 +5,13 @@
    "description": "The public API for reporting API calls to OpenPipe",
    "version": "0.1.0"
  },
-  "servers": [
-    {
-      "url": "https://app.openpipe.ai/api"
-    }
-  ],
+  "servers": [{ "url": "https://app.openpipe.ai/api" }],
  "paths": {
    "/v1/check-cache": {
      "post": {
        "operationId": "externalApi-checkCache",
        "description": "Check if a prompt is cached",
+        "security": [{ "Authorization": [] }],
        "requestBody": {
          "required": true,
          "content": {
@@ -26,20 +23,14 @@
                    "type": "number",
                    "description": "Unix timestamp in milliseconds"
                  },
-                  "reqPayload": {
-                    "description": "JSON-encoded request payload"
-                  },
+                  "reqPayload": { "description": "JSON-encoded request payload" },
                  "tags": {
                    "type": "object",
-                    "additionalProperties": {
-                      "type": "string"
-                    },
+                    "additionalProperties": { "type": "string" },
                    "description": "Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }"
                  }
                },
-                "required": [
-                  "startTime"
-                ],
+                "required": ["startTime"],
                "additionalProperties": false
              }
            }
@@ -54,18 +45,14 @@
                "schema": {
                  "type": "object",
                  "properties": {
-                    "respPayload": {
-                      "description": "JSON-encoded response payload"
-                    }
+                    "respPayload": { "description": "JSON-encoded response payload" }
                  },
                  "additionalProperties": false
                }
              }
            }
          },
-          "default": {
-            "$ref": "#/components/responses/error"
-          }
+          "default": { "$ref": "#/components/responses/error" }
        }
      }
    },
@@ -73,6 +60,7 @@
      "post": {
        "operationId": "externalApi-report",
        "description": "Report an API call",
+        "security": [{ "Authorization": [] }],
        "requestBody": {
          "required": true,
          "content": {
@@ -84,36 +72,18 @@
                    "type": "number",
                    "description": "Unix timestamp in milliseconds"
                  },
-                  "endTime": {
-                    "type": "number",
-                    "description": "Unix timestamp in milliseconds"
-                  },
-                  "reqPayload": {
-                    "description": "JSON-encoded request payload"
-                  },
-                  "respPayload": {
-                    "description": "JSON-encoded response payload"
-                  },
-                  "respStatus": {
-                    "type": "number",
-                    "description": "HTTP status code of response"
-                  },
-                  "error": {
-                    "type": "string",
-                    "description": "User-friendly error message"
-                  },
+                  "endTime": { "type": "number", "description": "Unix timestamp in milliseconds" },
+                  "reqPayload": { "description": "JSON-encoded request payload" },
+                  "respPayload": { "description": "JSON-encoded response payload" },
+                  "respStatus": { "type": "number", "description": "HTTP status code of response" },
+                  "error": { "type": "string", "description": "User-friendly error message" },
                  "tags": {
                    "type": "object",
-                    "additionalProperties": {
-                      "type": "string"
-                    },
+                    "additionalProperties": { "type": "string" },
                    "description": "Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }"
                  }
                },
-                "required": [
-                  "startTime",
-                  "endTime"
-                ],
+                "required": ["startTime", "endTime"],
                "additionalProperties": false
              }
            }
@@ -123,26 +93,15 @@
        "responses": {
          "200": {
            "description": "Successful response",
-            "content": {
-              "application/json": {
-                "schema": {}
-              }
-            }
+            "content": { "application/json": { "schema": {} } }
          },
-          "default": {
-            "$ref": "#/components/responses/error"
-          }
+          "default": { "$ref": "#/components/responses/error" }
        }
      }
    }
  },
  "components": {
-    "securitySchemes": {
-      "Authorization": {
-        "type": "http",
-        "scheme": "bearer"
-      }
-    },
+    "securitySchemes": { "Authorization": { "type": "http", "scheme": "bearer" } },
    "responses": {
      "error": {
        "description": "Error response",
@@ -151,32 +110,19 @@
            "schema": {
              "type": "object",
              "properties": {
-                "message": {
-                  "type": "string"
-                },
-                "code": {
-                  "type": "string"
-                },
+                "message": { "type": "string" },
+                "code": { "type": "string" },
                "issues": {
                  "type": "array",
                  "items": {
                    "type": "object",
-                    "properties": {
-                      "message": {
-                        "type": "string"
-                      }
-                    },
-                    "required": [
-                      "message"
-                    ],
+                    "properties": { "message": { "type": "string" } },
+                    "required": ["message"],
                    "additionalProperties": false
                  }
                }
              },
-              "required": [
-                "message",
-                "code"
-              ],
+              "required": ["message", "code"],
              "additionalProperties": false
            }
          }
--- a/client-libs/typescript/codegen/api.js
+++ b/client-libs/typescript/codegen/api.js
@@ -1,272 +0,0 @@
-"use strict";
-/* tslint:disable */
-/* eslint-disable */
-/**
- * OpenPipe API
- * The public API for reporting API calls to OpenPipe
- *
- * The version of the OpenAPI document: 0.1.0
- *
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
-var __extends = (this && this.__extends) || (function () {
-    var extendStatics = function (d, b) {
-        extendStatics = Object.setPrototypeOf ||
-            ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
-            function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
-        return extendStatics(d, b);
-    };
-    return function (d, b) {
-        if (typeof b !== "function" && b !== null)
-            throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
-        extendStatics(d, b);
-        function __() { this.constructor = d; }
-        d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
-    };
-})();
-var __assign = (this && this.__assign) || function () {
-    __assign = Object.assign || function(t) {
-        for (var s, i = 1, n = arguments.length; i < n; i++) {
-            s = arguments[i];
-            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
-                t[p] = s[p];
-        }
-        return t;
-    };
-    return __assign.apply(this, arguments);
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DefaultApi = exports.DefaultApiFactory = exports.DefaultApiFp = exports.DefaultApiAxiosParamCreator = void 0;
-var axios_1 = require("axios");
-// Some imports not used depending on template conditions
-// @ts-ignore
-var common_1 = require("./common");
-// @ts-ignore
-var base_1 = require("./base");
-/**
- * DefaultApi - axios parameter creator
- * @export
- */
-var DefaultApiAxiosParamCreator = function (configuration) {
-    var _this = this;
-    return {
-        /**
-         * Check if a prompt is cached
-         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        externalApiCheckCache: function (externalApiCheckCacheRequest, options) {
-            if (options === void 0) { options = {}; }
-            return __awaiter(_this, void 0, void 0, function () {
-                var localVarPath, localVarUrlObj, baseOptions, localVarRequestOptions, localVarHeaderParameter, localVarQueryParameter, headersFromBaseOptions;
-                return __generator(this, function (_a) {
-                    // verify required parameter 'externalApiCheckCacheRequest' is not null or undefined
-                    (0, common_1.assertParamExists)('externalApiCheckCache', 'externalApiCheckCacheRequest', externalApiCheckCacheRequest);
-                    localVarPath = "/v1/check-cache";
-                    localVarUrlObj = new URL(localVarPath, common_1.DUMMY_BASE_URL);
-                    if (configuration) {
-                        baseOptions = configuration.baseOptions;
-                    }
-                    localVarRequestOptions = __assign(__assign({ method: 'POST' }, baseOptions), options);
-                    localVarHeaderParameter = {};
-                    localVarQueryParameter = {};
-                    localVarHeaderParameter['Content-Type'] = 'application/json';
-                    (0, common_1.setSearchParams)(localVarUrlObj, localVarQueryParameter);
-                    headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
-                    localVarRequestOptions.headers = __assign(__assign(__assign({}, localVarHeaderParameter), headersFromBaseOptions), options.headers);
-                    localVarRequestOptions.data = (0, common_1.serializeDataIfNeeded)(externalApiCheckCacheRequest, localVarRequestOptions, configuration);
-                    return [2 /*return*/, {
-                            url: (0, common_1.toPathString)(localVarUrlObj),
-                            options: localVarRequestOptions,
-                        }];
-                });
-            });
-        },
-        /**
-         * Report an API call
-         * @param {ExternalApiReportRequest} externalApiReportRequest
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        externalApiReport: function (externalApiReportRequest, options) {
-            if (options === void 0) { options = {}; }
-            return __awaiter(_this, void 0, void 0, function () {
-                var localVarPath, localVarUrlObj, baseOptions, localVarRequestOptions, localVarHeaderParameter, localVarQueryParameter, headersFromBaseOptions;
-                return __generator(this, function (_a) {
-                    // verify required parameter 'externalApiReportRequest' is not null or undefined
-                    (0, common_1.assertParamExists)('externalApiReport', 'externalApiReportRequest', externalApiReportRequest);
-                    localVarPath = "/v1/report";
-                    localVarUrlObj = new URL(localVarPath, common_1.DUMMY_BASE_URL);
-                    if (configuration) {
-                        baseOptions = configuration.baseOptions;
-                    }
-                    localVarRequestOptions = __assign(__assign({ method: 'POST' }, baseOptions), options);
-                    localVarHeaderParameter = {};
-                    localVarQueryParameter = {};
-                    localVarHeaderParameter['Content-Type'] = 'application/json';
-                    (0, common_1.setSearchParams)(localVarUrlObj, localVarQueryParameter);
-                    headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
-                    localVarRequestOptions.headers = __assign(__assign(__assign({}, localVarHeaderParameter), headersFromBaseOptions), options.headers);
-                    localVarRequestOptions.data = (0, common_1.serializeDataIfNeeded)(externalApiReportRequest, localVarRequestOptions, configuration);
-                    return [2 /*return*/, {
-                            url: (0, common_1.toPathString)(localVarUrlObj),
-                            options: localVarRequestOptions,
-                        }];
-                });
-            });
-        },
-    };
-};
-exports.DefaultApiAxiosParamCreator = DefaultApiAxiosParamCreator;
-/**
- * DefaultApi - functional programming interface
- * @export
- */
-var DefaultApiFp = function (configuration) {
-    var localVarAxiosParamCreator = (0, exports.DefaultApiAxiosParamCreator)(configuration);
-    return {
-        /**
-         * Check if a prompt is cached
-         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        externalApiCheckCache: function (externalApiCheckCacheRequest, options) {
-            return __awaiter(this, void 0, void 0, function () {
-                var localVarAxiosArgs;
-                return __generator(this, function (_a) {
-                    switch (_a.label) {
-                        case 0: return [4 /*yield*/, localVarAxiosParamCreator.externalApiCheckCache(externalApiCheckCacheRequest, options)];
-                        case 1:
-                            localVarAxiosArgs = _a.sent();
-                            return [2 /*return*/, (0, common_1.createRequestFunction)(localVarAxiosArgs, axios_1.default, base_1.BASE_PATH, configuration)];
-                    }
-                });
-            });
-        },
-        /**
-         * Report an API call
-         * @param {ExternalApiReportRequest} externalApiReportRequest
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        externalApiReport: function (externalApiReportRequest, options) {
-            return __awaiter(this, void 0, void 0, function () {
-                var localVarAxiosArgs;
-                return __generator(this, function (_a) {
-                    switch (_a.label) {
-                        case 0: return [4 /*yield*/, localVarAxiosParamCreator.externalApiReport(externalApiReportRequest, options)];
-                        case 1:
-                            localVarAxiosArgs = _a.sent();
-                            return [2 /*return*/, (0, common_1.createRequestFunction)(localVarAxiosArgs, axios_1.default, base_1.BASE_PATH, configuration)];
-                    }
-                });
-            });
-        },
-    };
-};
-exports.DefaultApiFp = DefaultApiFp;
-/**
- * DefaultApi - factory interface
- * @export
- */
-var DefaultApiFactory = function (configuration, basePath, axios) {
-    var localVarFp = (0, exports.DefaultApiFp)(configuration);
-    return {
-        /**
-         * Check if a prompt is cached
-         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        externalApiCheckCache: function (externalApiCheckCacheRequest, options) {
-            return localVarFp.externalApiCheckCache(externalApiCheckCacheRequest, options).then(function (request) { return request(axios, basePath); });
-        },
-        /**
-         * Report an API call
-         * @param {ExternalApiReportRequest} externalApiReportRequest
-         * @param {*} [options] Override http request option.
-         * @throws {RequiredError}
-         */
-        externalApiReport: function (externalApiReportRequest, options) {
-            return localVarFp.externalApiReport(externalApiReportRequest, options).then(function (request) { return request(axios, basePath); });
-        },
-    };
-};
-exports.DefaultApiFactory = DefaultApiFactory;
-/**
- * DefaultApi - object-oriented interface
- * @export
- * @class DefaultApi
- * @extends {BaseAPI}
- */
-var DefaultApi = /** @class */ (function (_super) {
-    __extends(DefaultApi, _super);
-    function DefaultApi() {
-        return _super !== null && _super.apply(this, arguments) || this;
-    }
-    /**
-     * Check if a prompt is cached
-     * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest
-     * @param {*} [options] Override http request option.
-     * @throws {RequiredError}
-     * @memberof DefaultApi
-     */
-    DefaultApi.prototype.externalApiCheckCache = function (externalApiCheckCacheRequest, options) {
-        var _this = this;
-        return (0, exports.DefaultApiFp)(this.configuration).externalApiCheckCache(externalApiCheckCacheRequest, options).then(function (request) { return request(_this.axios, _this.basePath); });
-    };
-    /**
-     * Report an API call
-     * @param {ExternalApiReportRequest} externalApiReportRequest
-     * @param {*} [options] Override http request option.
-     * @throws {RequiredError}
-     * @memberof DefaultApi
-     */
-    DefaultApi.prototype.externalApiReport = function (externalApiReportRequest, options) {
-        var _this = this;
-        return (0, exports.DefaultApiFp)(this.configuration).externalApiReport(externalApiReportRequest, options).then(function (request) { return request(_this.axios, _this.basePath); });
-    };
-    return DefaultApi;
-}(base_1.BaseAPI));
-exports.DefaultApi = DefaultApi;
--- a/client-libs/typescript/codegen/api.ts
+++ b/client-libs/typescript/codegen/api.ts
@@ -85,7 +85,7 @@ export interface ExternalApiCheckCacheRequest {
     * @type {number}
     * @memberof ExternalApiCheckCacheRequest
     */
-    'startTime': number;
+    'requestedAt': number;
    /**
     * JSON-encoded request payload
     * @type {any}
@@ -110,13 +110,13 @@ export interface ExternalApiReportRequest {
     * @type {number}
     * @memberof ExternalApiReportRequest
     */
-    'startTime': number;
+    'requestedAt': number;
    /**
     * Unix timestamp in milliseconds
     * @type {number}
     * @memberof ExternalApiReportRequest
     */
-    'endTime': number;
+    'receivedAt': number;
    /**
     * JSON-encoded request payload
     * @type {any}
@@ -134,13 +134,13 @@ export interface ExternalApiReportRequest {
     * @type {number}
     * @memberof ExternalApiReportRequest
     */
-    'respStatus'?: number;
+    'statusCode'?: number;
    /**
     * User-friendly error message
     * @type {string}
     * @memberof ExternalApiReportRequest
     */
-    'error'?: string;
+    'errorMessage'?: string;
    /**
     * Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }
     * @type {{ [key: string]: string; }}
--- a/client-libs/typescript/codegen/base.js
+++ b/client-libs/typescript/codegen/base.js
@@ -1,80 +0,0 @@
-"use strict";
-/* tslint:disable */
-/* eslint-disable */
-/**
- * OpenPipe API
- * The public API for reporting API calls to OpenPipe
- *
- * The version of the OpenAPI document: 0.1.0
- *
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
-var __extends = (this && this.__extends) || (function () {
-    var extendStatics = function (d, b) {
-        extendStatics = Object.setPrototypeOf ||
-            ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
-            function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
-        return extendStatics(d, b);
-    };
-    return function (d, b) {
-        if (typeof b !== "function" && b !== null)
-            throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
-        extendStatics(d, b);
-        function __() { this.constructor = d; }
-        d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RequiredError = exports.BaseAPI = exports.COLLECTION_FORMATS = exports.BASE_PATH = void 0;
-var axios_1 = require("axios");
-exports.BASE_PATH = "https://app.openpipe.ai/api".replace(/\/+$/, "");
-/**
- *
- * @export
- */
-exports.COLLECTION_FORMATS = {
-    csv: ",",
-    ssv: " ",
-    tsv: "\t",
-    pipes: "|",
-};
-/**
- *
- * @export
- * @class BaseAPI
- */
-var BaseAPI = /** @class */ (function () {
-    function BaseAPI(configuration, basePath, axios) {
-        if (basePath === void 0) { basePath = exports.BASE_PATH; }
-        if (axios === void 0) { axios = axios_1.default; }
-        this.basePath = basePath;
-        this.axios = axios;
-        if (configuration) {
-            this.configuration = configuration;
-            this.basePath = configuration.basePath || this.basePath;
-        }
-    }
-    return BaseAPI;
-}());
-exports.BaseAPI = BaseAPI;
-;
-/**
- *
- * @export
- * @class RequiredError
- * @extends {Error}
- */
-var RequiredError = /** @class */ (function (_super) {
-    __extends(RequiredError, _super);
-    function RequiredError(field, msg) {
-        var _this = _super.call(this, msg) || this;
-        _this.field = field;
-        _this.name = "RequiredError";
-        return _this;
-    }
-    return RequiredError;
-}(Error));
-exports.RequiredError = RequiredError;
--- a/client-libs/typescript/codegen/common.js
+++ b/client-libs/typescript/codegen/common.js
@@ -1,252 +0,0 @@
-"use strict";
-/* tslint:disable */
-/* eslint-disable */
-/**
- * OpenPipe API
- * The public API for reporting API calls to OpenPipe
- *
- * The version of the OpenAPI document: 0.1.0
- *
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
-var __assign = (this && this.__assign) || function () {
-    __assign = Object.assign || function(t) {
-        for (var s, i = 1, n = arguments.length; i < n; i++) {
-            s = arguments[i];
-            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
-                t[p] = s[p];
-        }
-        return t;
-    };
-    return __assign.apply(this, arguments);
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createRequestFunction = exports.toPathString = exports.serializeDataIfNeeded = exports.setSearchParams = exports.setOAuthToObject = exports.setBearerAuthToObject = exports.setBasicAuthToObject = exports.setApiKeyToObject = exports.assertParamExists = exports.DUMMY_BASE_URL = void 0;
-var base_1 = require("./base");
-/**
- *
- * @export
- */
-exports.DUMMY_BASE_URL = 'https://example.com';
-/**
- *
- * @throws {RequiredError}
- * @export
- */
-var assertParamExists = function (functionName, paramName, paramValue) {
-    if (paramValue === null || paramValue === undefined) {
-        throw new base_1.RequiredError(paramName, "Required parameter ".concat(paramName, " was null or undefined when calling ").concat(functionName, "."));
-    }
-};
-exports.assertParamExists = assertParamExists;
-/**
- *
- * @export
- */
-var setApiKeyToObject = function (object, keyParamName, configuration) {
-    return __awaiter(this, void 0, void 0, function () {
-        var localVarApiKeyValue, _a;
-        return __generator(this, function (_b) {
-            switch (_b.label) {
-                case 0:
-                    if (!(configuration && configuration.apiKey)) return [3 /*break*/, 5];
-                    if (!(typeof configuration.apiKey === 'function')) return [3 /*break*/, 2];
-                    return [4 /*yield*/, configuration.apiKey(keyParamName)];
-                case 1:
-                    _a = _b.sent();
-                    return [3 /*break*/, 4];
-                case 2: return [4 /*yield*/, configuration.apiKey];
-                case 3:
-                    _a = _b.sent();
-                    _b.label = 4;
-                case 4:
-                    localVarApiKeyValue = _a;
-                    object[keyParamName] = localVarApiKeyValue;
-                    _b.label = 5;
-                case 5: return [2 /*return*/];
-            }
-        });
-    });
-};
-exports.setApiKeyToObject = setApiKeyToObject;
-/**
- *
- * @export
- */
-var setBasicAuthToObject = function (object, configuration) {
-    if (configuration && (configuration.username || configuration.password)) {
-        object["auth"] = { username: configuration.username, password: configuration.password };
-    }
-};
-exports.setBasicAuthToObject = setBasicAuthToObject;
-/**
- *
- * @export
- */
-var setBearerAuthToObject = function (object, configuration) {
-    return __awaiter(this, void 0, void 0, function () {
-        var accessToken, _a;
-        return __generator(this, function (_b) {
-            switch (_b.label) {
-                case 0:
-                    if (!(configuration && configuration.accessToken)) return [3 /*break*/, 5];
-                    if (!(typeof configuration.accessToken === 'function')) return [3 /*break*/, 2];
-                    return [4 /*yield*/, configuration.accessToken()];
-                case 1:
-                    _a = _b.sent();
-                    return [3 /*break*/, 4];
-                case 2: return [4 /*yield*/, configuration.accessToken];
-                case 3:
-                    _a = _b.sent();
-                    _b.label = 4;
-                case 4:
-                    accessToken = _a;
-                    object["Authorization"] = "Bearer " + accessToken;
-                    _b.label = 5;
-                case 5: return [2 /*return*/];
-            }
-        });
-    });
-};
-exports.setBearerAuthToObject = setBearerAuthToObject;
-/**
- *
- * @export
- */
-var setOAuthToObject = function (object, name, scopes, configuration) {
-    return __awaiter(this, void 0, void 0, function () {
-        var localVarAccessTokenValue, _a;
-        return __generator(this, function (_b) {
-            switch (_b.label) {
-                case 0:
-                    if (!(configuration && configuration.accessToken)) return [3 /*break*/, 5];
-                    if (!(typeof configuration.accessToken === 'function')) return [3 /*break*/, 2];
-                    return [4 /*yield*/, configuration.accessToken(name, scopes)];
-                case 1:
-                    _a = _b.sent();
-                    return [3 /*break*/, 4];
-                case 2: return [4 /*yield*/, configuration.accessToken];
-                case 3:
-                    _a = _b.sent();
-                    _b.label = 4;
-                case 4:
-                    localVarAccessTokenValue = _a;
-                    object["Authorization"] = "Bearer " + localVarAccessTokenValue;
-                    _b.label = 5;
-                case 5: return [2 /*return*/];
-            }
-        });
-    });
-};
-exports.setOAuthToObject = setOAuthToObject;
-function setFlattenedQueryParams(urlSearchParams, parameter, key) {
-    if (key === void 0) { key = ""; }
-    if (parameter == null)
-        return;
-    if (typeof parameter === "object") {
-        if (Array.isArray(parameter)) {
-            parameter.forEach(function (item) { return setFlattenedQueryParams(urlSearchParams, item, key); });
-        }
-        else {
-            Object.keys(parameter).forEach(function (currentKey) {
-                return setFlattenedQueryParams(urlSearchParams, parameter[currentKey], "".concat(key).concat(key !== '' ? '.' : '').concat(currentKey));
-            });
-        }
-    }
-    else {
-        if (urlSearchParams.has(key)) {
-            urlSearchParams.append(key, parameter);
-        }
-        else {
-            urlSearchParams.set(key, parameter);
-        }
-    }
-}
-/**
- *
- * @export
- */
-var setSearchParams = function (url) {
-    var objects = [];
-    for (var _i = 1; _i < arguments.length; _i++) {
-        objects[_i - 1] = arguments[_i];
-    }
-    var searchParams = new URLSearchParams(url.search);
-    setFlattenedQueryParams(searchParams, objects);
-    url.search = searchParams.toString();
-};
-exports.setSearchParams = setSearchParams;
-/**
- *
- * @export
- */
-var serializeDataIfNeeded = function (value, requestOptions, configuration) {
-    var nonString = typeof value !== 'string';
-    var needsSerialization = nonString && configuration && configuration.isJsonMime
-        ? configuration.isJsonMime(requestOptions.headers['Content-Type'])
-        : nonString;
-    return needsSerialization
-        ? JSON.stringify(value !== undefined ? value : {})
-        : (value || "");
-};
-exports.serializeDataIfNeeded = serializeDataIfNeeded;
-/**
- *
- * @export
- */
-var toPathString = function (url) {
-    return url.pathname + url.search + url.hash;
-};
-exports.toPathString = toPathString;
-/**
- *
- * @export
- */
-var createRequestFunction = function (axiosArgs, globalAxios, BASE_PATH, configuration) {
-    return function (axios, basePath) {
-        if (axios === void 0) { axios = globalAxios; }
-        if (basePath === void 0) { basePath = BASE_PATH; }
-        var axiosRequestArgs = __assign(__assign({}, axiosArgs.options), { url: ((configuration === null || configuration === void 0 ? void 0 : configuration.basePath) || basePath) + axiosArgs.url });
-        return axios.request(axiosRequestArgs);
-    };
-};
-exports.createRequestFunction = createRequestFunction;
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Kyle Corbitt	d7cff0f52e	Add caching in Python Still need it in JS	2023-08-11 19:02:35 -07:00
Kyle Corbitt	8ed47eb4dd	Add a python client library We still don't have any documentation and things are in flux, but you can report your OpenAI API calls to OpenPipe now.	2023-08-11 16:54:50 -07:00
arcticfly	d9db6d80ea	Update external types (#137 ) * Separate server and frontend error logic * Update types in external api	2023-08-11 15:02:14 -07:00
arcticfly	8d1ee62ff1	Record model and cost when reporting logs (#136 ) * Rename prompt and completion tokens to input and output tokens * Add getUsage function * Record model and cost when reporting log * Remove unused imports * Move UsageGraph to its own component * Standardize model response fields * Fix types	2023-08-11 13:56:47 -07:00
arcticfly	f270579283	Auto-resize project menu width (#135 )	2023-08-10 22:50:39 -07:00
arcticfly	81fbaeae44	Style project settings on mobile (#134 ) * Style project settings on mobile * Use auto-resize text area for display name * Remove unused import	2023-08-10 22:15:45 -07:00
arcticfly	5277afa199	Change logo (#133 ) * Change logo * Add more vertical padding on desktop * Fix prettier	2023-08-10 21:44:33 -07:00
arcticfly	76c34d64e6	Change menu styles (#132 ) * Change ProjectMenu placement * Reduce UserMenu width	2023-08-10 18:48:23 -07:00
Kyle Corbitt	454ac9a0d3	Merge pull request #131 from OpenPipe/better-template-vars Better scenario variable editing	2023-08-10 12:25:54 -07:00
Kyle Corbitt	5ed7adadf9	Better scenario variable editing Some users have gotten confused by the scenario variable editing interface. This change makes the interface easier to understand.	2023-08-10 12:08:17 -07:00
Kyle Corbitt	b8e0f392ab	Merge pull request #130 from OpenPipe/output-wrapping Preserve linebreaks in model output	2023-08-10 07:26:55 -07:00
Kyle Corbitt	b2af83341d	Preserve linebreaks in model output	2023-08-09 21:58:41 -07:00
Kyle Corbitt	e6d229d5f9	Merge pull request #129 from OpenPipe/persist-proj persist the currently-selected project	2023-08-09 17:05:17 -07:00
Kyle Corbitt	1a6ae3aef7	Merge pull request #128 from OpenPipe/proj-styling Sidebar styling	2023-08-09 17:05:02 -07:00
Kyle Corbitt	9051d80775	Sidebar styling Unify the menu styles between the UserMenu and ProjectMenu	2023-08-09 16:47:09 -07:00
Kyle Corbitt	6c060c6ea0	persist the currently-selected project	2023-08-09 16:45:54 -07:00
Kyle Corbitt	f70e73e338	Merge pull request #126 from OpenPipe/org-to-proj Rename Organization to Project	2023-08-09 16:04:36 -07:00
				`@@ -0,0 +1 @@`
				`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`
				`@@ -0,0 +1 @@`
				`""" Contains methods for accessing the API """`