Add support for CommonJS in sdk (#222)

* trying rollup

* Imports mostly working

* Remove yalc.lock

* Use tsup to build

* Disable code splitting

* Update cjs exports

* Use openpipe from npm registry in app

* Remove npmignore, fix lint

* Copy the README to dist

* Update openpipe version

* Remove second openai entrypoint

* Update openpipe version

---------

Co-authored-by: Kyle Corbitt <kyle@corbt.com>

.github/ISSUE_TEMPLATE/sweep-fast-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Fast Issue
+title: 'Sweep (fast): '
+description: For few-line fixes to be handled by Sweep, an AI-powered junior developer. Sweep will use GPT-3.5 to quickly create a PR for very small changes.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

.github/ISSUE_TEMPLATE/sweep-slow-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Slow Issue
+title: 'Sweep (slow): '
+description: For larger bugs, features, refactors, and tests to be handled by Sweep, an AI-powered junior developer. Sweep will perform a deeper search and more self-reviews but will take longer.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

.github/ISSUE_TEMPLATE/sweep-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Issue
+title: 'Sweep: '
+description: For small bugs, features, refactors, and tests to be handled by Sweep, an AI-powered junior developer.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

.gitignore

@@ -3,3 +3,4 @@
 *.pyc
 node_modules/
 *.tsbuildinfo
+dist/

README.md

@@ -1,16 +1,53 @@
-<!-- <img src="https://github.com/openpipe/openpipe/assets/41524992/ca59596e-eb80-40f9-921f-6d67f6e6d8fa" width="72px" /> -->
+ <p align="center">
+  <a href="https://openpipe.ai">
+    <img height="70" src="https://github.com/openpipe/openpipe/assets/41524992/70af25fb-1f90-42d9-8a20-3606e3b5aaba" alt="logo">
+  </a>
+</p>
+<h1 align="center">
+  OpenPipe
+</h1>
 
-# OpenPipe
+<p align="center">
+  <i>Turn expensive prompts into cheap fine-tuned models.</i>
+</p>
 
-OpenPipe is a flexible playground for comparing and optimizing LLM prompts. It lets you quickly generate, test and compare candidate prompts, and can automatically [translate](#-translate-between-model-apis) those prompts between models.
+<p align="center">
+  <a href="/LICENSE"><img alt="License Apache-2.0" src="https://img.shields.io/github/license/openpipe/openpipe?style=flat-square"></a>
+  <a href='http://makeapullrequest.com'><img alt='PRs Welcome' src='https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square'/></a>
+  <a href="https://github.com/openpipe/openpipe/graphs/commit-activity"><img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/openpipe/openpipe?style=flat-square"/></a>
+  <a href="https://github.com/openpipe/openpipe/issues"><img alt="GitHub closed issues" src="https://img.shields.io/github/issues-closed/openpipe/openpipe?style=flat-square"/></a>
+ <img src="https://img.shields.io/badge/Y%20Combinator-S23-orange?style=flat-square" alt="Y Combinator S23">
+</p>
 
-<img src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="demo">
+<p align="center">
+  <a href="https://app.openpipe.ai/">Hosted App</a> - <a href="#running-locally">Running Locally</a> - <a href="#sample-experiments">Experiments</a>
+</p>
+
+<br>
+Use powerful but expensive LLMs to fine-tune smaller and cheaper models suited to your exact needs. Evaluate model and prompt combinations in the playground. Query your past requests and export optimized training data. Try it out at https://app.openpipe.ai or <a href="#running-locally">run it locally</a>.
+<br>
+
+
+## Features
+
+ * <b>Experiment</b>
+   * Bulk-test wide-reaching scenarios using code templating.
+   * Seamlessly translate prompts across different model APIs.
+   * Tap into autogenerated scenarios for fresh test perspectives.
+
+ * <b>Fine-Tune (Beta)</b>
+   * Easy integration with OpenPipe's SDK in both Python and JS.
+   * Swiftly query logs using intuitive built-in filters.
+   * Export data in multiple training formats, including Alpaca and ChatGPT, with deduplication.
+   
+<img src="https://github.com/openpipe/openpipe/assets/41524992/eaa8b92d-4536-4f63-bbef-4b0b1a60f6b5" alt="fine-tune demo">
+
+<!-- <img height="400px" src="https://github.com/openpipe/openpipe/assets/41524992/66bb1843-cb72-4130-a369-eec2df3b8201" alt="playground demo"> -->
 
-You can use our hosted version of OpenPipe at https://openpipe.ai. You can also clone this repository and [run it locally](#running-locally).
 
 ## Sample Experiments
 
-These are simple experiments users have created that show how OpenPipe works. Feel free to fork them and start experimenting yourself.
+These are sample experiments users have created that show how OpenPipe works. Feel free to fork them and start experimenting yourself.
 
 - [Twitter Sentiment Analysis](https://app.openpipe.ai/experiments/62c20a73-2012-4a64-973c-4b665ad46a57)
 - [Reddit User Needs](https://app.openpipe.ai/experiments/22222222-2222-2222-2222-222222222222)
@@ -19,43 +56,25 @@ These are simple experiments users have created that show how OpenPipe works. Fe
 
 ## Supported Models
 
-- All models available through the OpenAI [chat completion API](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
-- Llama2 [7b chat](https://replicate.com/a16z-infra/llama7b-v2-chat), [13b chat](https://replicate.com/a16z-infra/llama13b-v2-chat), [70b chat](https://replicate.com/replicate/llama70b-v2-chat).
-- Anthropic's [Claude 1 Instant](https://www.anthropic.com/index/introducing-claude) and [Claude 2](https://www.anthropic.com/index/claude-2)
-
-## Features
-
-### 🔍 Visualize Responses
-
-Inspect prompt completions side-by-side.
-
-### 🧪 Bulk-Test
-
-OpenPipe lets you _template_ a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broad coverage of your problem space.
-
-### 📟 Translate between Model APIs
-
-Write your prompt in one format and automatically convert it to work with any other model.
-
-<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/1e19ccf2-96b6-4e93-a3a5-1449710d1b5b" alt="translate between models">
-
-<br><br>
-
-### 🛠️ Refine Your Prompts Automatically
-
-Use a growing database of best-practice refinements to improve your prompts automatically.
-
-<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/87a27fe7-daef-445c-a5e2-1c82b23f9f99" alt="add function call">
-
-<br><br>
-
-### 🪄 Auto-generate Test Scenarios
-
-OpenPipe includes a tool to generate new test scenarios based on your existing prompts and scenarios. Just click "Autogenerate Scenario" to try it out!
-
-<img width="600" src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="auto-generate">
-
-<br><br>
+#### OpenAI
+  - [GPT 3.5 Turbo](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
+  - [GPT 3.5 Turbo 16k](https://platform.openai.com/docs/guides/gpt/chat-completions-api)
+  - [GPT 4](https://openai.com/gpt-4)
+#### Llama2
+  - [7b chat](https://replicate.com/a16z-infra/llama7b-v2-chat)
+  - [13b chat](https://replicate.com/a16z-infra/llama13b-v2-chat)
+  - [70b chat](https://replicate.com/replicate/llama70b-v2-chat)
+#### Llama2 Fine-Tunes
+  - [Open-Orca/OpenOrcaxOpenChat-Preview2-13B](https://huggingface.co/Open-Orca/OpenOrcaxOpenChat-Preview2-13B)
+  - [Open-Orca/OpenOrca-Platypus2-13B](https://huggingface.co/Open-Orca/OpenOrca-Platypus2-13B)
+  - [NousResearch/Nous-Hermes-Llama2-13b](https://huggingface.co/NousResearch/Nous-Hermes-Llama2-13b)
+  - [jondurbin/airoboros-l2-13b-gpt4-2.0](https://huggingface.co/jondurbin/airoboros-l2-13b-gpt4-2.0)
+  - [lmsys/vicuna-13b-v1.5](https://huggingface.co/lmsys/vicuna-13b-v1.5)
+  - [Gryphe/MythoMax-L2-13b](https://huggingface.co/Gryphe/MythoMax-L2-13b)
+  - [NousResearch/Nous-Hermes-llama-2-7b](https://huggingface.co/NousResearch/Nous-Hermes-llama-2-7b)
+#### Anthropic
+  - [Claude 1 Instant](https://www.anthropic.com/index/introducing-claude)
+  - [Claude 2](https://www.anthropic.com/index/claude-2)
 
 ## Running Locally
 
@@ -75,4 +94,4 @@ OpenPipe includes a tool to generate new test scenarios based on your existing p
 1. Copy your `.env` file to `.env.test`.
 2. Update the `DATABASE_URL` to have a different database name than your development one
 3. Run `DATABASE_URL=[your new datatase url] pnpm prisma migrate dev --skip-seed --skip-generate`
-4. Run `pnpm test`
+4. Run `pnpm test`

app/.env.example

@@ -40,3 +40,8 @@ SMTP_HOST="placeholder"
 SMTP_PORT="placeholder"
 SMTP_LOGIN="placeholder"
 SMTP_PASSWORD="placeholder"
+
+# Azure credentials are necessary for uploading large training data files
+AZURE_STORAGE_ACCOUNT_NAME="placeholder"
+AZURE_STORAGE_ACCOUNT_KEY="placeholder"
+AZURE_STORAGE_CONTAINER_NAME="placeholder"

app/.gitignore

@@ -47,3 +47,7 @@ yarn-error.log*
 
 # custom openai intialization
 src/server/utils/openaiCustomConfig.json
+
+# yalc
+.yalc
+yalc.lock

app/@types/nextjs-routes.d.ts

@@ -19,10 +19,11 @@ declare module "nextjs-routes" {
     | DynamicRoute<"/api/v1/[...trpc]", { "trpc": string[] }>
     | StaticRoute<"/api/v1/openapi">
     | StaticRoute<"/dashboard">
-    | DynamicRoute<"/data/[id]", { "id": string }>
-    | StaticRoute<"/data">
+    | DynamicRoute<"/datasets/[id]", { "id": string }>
+    | StaticRoute<"/datasets">
     | DynamicRoute<"/experiments/[experimentSlug]", { "experimentSlug": string }>
     | StaticRoute<"/experiments">
+    | StaticRoute<"/fine-tunes">
     | StaticRoute<"/">
     | DynamicRoute<"/invitations/[invitationToken]", { "invitationToken": string }>
     | StaticRoute<"/project/settings">

app/Dockerfile

@@ -23,7 +23,6 @@ ARG NEXT_PUBLIC_SOCKET_URL
 ARG NEXT_PUBLIC_HOST
 ARG NEXT_PUBLIC_SENTRY_DSN
 ARG SENTRY_AUTH_TOKEN
-ARG NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS
 
 WORKDIR /code
 COPY --from=deps /code/node_modules ./node_modules

app/package.json

@@ -26,6 +26,8 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.5.8",
     "@apidevtools/json-schema-ref-parser": "^10.1.0",
+    "@azure/identity": "^3.3.0",
+    "@azure/storage-blob": "12.15.0",
     "@babel/standalone": "^7.22.9",
     "@chakra-ui/anatomy": "^2.2.0",
     "@chakra-ui/next-js": "^2.1.4",
@@ -48,6 +50,7 @@
     "@trpc/react-query": "^10.26.0",
     "@trpc/server": "^10.26.0",
     "@vercel/og": "^0.5.9",
+    "archiver": "^6.0.0",
     "ast-types": "^0.14.2",
     "chroma-js": "^2.4.2",
     "concurrently": "^8.2.0",
@@ -60,6 +63,7 @@
     "framer-motion": "^10.12.17",
     "gpt-tokens": "^1.0.10",
     "graphile-worker": "^0.13.0",
+    "human-id": "^4.0.0",
     "immer": "^10.0.2",
     "isolated-vm": "^4.5.0",
     "json-schema-to-typescript": "^13.0.2",
@@ -67,6 +71,7 @@
     "jsonschema": "^1.4.1",
     "kysely": "^0.26.1",
     "kysely-codegen": "^0.10.1",
+    "llama-tokenizer-js": "^1.1.3",
     "lodash-es": "^4.17.21",
     "lucide-react": "^0.265.0",
     "marked": "^7.0.3",
@@ -77,7 +82,8 @@
     "nextjs-routes": "^2.0.1",
     "nodemailer": "^6.9.4",
     "openai": "4.0.0-beta.7",
-    "openpipe": "workspace:*",
+    "openpipe": "0.4.0-beta.1",
+    "openpipe-dev": "workspace:^",
     "pg": "^8.11.2",
     "pluralize": "^8.0.0",
     "posthog-js": "^1.75.3",
@@ -98,6 +104,7 @@
     "replicate": "^0.12.3",
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
+    "stream-buffers": "^3.0.2",
     "superjson": "1.12.2",
     "trpc-openapi": "^1.2.0",
     "tsx": "^3.12.7",
@@ -110,6 +117,7 @@
   },
   "devDependencies": {
     "@openapi-contrib/openapi-schema-to-json-schema": "^4.0.5",
+    "@types/archiver": "^5.3.2",
     "@types/babel__core": "^7.20.1",
     "@types/babel__standalone": "^7.1.4",
     "@types/chroma-js": "^2.4.0",
@@ -126,6 +134,7 @@
     "@types/react": "^18.2.6",
     "@types/react-dom": "^18.2.4",
     "@types/react-syntax-highlighter": "^15.5.7",
+    "@types/stream-buffers": "^3.0.4",
     "@types/uuid": "^9.0.2",
     "@typescript-eslint/eslint-plugin": "^5.59.6",
     "@typescript-eslint/parser": "^5.59.6",

app/prisma/deleteOneFineTune.ts

@@ -0,0 +1,12 @@
+import { prisma } from "~/server/db";
+
+// delete most recent fineTune
+const mostRecentFineTune = await prisma.fineTune.findFirst({
+  orderBy: { createdAt: "desc" },
+});
+
+if (mostRecentFineTune) {
+  await prisma.fineTune.delete({
+    where: { id: mostRecentFineTune.id },
+  });
+}

app/prisma/migrations/20230822234224_add_fine_tunes/migration.sql

@@ -0,0 +1,48 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `input` on the `DatasetEntry` table. All the data in the column will be lost.
+  - You are about to drop the column `output` on the `DatasetEntry` table. All the data in the column will be lost.
+  - Added the required column `loggedCallId` to the `DatasetEntry` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "DatasetEntry" DROP COLUMN "input",
+DROP COLUMN "output",
+ADD COLUMN     "loggedCallId" UUID NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "DatasetEntry" ADD CONSTRAINT "DatasetEntry_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AlterTable
+ALTER TABLE "LoggedCallModelResponse" ALTER COLUMN "cost" SET DATA TYPE DOUBLE PRECISION;
+
+-- CreateEnum
+CREATE TYPE "FineTuneStatus" AS ENUM ('PENDING', 'TRAINING', 'AWAITING_DEPLOYMENT', 'DEPLOYING', 'DEPLOYED', 'ERROR');
+
+-- CreateTable
+CREATE TABLE "FineTune" (
+    "id" UUID NOT NULL,
+    "slug" TEXT NOT NULL,
+    "baseModel" TEXT NOT NULL,
+    "status" "FineTuneStatus" NOT NULL DEFAULT 'PENDING',
+    "trainingStartedAt" TIMESTAMP(3),
+    "trainingFinishedAt" TIMESTAMP(3),
+    "deploymentStartedAt" TIMESTAMP(3),
+    "deploymentFinishedAt" TIMESTAMP(3),
+    "datasetId" UUID NOT NULL,
+    "projectId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "FineTune_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "FineTune_slug_key" ON "FineTune"("slug");
+
+-- AddForeignKey
+ALTER TABLE "FineTune" ADD CONSTRAINT "FineTune_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "FineTune" ADD CONSTRAINT "FineTune_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/migrations/20230905233758_revamp_dataset_entry/migration.sql

@@ -0,0 +1,26 @@
+/*
+  Warnings:
+
+  - Added the required column `inputTokens` to the `DatasetEntry` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `outputTokens` to the `DatasetEntry` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `type` to the `DatasetEntry` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- CreateEnum
+CREATE TYPE "DatasetEntryType" AS ENUM ('TRAIN', 'TEST');
+
+-- AlterTable
+ALTER TABLE "Dataset" ADD COLUMN     "trainingRatio" DOUBLE PRECISION NOT NULL DEFAULT 0.8;
+
+-- AlterTable
+ALTER TABLE "DatasetEntry" ADD COLUMN     "input" JSONB NOT NULL DEFAULT '[]',
+ADD COLUMN     "inputTokens" INTEGER NOT NULL DEFAULT 0,
+ADD COLUMN     "output" JSONB,
+ADD COLUMN     "outputTokens" INTEGER NOT NULL DEFAULT 0,
+ADD COLUMN     "type" "DatasetEntryType" NOT NULL DEFAULT 'TRAIN';
+
+-- CreateIndex
+CREATE INDEX "DatasetEntry_datasetId_createdAt_id_idx" ON "DatasetEntry"("datasetId", "createdAt", "id");
+
+-- CreateIndex
+CREATE INDEX "DatasetEntry_datasetId_type_idx" ON "DatasetEntry"("datasetId", "type");

app/prisma/migrations/20230906052922_make_dataset_entry_logged_call_optional/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "DatasetEntry" ALTER COLUMN "loggedCallId" DROP NOT NULL,
+ALTER COLUMN "inputTokens" DROP DEFAULT,
+ALTER COLUMN "outputTokens" DROP DEFAULT,
+ALTER COLUMN "type" DROP DEFAULT;

app/prisma/migrations/20230907120707_add_dataset_file_upload/migration.sql

@@ -0,0 +1,23 @@
+-- CreateEnum
+CREATE TYPE "DatasetFileUploadStatus" AS ENUM ('PENDING', 'DOWNLOADING', 'PROCESSING', 'SAVING', 'COMPLETE', 'ERROR');
+
+-- CreateTable
+CREATE TABLE "DatasetFileUpload" (
+    "id" UUID NOT NULL,
+    "datasetId" UUID NOT NULL,
+    "blobName" TEXT NOT NULL,
+    "fileName" TEXT NOT NULL,
+    "fileSize" INTEGER NOT NULL,
+    "progress" INTEGER NOT NULL DEFAULT 0,
+    "status" "DatasetFileUploadStatus" NOT NULL DEFAULT 'PENDING',
+    "uploadedAt" TIMESTAMP(3) NOT NULL,
+    "visible" BOOLEAN NOT NULL DEFAULT true,
+    "errorMessage" TEXT,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "DatasetFileUpload_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "DatasetFileUpload" ADD CONSTRAINT "DatasetFileUpload_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/schema.prisma

@@ -176,11 +176,42 @@ model OutputEvaluation {
     @@unique([modelResponseId, evaluationId])
 }
 
+
+enum DatasetFileUploadStatus {
+    PENDING
+    DOWNLOADING
+    PROCESSING
+    SAVING
+    COMPLETE
+    ERROR
+}
+
+model DatasetFileUpload {
+    id String @id @default(uuid()) @db.Uuid
+
+    datasetId String   @db.Uuid
+    dataset   Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    blobName  String
+    fileName String
+    fileSize Int
+    progress Int @default(0) // Percentage
+    status   DatasetFileUploadStatus @default(PENDING)
+    uploadedAt DateTime
+    visible  Boolean @default(true)
+    errorMessage String?
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
 model Dataset {
     id String @id @default(uuid()) @db.Uuid
 
-    name           String
-    datasetEntries DatasetEntry[]
+    name                String
+    datasetEntries      DatasetEntry[]
+    fineTunes           FineTune[]
+    datasetFileUploads  DatasetFileUpload[]
+    trainingRatio  Float @default(0.8)
 
     projectId String  @db.Uuid
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -189,17 +220,32 @@ model Dataset {
     updatedAt DateTime @updatedAt
 }
 
+enum DatasetEntryType {
+    TRAIN
+    TEST
+}
+
 model DatasetEntry {
     id String @id @default(uuid()) @db.Uuid
 
-    input  String
-    output String?
+    loggedCallId String? @db.Uuid
+    loggedCall   LoggedCall? @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
+
+    input        Json  @default("[]")
+    output       Json?
+    inputTokens  Int
+    outputTokens Int
+
+    type DatasetEntryType
 
     datasetId String   @db.Uuid
     dataset   Dataset? @relation(fields: [datasetId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
+
+    @@index([datasetId, createdAt, id])
+    @@index([datasetId, type])
 }
 
 model Project {
@@ -216,6 +262,7 @@ model Project {
     experiments            Experiment[]
     datasets               Dataset[]
     loggedCalls            LoggedCall[]
+    fineTunes              FineTune[]
     apiKeys                ApiKey[]
 }
 
@@ -276,8 +323,9 @@ model LoggedCall {
     projectId String   @db.Uuid
     project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
-    model String?
-    tags  LoggedCallTag[]
+    model           String?
+    tags            LoggedCallTag[]
+    datasetEntries  DatasetEntry[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -312,7 +360,7 @@ model LoggedCallModelResponse {
     outputTokens Int?
     finishReason String?
     completionId String?
-    cost         Decimal? @db.Decimal(18, 12)
+    cost         Float?
 
     // The LoggedCall that created this LoggedCallModelResponse
     originalLoggedCallId String     @unique @db.Uuid
@@ -427,3 +475,33 @@ model VerificationToken {
 
     @@unique([identifier, token])
 }
+
+enum FineTuneStatus {
+    PENDING
+    TRAINING
+    AWAITING_DEPLOYMENT
+    DEPLOYING
+    DEPLOYED
+    ERROR
+}
+
+model FineTune {
+    id String @id @default(uuid()) @db.Uuid
+
+    slug      String @unique
+    baseModel String
+    status    FineTuneStatus @default(PENDING)
+    trainingStartedAt    DateTime?
+    trainingFinishedAt   DateTime?
+    deploymentStartedAt  DateTime?
+    deploymentFinishedAt DateTime?
+
+    datasetId String   @db.Uuid
+    dataset    Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+
+    projectId String  @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}

app/prisma/seedAgiEval.ts

@@ -1,5 +1,4 @@
 import { prisma } from "~/server/db";
-import { generateNewCell } from "~/server/utils/generateNewCell";
 import dedent from "dedent";
 import { execSync } from "child_process";
 import fs from "fs";

app/prisma/seedDashboard.ts

@@ -80,7 +80,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     },
     respStatus: 200,
     respPayload: {
-      id: "chatcmpl-7lNspqePJWVyXwXebupxb1eMozo6Q",
+      id: "chatcmpl-7",
       model: "gpt-3.5-turbo-0613",
       usage: {
         total_tokens: 241,
@@ -108,7 +108,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 236,
     outputTokens: 5,
     finishReason: "stop",
-    tags: [],
+    tags: [{ name: "prompt_id", value: "add_scenario" }],
   },
   {
     reqPayload: {
@@ -167,7 +167,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     },
     respStatus: 200,
     respPayload: {
-      id: "chatcmpl-7lNifmc5AncyAvleZRDBhAcLFYBIT",
+      id: "chatcmpl-7",
       model: "gpt-3.5-turbo-0613",
       usage: {
         total_tokens: 227,
@@ -210,7 +210,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     },
     respStatus: 200,
     respPayload: {
-      id: "chatcmpl-7lNh1TtrsJVgz3Nj70bKkZZk7xPi7",
+      id: "chatcmpl-7",
       model: "gpt-3.5-turbo-0613",
       usage: {
         total_tokens: 21,
@@ -234,7 +234,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 14,
     outputTokens: 7,
     finishReason: "stop",
-    tags: [{ name: "prompt_id", value: "id2" }],
+    tags: [{ name: "prompt_id", value: "translate_text" }],
   },
   {
     reqPayload: {
@@ -281,7 +281,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     },
     respStatus: 200,
     respPayload: {
-      id: "chatcmpl-7lQS3MktOT8BTgNEytl9dkyssCQqL",
+      id: "chatcmpl-7",
       model: "gpt-4-0613",
       usage: {
         total_tokens: 2910,
@@ -311,7 +311,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     outputTokens: 108,
     finishReason: "stop",
     tags: [
-      { name: "prompt_id", value: "chatcmpl-7lQS3MktOT8BTgNEytl9dkyssCQqL" },
+      { name: "prompt_id", value: "define_func" },
       { name: "some_other_tag", value: "some_other_value" },
     ],
   },
@@ -339,7 +339,7 @@ const loggedCallsToCreate: Prisma.LoggedCallCreateManyInput[] = [];
 const loggedCallModelResponsesToCreate: Prisma.LoggedCallModelResponseCreateManyInput[] = [];
 const loggedCallsToUpdate: Prisma.LoggedCallUpdateArgs[] = [];
 const loggedCallTagsToCreate: Prisma.LoggedCallTagCreateManyInput[] = [];
-for (let i = 0; i < 1437; i++) {
+for (let i = 0; i < 11437; i++) {
   const loggedCallId = uuidv4();
   const loggedCallModelResponseId = uuidv4();
   const template =

app/sentry.server.config.ts

@@ -3,6 +3,7 @@
 // https://docs.sentry.io/platforms/javascript/guides/nextjs/
 
 import * as Sentry from "@sentry/nextjs";
+import { isError } from "lodash-es";
 import { env } from "~/env.mjs";
 
 if (env.NEXT_PUBLIC_SENTRY_DSN) {
@@ -15,4 +16,10 @@ if (env.NEXT_PUBLIC_SENTRY_DSN) {
     // Setting this option to true will print useful information to the console while you're setting up Sentry.
     debug: false,
   });
+} else {
+  // Install local debug exception handler for rejected promises
+  process.on("unhandledRejection", (reason) => {
+    const reasonDetails = isError(reason) ? reason?.stack : reason;
+    console.log("Unhandled Rejection at:", reasonDetails);
+  });
 }

app/src/components/ActionButton.tsx

@@ -0,0 +1,55 @@
+import { useState } from "react";
+
+import { Button, HStack, type ButtonProps, Icon, Text } from "@chakra-ui/react";
+import { type IconType } from "react-icons";
+import { useAppStore } from "~/state/store";
+import { BetaModal } from "./BetaModal";
+
+const ActionButton = ({
+  icon,
+  iconBoxSize = 3.5,
+  label,
+  requireBeta = false,
+  onClick,
+  ...buttonProps
+}: {
+  icon: IconType;
+  iconBoxSize?: number;
+  label: string;
+  requireBeta?: boolean;
+  onClick?: () => void;
+} & ButtonProps) => {
+  const flags = useAppStore((s) => s.featureFlags.featureFlags);
+  const flagsLoaded = useAppStore((s) => s.featureFlags.flagsLoaded);
+
+  const [betaModalOpen, setBetaModalOpen] = useState(false);
+
+  const isBetaBlocked = requireBeta && flagsLoaded && !flags.betaAccess;
+  return (
+    <>
+      <Button
+        colorScheme="blue"
+        color="black"
+        bgColor="white"
+        borderColor="gray.300"
+        borderRadius={4}
+        variant="outline"
+        size="sm"
+        fontSize="sm"
+        fontWeight="normal"
+        onClick={isBetaBlocked ? () => setBetaModalOpen(true) : onClick}
+        {...buttonProps}
+      >
+        <HStack spacing={1}>
+          {icon && (
+            <Icon as={icon} boxSize={iconBoxSize} color={requireBeta ? "orange.400" : undefined} />
+          )}
+          <Text display={{ base: "none", md: "flex" }}>{label}</Text>
+        </HStack>
+      </Button>
+      <BetaModal isOpen={betaModalOpen} onClose={() => setBetaModalOpen(false)} />
+    </>
+  );
+};
+
+export default ActionButton;

app/src/components/BetaModal.tsx

@@ -0,0 +1,65 @@
+import {
+  Button,
+  Modal,
+  ModalBody,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  VStack,
+  Text,
+  HStack,
+  Icon,
+  Link,
+} from "@chakra-ui/react";
+import { BsStars } from "react-icons/bs";
+import { useSession } from "next-auth/react";
+
+export const BetaModal = ({ isOpen, onClose }: { isOpen: boolean; onClose: () => void }) => {
+  const session = useSession();
+
+  const email = session.data?.user.email ?? "";
+
+  return (
+    <Modal
+      isOpen={isOpen}
+      onClose={onClose}
+      closeOnOverlayClick={false}
+      size={{ base: "xl", md: "2xl" }}
+    >
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={BsStars} />
+            <Text>Beta-Only Feature</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalBody maxW="unset">
+          <VStack spacing={8} py={4} alignItems="flex-start">
+            <Text fontSize="md">
+              This feature is currently in beta. To receive early access to beta-only features, join
+              the waitlist. You'll receive an email at <b>{email}</b> when you're approved.
+            </Text>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack spacing={4}>
+            <Button
+              as={Link}
+              textDecoration="none !important"
+              colorScheme="orange"
+              target="_blank"
+              href={`https://ax3nafkw0jp.typeform.com/to/ZNpYqvAc#email=${email}`}
+            >
+              Join Waitlist
+            </Button>
+            <Button colorScheme="blue" onClick={onClose}>
+              Done
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/ChangeModelModal/ChangeModelModal.tsx

@@ -1,3 +1,4 @@
+import { useState, useMemo, useCallback } from "react";
 import {
   Button,
   HStack,
@@ -14,16 +15,18 @@ import {
   VStack,
 } from "@chakra-ui/react";
 import { type PromptVariant } from "@prisma/client";
-import { isObject, isString } from "lodash-es";
-import { useState } from "react";
+import { isString } from "lodash-es";
 import { RiExchangeFundsFill } from "react-icons/ri";
+
 import { type ProviderModel } from "~/modelProviders/types";
 import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import { lookupModel, modelLabel } from "~/utils/utils";
 import CompareFunctions from "../RefinePromptModal/CompareFunctions";
 import { ModelSearch } from "./ModelSearch";
 import { ModelStatsCard } from "./ModelStatsCard";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { useAppStore } from "~/state/store";
 
 export const ChangeModelModal = ({
   variant,
@@ -32,48 +35,43 @@ export const ChangeModelModal = ({
   variant: PromptVariant;
   onClose: () => void;
 }) => {
+  const editorOptionsMap = useAppStore((s) => s.sharedVariantEditor.editorOptionsMap);
+  const originalPromptFn = useMemo(
+    () => editorOptionsMap[variant.uiId]?.getContent() || "",
+    [editorOptionsMap, variant.uiId],
+  );
+
   const originalModel = lookupModel(variant.modelProvider, variant.model);
   const [selectedModel, setSelectedModel] = useState({
     provider: variant.modelProvider,
     model: variant.model,
   } as ProviderModel);
   const [convertedModel, setConvertedModel] = useState<ProviderModel | undefined>();
-  const visibleScenarios = useVisibleScenarioIds();
-
-  const utils = api.useContext();
+  const [modifiedPromptFn, setModifiedPromptFn] = useState<string>();
 
   const experiment = useExperiment();
 
-  const { mutateAsync: getModifiedPromptMutateAsync, data: modifiedPromptFn } =
+  const { mutateAsync: getModifiedPromptMutateAsync } =
     api.promptVariants.getModifiedPromptFn.useMutation();
 
   const [getModifiedPromptFn, modificationInProgress] = useHandledAsyncCallback(async () => {
     if (!experiment) return;
 
-    await getModifiedPromptMutateAsync({
+    const resp = await getModifiedPromptMutateAsync({
       id: variant.id,
+      originalPromptFn,
       newModel: selectedModel,
     });
+    if (maybeReportError(resp)) return;
+    setModifiedPromptFn(resp.payload);
     setConvertedModel(selectedModel);
   }, [getModifiedPromptMutateAsync, onClose, experiment, variant, selectedModel]);
 
-  const replaceVariantMutation = api.promptVariants.replaceVariant.useMutation();
-
-  const [replaceVariant, replacementInProgress] = useHandledAsyncCallback(async () => {
-    if (
-      !variant.experimentId ||
-      !modifiedPromptFn ||
-      (isObject(modifiedPromptFn) && "status" in modifiedPromptFn)
-    )
-      return;
-    await replaceVariantMutation.mutateAsync({
-      id: variant.id,
-      promptConstructor: modifiedPromptFn,
-      streamScenarios: visibleScenarios,
-    });
-    await utils.promptVariants.list.invalidate();
+  const replaceVariant = useCallback(() => {
+    if (!modifiedPromptFn) return;
+    editorOptionsMap[variant.uiId]?.setContent(modifiedPromptFn);
     onClose();
-  }, [replaceVariantMutation, variant, onClose, modifiedPromptFn]);
+  }, [variant.uiId, editorOptionsMap, onClose, modifiedPromptFn]);
 
   const originalLabel = modelLabel(variant.modelProvider, variant.model);
   const selectedLabel = modelLabel(selectedModel.provider, selectedModel.model);
@@ -130,9 +128,9 @@ export const ChangeModelModal = ({
               colorScheme="blue"
               onClick={replaceVariant}
               minW={24}
-              isDisabled={!convertedModel || modificationInProgress || replacementInProgress}
+              isDisabled={!convertedModel || modificationInProgress}
             >
-              {replacementInProgress ? <Spinner boxSize={4} /> : <Text>Accept</Text>}
+              Accept
             </Button>
           </HStack>
         </ModalFooter>

app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -1,74 +0,0 @@
-import {
-  Button,
-  Icon,
-  AlertDialog,
-  AlertDialogBody,
-  AlertDialogFooter,
-  AlertDialogHeader,
-  AlertDialogContent,
-  AlertDialogOverlay,
-  useDisclosure,
-  Text,
-} from "@chakra-ui/react";
-
-import { useRouter } from "next/router";
-import { useRef } from "react";
-import { BsTrash } from "react-icons/bs";
-import { useAppStore } from "~/state/store";
-import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-
-export const DeleteButton = () => {
-  const experiment = useExperiment();
-  const mutation = api.experiments.delete.useMutation();
-  const utils = api.useContext();
-  const router = useRouter();
-
-  const closeDrawer = useAppStore((s) => s.closeDrawer);
-
-  const { isOpen, onOpen, onClose } = useDisclosure();
-  const cancelRef = useRef<HTMLButtonElement>(null);
-
-  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id) return;
-    await mutation.mutateAsync({ id: experiment.data.id });
-    await utils.experiments.list.invalidate();
-    await router.push({ pathname: "/experiments" });
-    closeDrawer();
-
-    onClose();
-  }, [mutation, experiment.data?.id, router]);
-
-  return (
-    <>
-      <Button size="sm" variant="ghost" colorScheme="red" fontWeight="normal" onClick={onOpen}>
-        <Icon as={BsTrash} boxSize={4} />
-        <Text ml={2}>Delete Experiment</Text>
-      </Button>
-
-      <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
-        <AlertDialogOverlay>
-          <AlertDialogContent>
-            <AlertDialogHeader fontSize="lg" fontWeight="bold">
-              Delete Experiment
-            </AlertDialogHeader>
-
-            <AlertDialogBody>
-              If you delete this experiment all the associated prompts and scenarios will be deleted
-              as well. Are you sure?
-            </AlertDialogBody>
-
-            <AlertDialogFooter>
-              <Button ref={cancelRef} onClick={onClose}>
-                Cancel
-              </Button>
-              <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
-                Delete
-              </Button>
-            </AlertDialogFooter>
-          </AlertDialogContent>
-        </AlertDialogOverlay>
-      </AlertDialog>
-    </>
-  );
-};

app/src/components/InfoCircle.tsx

@@ -0,0 +1,14 @@
+import { Tooltip, Icon, VStack } from "@chakra-ui/react";
+import { RiInformationFill } from "react-icons/ri";
+
+const InfoCircle = ({ tooltipText }: { tooltipText: string }) => {
+  return (
+    <Tooltip label={tooltipText} fontSize="sm" shouldWrapChildren maxW={80}>
+      <VStack>
+        <Icon as={RiInformationFill} boxSize={5} color="gray.500" />
+      </VStack>
+    </Tooltip>
+  );
+};
+
+export default InfoCircle;

app/src/components/InputDropdown.tsx

@@ -11,26 +11,43 @@ import {
   Button,
   Text,
   useDisclosure,
+  type InputGroupProps,
 } from "@chakra-ui/react";
 
 import { FiChevronDown } from "react-icons/fi";
 import { BiCheck } from "react-icons/bi";
+import { isEqual } from "lodash-es";
+import React from "react";
 
 type InputDropdownProps<T> = {
   options: ReadonlyArray<T>;
   selectedOption: T;
   onSelect: (option: T) => void;
+  inputGroupProps?: InputGroupProps;
+  getDisplayLabel?: (option: T) => string;
+  isDisabled?: boolean;
 };
 
-const InputDropdown = <T,>({ options, selectedOption, onSelect }: InputDropdownProps<T>) => {
-  const popover = useDisclosure();
+const InputDropdown = <T,>({
+  options,
+  selectedOption,
+  onSelect,
+  inputGroupProps,
+  getDisplayLabel = (option) => option as string,
+  isDisabled,
+}: InputDropdownProps<T>) => {
+  const { onOpen, ...popover } = useDisclosure();
 
   return (
-    <Popover placement="bottom-start" {...popover}>
+    <Popover placement="bottom-start" onOpen={isDisabled ? undefined : onOpen} {...popover}>
       <PopoverTrigger>
-        <InputGroup cursor="pointer" w={(selectedOption as string).length * 14 + 180}>
+        <InputGroup
+          cursor="pointer"
+          w={getDisplayLabel(selectedOption).length * 14 + 180}
+          {...inputGroupProps}
+        >
           <Input
-            value={selectedOption as string}
+            value={getDisplayLabel(selectedOption)}
             // eslint-disable-next-line @typescript-eslint/no-empty-function -- controlled input requires onChange
             onChange={() => {}}
             cursor="pointer"
@@ -41,9 +58,10 @@ const InputDropdown = <T,>({ options, selectedOption, onSelect }: InputDropdownP
             onFocus={(e) => {
               e.target.blur();
             }}
+            isDisabled={isDisabled}
           />
           <InputRightElement>
-            <Icon as={FiChevronDown} />
+            <Icon as={FiChevronDown} color={isDisabled ? "gray.300" : undefined} />
           </InputRightElement>
         </InputGroup>
       </PopoverTrigger>
@@ -67,8 +85,10 @@ const InputDropdown = <T,>({ options, selectedOption, onSelect }: InputDropdownP
               fontSize="sm"
               borderBottomWidth={1}
             >
-              <Text mr={16}>{option as string}</Text>
-              {option === selectedOption && <Icon as={BiCheck} color="blue.500" boxSize={5} />}
+              <Text mr={16}>{getDisplayLabel(option)}</Text>
+              {isEqual(option, selectedOption) && (
+                <Icon as={BiCheck} color="blue.500" boxSize={5} />
+              )}
             </HStack>
           ))}
         </VStack>

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -147,9 +147,10 @@ export default function OutputCell({
                 <ResponseLog
                   time={response.receivedAt}
                   title="Response received from API"
-                  message={`statusCode: ${response.statusCode ?? ""}\n ${
-                    response.errorMessage ?? ""
-                  }`}
+                  message={[
+                    response.statusCode ? `Status: ${response.statusCode}\n` : "",
+                    response.errorMessage ?? "",
+                  ].join("")}
                 />
               )}
             </Fragment>

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -19,15 +19,13 @@ import {
   useScenarios,
 } from "~/utils/hooks";
 import { BsGear, BsPencil, BsPlus, BsStars } from "react-icons/bs";
-import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 
 export const ActionButton = (props: ButtonProps) => (
   <Button size="sm" variant="ghost" color="gray.600" {...props} />
 );
 
-export const ScenariosHeader = () => {
-  const openDrawer = useAppStore((s) => s.openDrawer);
+export const ScenariosHeader = ({ openDrawer }: { openDrawer: () => void }) => {
   const { canModify } = useExperimentAccess();
   const scenarios = useScenarios();
 

app/src/components/OutputsTable/VariantEditor.tsx

@@ -10,7 +10,7 @@ import {
 } from "@chakra-ui/react";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { FiMaximize, FiMinimize } from "react-icons/fi";
-import { editorBackground } from "~/state/sharedVariantEditor.slice";
+import { type CreatedEditor, editorBackground } from "~/state/sharedVariantEditor.slice";
 import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 import {
@@ -24,8 +24,10 @@ import { type PromptVariant } from "./types";
 export default function VariantEditor(props: { variant: PromptVariant }) {
   const { canModify } = useExperimentAccess();
   const monaco = useAppStore.use.sharedVariantEditor.monaco();
-  const editorRef = useRef<ReturnType<NonNullable<typeof monaco>["editor"]["create"]> | null>(null);
+  const updateOptionsForEditor = useAppStore.use.sharedVariantEditor.updateOptionsForEditor();
+  const editorRef = useRef<CreatedEditor | null>(null);
   const containerRef = useRef<HTMLDivElement | null>(null);
+  const lastSavedFnRef = useRef(props.variant.promptConstructor);
   const [editorId] = useState(() => `editor_${Math.random().toString(36).substring(7)}`);
   const [isChanged, setIsChanged] = useState(false);
 
@@ -48,22 +50,18 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
   }, [isFullscreen, toggleFullscreen]);
 
   const lastSavedFn = props.variant.promptConstructor;
+  useEffect(() => {
+    // Store in ref so that we can access it dynamically
+    lastSavedFnRef.current = lastSavedFn;
+  }, [lastSavedFn]);
 
   const modifierKey = useModifierKeyLabel();
 
   const checkForChanges = useCallback(() => {
     if (!editorRef.current) return;
     const currentFn = editorRef.current.getValue();
-    setIsChanged(currentFn.length > 0 && currentFn !== lastSavedFn);
-  }, [lastSavedFn]);
-
-  const matchUpdatedSavedFn = useCallback(() => {
-    if (!editorRef.current) return;
-    editorRef.current.setValue(lastSavedFn);
-    setIsChanged(false);
-  }, [lastSavedFn]);
-
-  useEffect(matchUpdatedSavedFn, [matchUpdatedSavedFn, lastSavedFn]);
+    setIsChanged(currentFn.length > 0 && currentFn !== lastSavedFnRef.current);
+  }, [editorRef]);
 
   const replaceVariant = api.promptVariants.replaceVariant.useMutation();
   const utils = api.useContext();
@@ -136,6 +134,11 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
         readOnly: !canModify,
       });
 
+      updateOptionsForEditor(props.variant.uiId, {
+        getContent: () => editorRef.current?.getValue() || "",
+        setContent: (content) => editorRef.current?.setValue(content),
+      });
+
       // Workaround because otherwise the commands only work on whatever
       // editor was loaded on the page last.
       // https://github.com/microsoft/monaco-editor/issues/2947#issuecomment-1422265201
@@ -155,7 +158,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
         });
       });
 
-      editorRef.current.onDidChangeModelContent(checkForChanges);
+      const checkForChangesListener = editorRef.current.onDidChangeModelContent(checkForChanges);
 
       const resizeObserver = new ResizeObserver(() => {
         editorRef.current?.layout();
@@ -164,6 +167,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
 
       return () => {
         resizeObserver.disconnect();
+        checkForChangesListener.dispose();
         editorRef.current?.dispose();
       };
     }
@@ -171,7 +175,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
     // We intentionally skip the onSave and props.savedConfig dependencies here because
     // we don't want to re-render the editor from scratch
     /* eslint-disable-next-line react-hooks/exhaustive-deps */
-  }, [monaco, editorId]);
+  }, [monaco, editorId, updateOptionsForEditor]);
 
   useEffect(() => {
     if (!editorRef.current) return;

app/src/components/OutputsTable/VariantStats.tsx

@@ -20,7 +20,7 @@ export default function VariantStats(props: { variant: PromptVariant }) {
         inputTokens: 0,
         outputTokens: 0,
         scenarioCount: 0,
-        outputCount: 0,
+        finishedCount: 0,
         awaitingCompletions: false,
         awaitingEvals: false,
       },
@@ -42,7 +42,7 @@ export default function VariantStats(props: { variant: PromptVariant }) {
 
   const scale = chroma.scale([failColor, neutralColor, passColor]).domain([0, 0.5, 1]);
 
-  const showNumFinished = data.scenarioCount > 0 && data.scenarioCount !== data.outputCount;
+  const showNumFinished = data.scenarioCount > 0 && data.scenarioCount !== data.finishedCount;
 
   return (
     <HStack
@@ -55,7 +55,7 @@ export default function VariantStats(props: { variant: PromptVariant }) {
       <HStack px={cellPadding.x} flexWrap="wrap">
         {showNumFinished && (
           <Text>
-            {data.outputCount} / {data.scenarioCount}
+            {data.finishedCount} / {data.scenarioCount}
           </Text>
         )}
         {data.evalResults.map((result) => {

app/src/components/OutputsTable/index.tsx

@@ -12,7 +12,13 @@ import ScenarioPaginator from "./ScenarioPaginator";
 import { Fragment } from "react";
 import useScrolledPast from "./useHasScrolledPast";
 
-export default function OutputsTable({ experimentId }: { experimentId: string | undefined }) {
+export default function OutputsTable({
+  experimentId,
+  openDrawer,
+}: {
+  experimentId: string | undefined;
+  openDrawer: () => void;
+}) {
   const variants = api.promptVariants.list.useQuery(
     { experimentId: experimentId as string },
     { enabled: !!experimentId },
@@ -77,6 +83,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
               {...sharedProps}
               borderBottomLeftRadius={isFirst ? 8 : 0}
               borderBottomRightRadius={isLast ? 8 : 0}
+              boxShadow="5px 5px 15px 1px rgba(0, 0, 0, 0.1);"
             >
               <VariantStats variant={variant} />
             </GridItem>
@@ -90,7 +97,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
         colStart={1}
         borderRightWidth={0}
       >
-        <ScenariosHeader />
+        <ScenariosHeader openDrawer={openDrawer} />
       </GridItem>
 
       {scenarios.data.scenarios.map((scenario, i) => (

app/src/components/Paginator.tsx

@@ -1,15 +1,19 @@
-import { HStack, IconButton, Text, Select, type StackProps, Icon } from "@chakra-ui/react";
+import {
+  HStack,
+  IconButton,
+  Text,
+  Select,
+  type StackProps,
+  Icon,
+  useBreakpointValue,
+} from "@chakra-ui/react";
 import React, { useCallback } from "react";
 import { FiChevronsLeft, FiChevronsRight, FiChevronLeft, FiChevronRight } from "react-icons/fi";
 import { usePageParams } from "~/utils/hooks";
 
 const pageSizeOptions = [10, 25, 50, 100];
 
-const Paginator = ({
-  count,
-  condense,
-  ...props
-}: { count: number; condense?: boolean } & StackProps) => {
+const Paginator = ({ count, ...props }: { count: number; condense?: boolean } & StackProps) => {
   const { page, pageSize, setPageParams } = usePageParams();
 
   const lastPage = Math.ceil(count / pageSize);
@@ -37,6 +41,9 @@ const Paginator = ({
   const goToLastPage = () => setPageParams({ page: lastPage }, "replace");
   const goToFirstPage = () => setPageParams({ page: 1 }, "replace");
 
+  const isMobile = useBreakpointValue({ base: true, md: false });
+  const condense = isMobile || props.condense;
+
   if (count === 0) return null;
 
   return (

app/src/components/RefinePromptModal/RefinePromptModal.tsx

@@ -1,3 +1,4 @@
+import { useState, useMemo, useCallback } from "react";
 import {
   Button,
   Modal,
@@ -9,22 +10,23 @@ import {
   ModalOverlay,
   VStack,
   Text,
-  Spinner,
   HStack,
   Icon,
   SimpleGrid,
 } from "@chakra-ui/react";
 import { BsStars } from "react-icons/bs";
 import { api } from "~/utils/api";
-import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { useHandledAsyncCallback } from "~/utils/hooks";
 import { type PromptVariant } from "@prisma/client";
-import { useState } from "react";
+
 import CompareFunctions from "./CompareFunctions";
 import { CustomInstructionsInput } from "../CustomInstructionsInput";
 import { RefineAction } from "./RefineAction";
-import { isObject, isString } from "lodash-es";
+import { isString } from "lodash-es";
 import { type RefinementAction, type SupportedProvider } from "~/modelProviders/types";
 import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { useAppStore } from "~/state/store";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
 
 export const RefinePromptModal = ({
   variant,
@@ -33,19 +35,23 @@ export const RefinePromptModal = ({
   variant: PromptVariant;
   onClose: () => void;
 }) => {
-  const utils = api.useContext();
-  const visibleScenarios = useVisibleScenarioIds();
+  const editorOptionsMap = useAppStore((s) => s.sharedVariantEditor.editorOptionsMap);
+  const originalPromptFn = useMemo(
+    () => editorOptionsMap[variant.uiId]?.getContent() || "",
+    [editorOptionsMap, variant.uiId],
+  );
 
   const refinementActions =
     frontendModelProviders[variant.modelProvider as SupportedProvider].refinementActions || {};
 
-  const { mutateAsync: getModifiedPromptMutateAsync, data: refinedPromptFn } =
+  const { mutateAsync: getModifiedPromptMutateAsync } =
     api.promptVariants.getModifiedPromptFn.useMutation();
   const [instructions, setInstructions] = useState<string>("");
 
   const [activeRefineActionLabel, setActiveRefineActionLabel] = useState<string | undefined>(
     undefined,
   );
+  const [refinedPromptFn, setRefinedPromptFn] = useState<string>();
 
   const [getModifiedPromptFn, modificationInProgress] = useHandledAsyncCallback(
     async (label?: string) => {
@@ -54,31 +60,22 @@ export const RefinePromptModal = ({
         ? (refinementActions[label] as RefinementAction).instructions
         : instructions;
       setActiveRefineActionLabel(label);
-      await getModifiedPromptMutateAsync({
+      const resp = await getModifiedPromptMutateAsync({
         id: variant.id,
+        originalPromptFn,
         instructions: updatedInstructions,
       });
+      if (maybeReportError(resp)) return;
+      setRefinedPromptFn(resp.payload);
     },
     [getModifiedPromptMutateAsync, onClose, variant, instructions, setActiveRefineActionLabel],
   );
 
-  const replaceVariantMutation = api.promptVariants.replaceVariant.useMutation();
-
-  const [replaceVariant, replacementInProgress] = useHandledAsyncCallback(async () => {
-    if (
-      !variant.experimentId ||
-      !refinedPromptFn ||
-      (isObject(refinedPromptFn) && "status" in refinedPromptFn)
-    )
-      return;
-    await replaceVariantMutation.mutateAsync({
-      id: variant.id,
-      promptConstructor: refinedPromptFn,
-      streamScenarios: visibleScenarios,
-    });
-    await utils.promptVariants.list.invalidate();
+  const replaceVariant = useCallback(() => {
+    if (!refinedPromptFn) return;
+    editorOptionsMap[variant.uiId]?.setContent(refinedPromptFn);
     onClose();
-  }, [replaceVariantMutation, variant, onClose, refinedPromptFn]);
+  }, [variant.uiId, editorOptionsMap, onClose, refinedPromptFn]);
 
   return (
     <Modal
@@ -126,7 +123,7 @@ export const RefinePromptModal = ({
               />
             </VStack>
             <CompareFunctions
-              originalFunction={variant.promptConstructor}
+              originalFunction={originalPromptFn}
               newFunction={isString(refinedPromptFn) ? refinedPromptFn : undefined}
               maxH="40vh"
             />
@@ -139,9 +136,9 @@ export const RefinePromptModal = ({
               colorScheme="blue"
               onClick={replaceVariant}
               minW={24}
-              isDisabled={replacementInProgress || !refinedPromptFn}
+              isDisabled={!refinedPromptFn}
             >
-              {replacementInProgress ? <Spinner boxSize={4} /> : <Text>Accept</Text>}
+              Accept
             </Button>
           </HStack>
         </ModalFooter>

app/src/components/StatsCard.tsx

@@ -1,26 +0,0 @@
-import { VStack, HStack, type StackProps, Text, Divider } from "@chakra-ui/react";
-import Link, { type LinkProps } from "next/link";
-
-const StatsCard = ({
-  title,
-  href,
-  children,
-  ...rest
-}: { title: string; href: string } & StackProps & LinkProps) => {
-  return (
-    <VStack flex={1} borderWidth={1} padding={4} borderRadius={4} borderColor="gray.300" {...rest}>
-      <HStack w="full" justifyContent="space-between">
-        <Text fontSize="md" fontWeight="bold">
-          {title}
-        </Text>
-        <Link href={href}>
-          <Text color="blue">View all</Text>
-        </Link>
-      </HStack>
-      <Divider />
-      {children}
-    </VStack>
-  );
-};
-
-export default StatsCard;

app/src/components/dashboard/LoggedCallsTable.tsx

@@ -2,11 +2,12 @@ import { Card, CardHeader, Heading, Table, Tbody, HStack, Button, Text } from "@
 import { useState } from "react";
 import Link from "next/link";
 import { useLoggedCalls } from "~/utils/hooks";
-import { TableHeader, TableRow } from "../requestLogs/TableRow";
+import { EmptyTableRow, TableHeader, TableRow } from "../requestLogs/TableRow";
 
 export default function LoggedCallsTable() {
+  const { data: loggedCalls } = useLoggedCalls(false);
+
   const [expandedRow, setExpandedRow] = useState<string | null>(null);
-  const { data: loggedCalls } = useLoggedCalls();
 
   return (
     <Card width="100%" overflow="hidden">
@@ -23,22 +24,26 @@ export default function LoggedCallsTable() {
       <Table>
         <TableHeader />
         <Tbody>
-          {loggedCalls?.calls.map((loggedCall) => {
-            return (
-              <TableRow
-                key={loggedCall.id}
-                loggedCall={loggedCall}
-                isExpanded={loggedCall.id === expandedRow}
-                onToggle={() => {
-                  if (loggedCall.id === expandedRow) {
-                    setExpandedRow(null);
-                  } else {
-                    setExpandedRow(loggedCall.id);
-                  }
-                }}
-              />
-            );
-          })}
+          {loggedCalls?.calls.length ? (
+            loggedCalls?.calls.map((loggedCall) => {
+              return (
+                <TableRow
+                  key={loggedCall.id}
+                  loggedCall={loggedCall}
+                  isExpanded={loggedCall.id === expandedRow}
+                  onToggle={() => {
+                    if (loggedCall.id === expandedRow) {
+                      setExpandedRow(null);
+                    } else {
+                      setExpandedRow(loggedCall.id);
+                    }
+                  }}
+                />
+              );
+            })
+          ) : (
+            <EmptyTableRow filtersApplied={false} />
+          )}
         </Tbody>
       </Table>
     </Card>

app/src/components/datasets/DatasetCard.tsx

@@ -1,112 +0,0 @@
-import {
-  HStack,
-  Icon,
-  VStack,
-  Text,
-  Divider,
-  Spinner,
-  AspectRatio,
-  SkeletonText,
-} from "@chakra-ui/react";
-import { RiDatabase2Line } from "react-icons/ri";
-import { formatTimePast } from "~/utils/dayjs";
-import Link from "next/link";
-import { useRouter } from "next/router";
-import { BsPlusSquare } from "react-icons/bs";
-import { api } from "~/utils/api";
-import { useHandledAsyncCallback } from "~/utils/hooks";
-import { useAppStore } from "~/state/store";
-
-type DatasetData = {
-  name: string;
-  numEntries: number;
-  id: string;
-  createdAt: Date;
-  updatedAt: Date;
-};
-
-export const DatasetCard = ({ dataset }: { dataset: DatasetData }) => {
-  return (
-    <AspectRatio ratio={1.2} w="full">
-      <VStack
-        as={Link}
-        href={{ pathname: "/data/[id]", query: { id: dataset.id } }}
-        bg="gray.50"
-        _hover={{ bg: "gray.100" }}
-        transition="background 0.2s"
-        cursor="pointer"
-        borderColor="gray.200"
-        borderWidth={1}
-        p={4}
-        justify="space-between"
-      >
-        <HStack w="full" color="gray.700" justify="center">
-          <Icon as={RiDatabase2Line} boxSize={4} />
-          <Text fontWeight="bold">{dataset.name}</Text>
-        </HStack>
-        <HStack h="full" spacing={4} flex={1} align="center">
-          <CountLabel label="Rows" count={dataset.numEntries} />
-        </HStack>
-        <HStack w="full" color="gray.500" fontSize="xs" textAlign="center">
-          <Text flex={1}>Created {formatTimePast(dataset.createdAt)}</Text>
-          <Divider h={4} orientation="vertical" />
-          <Text flex={1}>Updated {formatTimePast(dataset.updatedAt)}</Text>
-        </HStack>
-      </VStack>
-    </AspectRatio>
-  );
-};
-
-const CountLabel = ({ label, count }: { label: string; count: number }) => {
-  return (
-    <VStack alignItems="center" flex={1}>
-      <Text color="gray.500" fontWeight="bold">
-        {label}
-      </Text>
-      <Text fontSize="sm" color="gray.500">
-        {count}
-      </Text>
-    </VStack>
-  );
-};
-
-export const NewDatasetCard = () => {
-  const router = useRouter();
-  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
-  const createMutation = api.datasets.create.useMutation();
-  const [createDataset, isLoading] = useHandledAsyncCallback(async () => {
-    const newDataset = await createMutation.mutateAsync({ projectId: selectedProjectId ?? "" });
-    await router.push({ pathname: "/data/[id]", query: { id: newDataset.id } });
-  }, [createMutation, router, selectedProjectId]);
-
-  return (
-    <AspectRatio ratio={1.2} w="full">
-      <VStack
-        align="center"
-        justify="center"
-        _hover={{ cursor: "pointer", bg: "gray.50" }}
-        transition="background 0.2s"
-        cursor="pointer"
-        borderColor="gray.200"
-        borderWidth={1}
-        p={4}
-        onClick={createDataset}
-      >
-        <Icon as={isLoading ? Spinner : BsPlusSquare} boxSize={8} />
-        <Text display={{ base: "none", md: "block" }} ml={2}>
-          New Dataset
-        </Text>
-      </VStack>
-    </AspectRatio>
-  );
-};
-
-export const DatasetCardSkeleton = () => (
-  <AspectRatio ratio={1.2} w="full">
-    <VStack align="center" borderColor="gray.200" borderWidth={1} p={4} bg="gray.50">
-      <SkeletonText noOfLines={1} w="80%" />
-      <SkeletonText noOfLines={2} w="60%" />
-      <SkeletonText noOfLines={1} w="80%" />
-    </VStack>
-  </AspectRatio>
-);

app/src/components/datasets/DatasetConfigurationDrawer/DatasetConfigurationDrawer.tsx

@@ -0,0 +1,37 @@
+import {
+  Drawer,
+  DrawerBody,
+  DrawerCloseButton,
+  DrawerContent,
+  DrawerHeader,
+  DrawerOverlay,
+  Heading,
+  VStack,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+
+import { DeleteButton } from "./DeleteButton";
+
+export default function DatasetConfigurationDrawer({
+  disclosure,
+}: {
+  disclosure: UseDisclosureReturn;
+}) {
+  return (
+    <Drawer placement="right" size="md" {...disclosure}>
+      <DrawerOverlay />
+      <DrawerContent>
+        <DrawerCloseButton />
+        <DrawerHeader>
+          <Heading size="md">Dataset Configuration</Heading>
+        </DrawerHeader>
+        <DrawerBody h="full" pb={4}>
+          <VStack h="full" justifyContent="space-between">
+            <VStack spacing={6}></VStack>
+            <DeleteButton closeDrawer={disclosure.onClose} />
+          </VStack>
+        </DrawerBody>
+      </DrawerContent>
+    </Drawer>
+  );
+}

app/src/components/datasets/DatasetConfigurationDrawer/DeleteButton.tsx

@@ -0,0 +1,39 @@
+import { Button, Icon, useDisclosure, Text } from "@chakra-ui/react";
+import { useRouter } from "next/router";
+import { BsTrash } from "react-icons/bs";
+
+import { useHandledAsyncCallback, useDataset } from "~/utils/hooks";
+import DeleteDatasetDialog from "./DeleteDatasetDialog";
+
+export const DeleteButton = ({ closeDrawer }: { closeDrawer: () => void }) => {
+  const dataset = useDataset();
+  const router = useRouter();
+
+  const disclosure = useDisclosure();
+
+  const [onDelete] = useHandledAsyncCallback(async () => {
+    await router.push({ pathname: "/datasets" });
+    closeDrawer();
+  }, [router, closeDrawer]);
+
+  return (
+    <>
+      <Button
+        size="sm"
+        variant="ghost"
+        colorScheme="red"
+        fontWeight="normal"
+        onClick={disclosure.onOpen}
+      >
+        <Icon as={BsTrash} boxSize={4} />
+        <Text ml={2}>Delete Dataset</Text>
+      </Button>
+
+      <DeleteDatasetDialog
+        datasetId={dataset.data?.id}
+        onDelete={onDelete}
+        disclosure={disclosure}
+      />
+    </>
+  );
+};

app/src/components/datasets/DatasetConfigurationDrawer/DeleteDatasetDialog.tsx

@@ -0,0 +1,73 @@
+import { useRef } from "react";
+import {
+  type UseDisclosureReturn,
+  AlertDialog,
+  AlertDialogOverlay,
+  AlertDialogContent,
+  AlertDialogHeader,
+  AlertDialogBody,
+  AlertDialogFooter,
+  Button,
+} from "@chakra-ui/react";
+import { api } from "~/utils/api";
+
+import { useHandledAsyncCallback } from "~/utils/hooks";
+
+const DeleteDatasetDialog = ({
+  datasetId,
+  onDelete,
+  disclosure,
+}: {
+  datasetId?: string;
+  onDelete?: () => void;
+  disclosure: UseDisclosureReturn;
+}) => {
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const mutation = api.datasets.delete.useMutation();
+  const utils = api.useContext();
+
+  const [onDeleteConfirm, deletionInProgress] = useHandledAsyncCallback(async () => {
+    if (!datasetId) return;
+    await mutation.mutateAsync({ id: datasetId });
+    await utils.datasets.list.invalidate();
+    onDelete?.();
+
+    disclosure.onClose();
+  }, [mutation, datasetId, disclosure.onClose]);
+
+  console.log("dataset id", datasetId);
+
+  return (
+    <AlertDialog leastDestructiveRef={cancelRef} {...disclosure}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Delete Dataset
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            If you delete this dataset all the associated dataset entries will be deleted as well.
+            Are you sure?
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={disclosure.onClose}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="red"
+              isLoading={deletionInProgress}
+              onClick={onDeleteConfirm}
+              ml={3}
+            >
+              Delete
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};
+
+export default DeleteDatasetDialog;

app/src/components/datasets/DatasetEntriesTable.tsx

@@ -1,31 +0,0 @@
-import { type StackProps, VStack, Table, Th, Tr, Thead, Tbody, Text } from "@chakra-ui/react";
-import { useDatasetEntries } from "~/utils/hooks";
-import TableRow from "./TableRow";
-import DatasetEntriesPaginator from "./DatasetEntriesPaginator";
-
-const DatasetEntriesTable = (props: StackProps) => {
-  const { data } = useDatasetEntries();
-
-  return (
-    <VStack justifyContent="space-between" {...props}>
-      <Table variant="simple" sx={{ "table-layout": "fixed", width: "full" }}>
-        <Thead>
-          <Tr>
-            <Th>Input</Th>
-            <Th>Output</Th>
-          </Tr>
-        </Thead>
-        <Tbody>{data?.entries.map((entry) => <TableRow key={entry.id} entry={entry} />)}</Tbody>
-      </Table>
-      {(!data || data.entries.length) === 0 ? (
-        <Text alignSelf="flex-start" pl={6} color="gray.500">
-          No entries found
-        </Text>
-      ) : (
-        <DatasetEntriesPaginator />
-      )}
-    </VStack>
-  );
-};
-
-export default DatasetEntriesTable;

app/src/components/datasets/DatasetEntriesTable/DatasetEntriesTable.tsx

@@ -0,0 +1,46 @@
+import { Card, Table, Tbody } from "@chakra-ui/react";
+import { useState } from "react";
+import { useDatasetEntries } from "~/utils/hooks";
+import { TableHeader, TableRow, EmptyTableRow } from "./TableRow";
+import DatasetEntryEditorDrawer from "./DatasetEntryEditorDrawer";
+
+export default function DatasetEntriesTable() {
+  const [expandedDatasetEntryId, setExpandedDatasetEntryId] = useState<string | null>(null);
+  const datasetEntries = useDatasetEntries().data?.entries;
+
+  return (
+    <>
+      <Card width="100%" overflowX="auto">
+        <Table>
+          <TableHeader />
+          <Tbody>
+            {datasetEntries?.length ? (
+              datasetEntries?.map((entry) => {
+                return (
+                  <TableRow
+                    key={entry.id}
+                    datasetEntry={entry}
+                    onToggle={() => {
+                      if (entry.id === expandedDatasetEntryId) {
+                        setExpandedDatasetEntryId(null);
+                      } else {
+                        setExpandedDatasetEntryId(entry.id);
+                      }
+                    }}
+                    showOptions
+                  />
+                );
+              })
+            ) : (
+              <EmptyTableRow />
+            )}
+          </Tbody>
+        </Table>
+      </Card>
+      <DatasetEntryEditorDrawer
+        datasetEntryId={expandedDatasetEntryId}
+        clearDatasetEntryId={() => setExpandedDatasetEntryId(null)}
+      />
+    </>
+  );
+}

app/src/components/datasets/DatasetEntriesTable/DatasetEntryEditorDrawer.tsx

@@ -0,0 +1,174 @@
+import { useState, useEffect, useMemo } from "react";
+import {
+  Drawer,
+  DrawerBody,
+  DrawerCloseButton,
+  DrawerContent,
+  DrawerHeader,
+  DrawerOverlay,
+  DrawerFooter,
+  Heading,
+  VStack,
+  HStack,
+  Button,
+  Text,
+  Divider,
+  Icon,
+} from "@chakra-ui/react";
+import { type CreateChatCompletionRequestMessage } from "openai/resources/chat";
+import { BsPlus } from "react-icons/bs";
+import { type DatasetEntryType } from "@prisma/client";
+
+import { api } from "~/utils/api";
+import { useDatasetEntry, useHandledAsyncCallback } from "~/utils/hooks";
+import EditableMessage from "./EditableMessage";
+import EntryTypeDropdown from "./EntryTypeDropdown";
+
+export default function DatasetDentryEditorDrawer({
+  datasetEntryId,
+  clearDatasetEntryId,
+}: {
+  datasetEntryId: string | null;
+  clearDatasetEntryId: () => void;
+}) {
+  const utils = api.useContext();
+
+  const datasetEntry = useDatasetEntry(datasetEntryId).data;
+
+  const savedInputMessages = useMemo(
+    () => datasetEntry?.input as unknown as CreateChatCompletionRequestMessage[],
+    [datasetEntry],
+  );
+  const savedOutputMessage = useMemo(
+    () => datasetEntry?.output as unknown as CreateChatCompletionRequestMessage,
+    [datasetEntry],
+  );
+
+  const [inputMessagesToSave, setInputMessagesToSave] = useState<
+    CreateChatCompletionRequestMessage[]
+  >([]);
+  const [outputMessageToSave, setOutputMessageToSave] =
+    useState<CreateChatCompletionRequestMessage | null>(null);
+
+  useEffect(() => {
+    if (savedInputMessages) {
+      setInputMessagesToSave(savedInputMessages);
+      setOutputMessageToSave(savedOutputMessage);
+    }
+  }, [savedInputMessages, savedOutputMessage]);
+
+  const updateMutation = api.datasetEntries.update.useMutation();
+  const [onSave, savingInProgress] = useHandledAsyncCallback(async () => {
+    if (!datasetEntryId || !inputMessagesToSave) return;
+    await updateMutation.mutateAsync({
+      id: datasetEntryId,
+      updates: {
+        input: JSON.stringify(inputMessagesToSave),
+        output: JSON.stringify(outputMessageToSave),
+      },
+    });
+    await utils.datasetEntries.list.invalidate();
+    await utils.datasetEntries.get.invalidate({ id: datasetEntryId });
+  }, [updateMutation, datasetEntryId, inputMessagesToSave, outputMessageToSave, utils]);
+
+  const [onUpdateType] = useHandledAsyncCallback(
+    async (type: DatasetEntryType) => {
+      if (!datasetEntryId) return;
+      await updateMutation.mutateAsync({
+        id: datasetEntryId,
+        updates: {
+          type,
+        },
+      });
+      await utils.datasetEntries.list.invalidate();
+      await utils.datasetEntries.get.invalidate({ id: datasetEntryId });
+    },
+    [updateMutation, datasetEntryId, utils],
+  );
+
+  return (
+    <Drawer isOpen={!!datasetEntryId} onClose={clearDatasetEntryId} placement="right" size="md">
+      <DrawerOverlay />
+      <DrawerContent>
+        <DrawerCloseButton pt={6} />
+        <DrawerHeader bgColor="orange.50">
+          <HStack w="full" justifyContent="space-between" pr={8}>
+            <Heading size="md">Dataset Entry</Heading>
+            {datasetEntry && (
+              <EntryTypeDropdown type={datasetEntry.type} onTypeChange={onUpdateType} />
+            )}
+          </HStack>
+        </DrawerHeader>
+        <DrawerBody h="full" pb={4} bgColor="orange.50">
+          <VStack h="full" justifyContent="space-between">
+            <VStack w="full" spacing={12} py={4}>
+              <VStack w="full" alignItems="flex-start">
+                <Text fontWeight="bold">Input</Text>
+                {inputMessagesToSave.map((message, i) => {
+                  return (
+                    <>
+                      <Divider key={`divider-${i}`} my={4} />
+                      <EditableMessage
+                        key={i}
+                        message={message}
+                        onEdit={(message) => {
+                          const newInputMessages = [...inputMessagesToSave];
+                          newInputMessages[i] = message;
+                          setInputMessagesToSave(newInputMessages);
+                        }}
+                        onDelete={() => {
+                          const newInputMessages = [...inputMessagesToSave];
+                          newInputMessages.splice(i, 1);
+                          setInputMessagesToSave(newInputMessages);
+                        }}
+                      />
+                    </>
+                  );
+                })}
+                <Divider my={4} />
+                <Button
+                  w="full"
+                  onClick={() =>
+                    setInputMessagesToSave([...inputMessagesToSave, { role: "user", content: "" }])
+                  }
+                  variant="outline"
+                  color="gray.500"
+                  _hover={{ bgColor: "orange.100" }}
+                >
+                  <HStack spacing={0}>
+                    <Text>Add Message</Text>
+                    <Icon as={BsPlus} boxSize={6} />
+                  </HStack>
+                </Button>
+              </VStack>
+              <VStack w="full" alignItems="flex-start">
+                <Text fontWeight="bold">Output</Text>
+                <Divider my={4} />
+                <EditableMessage
+                  message={outputMessageToSave}
+                  onEdit={(message) => setOutputMessageToSave(message)}
+                  isOutput
+                />
+              </VStack>
+            </VStack>
+          </VStack>
+        </DrawerBody>
+        <DrawerFooter bgColor="orange.50">
+          <HStack>
+            <Button
+              onClick={() => {
+                setInputMessagesToSave(savedInputMessages);
+                setOutputMessageToSave(savedOutputMessage);
+              }}
+            >
+              Reset
+            </Button>
+            <Button isLoading={savingInProgress} onClick={onSave} colorScheme="orange">
+              Save
+            </Button>
+          </HStack>
+        </DrawerFooter>
+      </DrawerContent>
+    </Drawer>
+  );
+}

app/src/components/datasets/DatasetEntriesTable/EditableMessage.tsx

@@ -0,0 +1,105 @@
+import { VStack, HStack, Tooltip, IconButton, Icon } from "@chakra-ui/react";
+import { type CreateChatCompletionRequestMessage } from "openai/resources/chat";
+import { BsX } from "react-icons/bs";
+
+import AutoResizeTextArea from "~/components/AutoResizeTextArea";
+import InputDropdown from "~/components/InputDropdown";
+import { parseableToFunctionCall } from "~/utils/utils";
+import FunctionCallEditor from "./FunctionCallEditor";
+
+const MESSAGE_ROLE_OPTIONS = ["system", "user", "assistant", "function"] as const;
+const OUTPUT_OPTIONS = ["plaintext", "func_call"] as const;
+
+const EditableMessage = ({
+  message,
+  onEdit,
+  onDelete,
+  isOutput,
+}: {
+  message: CreateChatCompletionRequestMessage | null;
+  onEdit: (message: CreateChatCompletionRequestMessage) => void;
+  onDelete?: () => void;
+  isOutput?: boolean;
+}) => {
+  const { role = "assistant", content = "", function_call } = message || {};
+
+  const currentOutputOption: (typeof OUTPUT_OPTIONS)[number] = function_call
+    ? "func_call"
+    : "plaintext";
+
+  return (
+    <VStack w="full">
+      <HStack w="full" justifyContent="space-between">
+        <HStack>
+          {!isOutput && (
+            <InputDropdown
+              options={MESSAGE_ROLE_OPTIONS}
+              selectedOption={role}
+              onSelect={(option) => {
+                const updatedMessage = { role: option, content };
+                if (role === "assistant" && currentOutputOption === "func_call") {
+                  updatedMessage.content = JSON.stringify(function_call, null, 2);
+                }
+                onEdit(updatedMessage);
+              }}
+              inputGroupProps={{ w: "32", bgColor: "white" }}
+            />
+          )}
+          {role === "assistant" && (
+            <InputDropdown
+              options={OUTPUT_OPTIONS}
+              selectedOption={currentOutputOption}
+              onSelect={(option) => {
+                const updatedMessage: CreateChatCompletionRequestMessage = {
+                  role,
+                  content: null,
+                  function_call: undefined,
+                };
+                if (option === "plaintext") {
+                  updatedMessage.content = JSON.stringify(function_call, null, 2);
+                } else if (option === "func_call") {
+                  updatedMessage.function_call =
+                    content && parseableToFunctionCall(content)
+                      ? JSON.parse(content)
+                      : { name: "", arguments: "{}" };
+                }
+                onEdit(updatedMessage);
+              }}
+              inputGroupProps={{ w: "32", bgColor: "white" }}
+            />
+          )}
+        </HStack>
+        {!isOutput && (
+          <HStack>
+            <Tooltip label="Delete" hasArrow>
+              <IconButton
+                aria-label="Delete"
+                icon={<Icon as={BsX} boxSize={6} />}
+                onClick={onDelete}
+                size="xs"
+                display="flex"
+                colorScheme="gray"
+                color="gray.500"
+                variant="ghost"
+              />
+            </Tooltip>
+          </HStack>
+        )}
+      </HStack>
+      {function_call ? (
+        <FunctionCallEditor
+          function_call={function_call}
+          onEdit={(function_call) => onEdit({ role, function_call, content: null })}
+        />
+      ) : (
+        <AutoResizeTextArea
+          value={content || JSON.stringify(function_call, null, 2)}
+          onChange={(e) => onEdit({ role, content: e.target.value })}
+          bgColor="white"
+        />
+      )}
+    </VStack>
+  );
+};
+
+export default EditableMessage;

app/src/components/datasets/DatasetEntriesTable/EntryTypeDropdown.tsx

@@ -0,0 +1,24 @@
+import { type DatasetEntryType } from "@prisma/client";
+
+import InputDropdown from "~/components/InputDropdown";
+
+const ENTRY_TYPE_OPTIONS: DatasetEntryType[] = ["TRAIN", "TEST"];
+
+const EntryTypeDropdown = ({
+  type,
+  onTypeChange,
+}: {
+  type: DatasetEntryType;
+  onTypeChange: (type: DatasetEntryType) => void;
+}) => {
+  return (
+    <InputDropdown
+      options={ENTRY_TYPE_OPTIONS}
+      selectedOption={type}
+      onSelect={onTypeChange}
+      inputGroupProps={{ w: "32", bgColor: "white" }}
+    />
+  );
+};
+
+export default EntryTypeDropdown;

app/src/components/datasets/DatasetEntriesTable/FunctionCallEditor.tsx

@@ -0,0 +1,125 @@
+import { useRef, useMemo, useEffect } from "react";
+import { VStack, HStack, Text, Input, Box } from "@chakra-ui/react";
+import { type CreateChatCompletionRequestMessage } from "openai/resources/chat";
+
+import { useAppStore } from "~/state/store";
+import { type CreatedEditor } from "~/state/sharedVariantEditor.slice";
+
+const FunctionCallEditor = ({
+  function_call,
+  onEdit,
+}: {
+  function_call: CreateChatCompletionRequestMessage.FunctionCall;
+  onEdit: (function_call: CreateChatCompletionRequestMessage.FunctionCall) => void;
+}) => {
+  const monaco = useAppStore.use.sharedArgumentsEditor.monaco();
+  const editorRef = useRef<CreatedEditor | null>(null);
+  const editorId = useMemo(() => `editor_${Math.random().toString(36).substring(7)}`, []);
+
+  useEffect(() => {
+    if (monaco) {
+      const container = document.getElementById(editorId) as HTMLElement;
+
+      const editor = monaco.editor.create(container, {
+        value: function_call.arguments,
+        language: "json",
+        theme: "customTheme",
+        lineNumbers: "off",
+        minimap: { enabled: false },
+        wrappingIndent: "indent",
+        wrappingStrategy: "advanced",
+        wordWrap: "on",
+        folding: false,
+        scrollbar: {
+          alwaysConsumeMouseWheel: false,
+          verticalScrollbarSize: 0,
+        },
+        wordWrapBreakAfterCharacters: "",
+        wordWrapBreakBeforeCharacters: "",
+        quickSuggestions: true,
+        renderLineHighlight: "none",
+        fontSize: 14,
+        scrollBeyondLastLine: false,
+      });
+
+      editorRef.current = editor;
+
+      const updateHeight = () => {
+        const contentHeight = editor.getContentHeight();
+        container.style.height = `${contentHeight}px`;
+        editor.layout();
+      };
+
+      const attemptDocumentFormat = () => {
+        const action = editor.getAction("editor.action.formatDocument");
+        if (action) {
+          action
+            .run()
+            .then(updateHeight)
+            .catch((error) => {
+              console.error("Error running formatDocument:", error);
+            });
+          return true;
+        }
+        return false;
+      };
+
+      editor.onDidBlurEditorText(() => {
+        attemptDocumentFormat();
+        onEdit({ name: function_call.name, arguments: editor.getValue() });
+      });
+
+      // Interval function to check for action availability
+      const checkForActionInterval = setInterval(() => {
+        const formatted = attemptDocumentFormat();
+        if (formatted) {
+          clearInterval(checkForActionInterval); // Clear the interval once the action is found and run
+        }
+      }, 100); // Check every 100ms
+
+      // Add content change listener
+      const contentChangeListener = editor.onDidChangeModelContent(updateHeight);
+
+      const resizeObserver = new ResizeObserver(() => {
+        editor.layout();
+      });
+      resizeObserver.observe(container);
+
+      return () => {
+        contentChangeListener.dispose();
+        resizeObserver.disconnect();
+        editor?.dispose();
+      };
+    }
+  }, [monaco, editorId, function_call.name, function_call.arguments, onEdit]);
+
+  return (
+    <VStack w="full" alignItems="flex-start">
+      <HStack w="full">
+        <Text fontWeight="bold" w={192}>
+          Name:
+        </Text>
+        <Input
+          value={function_call.name}
+          onChange={(e) => onEdit({ name: e.target.value, arguments: function_call.arguments })}
+          bgColor="white"
+        />
+      </HStack>
+      <Text fontWeight="bold" w={32}>
+        Arguments
+      </Text>
+      <VStack
+        borderRadius={4}
+        border="1px solid"
+        borderColor="gray.200"
+        w="full"
+        py={1}
+        bgColor="white"
+      >
+        <Box id={editorId} w="full" />
+      </VStack>
+    </VStack>
+  );
+};
+
+export default FunctionCallEditor;

app/src/components/datasets/DatasetEntriesTable/TableRow.tsx

@@ -0,0 +1,128 @@
+import { Box, Td, Tr, Thead, Th, Tooltip, HStack, Text, Checkbox } from "@chakra-ui/react";
+import Link from "next/link";
+
+import dayjs from "~/utils/dayjs";
+import { type RouterOutputs } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import { useIsClientRehydrated, useDatasetEntries } from "~/utils/hooks";
+import { useMemo } from "react";
+
+type DatasetEntry = RouterOutputs["datasetEntries"]["list"]["entries"][0];
+
+export const TableHeader = () => {
+  const matchingDatasetEntryIds = useDatasetEntries().data?.matchingEntryIds;
+  const selectedDatasetEntryIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+  const addSelectedIds = useAppStore((s) => s.selectedDatasetEntries.addSelectedIds);
+  const clearSelectedIds = useAppStore((s) => s.selectedDatasetEntries.clearSelectedIds);
+  const allSelected = useMemo(() => {
+    if (!matchingDatasetEntryIds || !matchingDatasetEntryIds.length) return false;
+    return matchingDatasetEntryIds.every((id) => selectedDatasetEntryIds.has(id));
+  }, [matchingDatasetEntryIds, selectedDatasetEntryIds]);
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
+  return (
+    <Thead>
+      <Tr>
+        <Th pr={0}>
+          <HStack minW={16}>
+            <Checkbox
+              isChecked={allSelected}
+              onChange={() => {
+                allSelected ? clearSelectedIds() : addSelectedIds(matchingDatasetEntryIds || []);
+              }}
+            />
+            <Text>
+              ({selectedDatasetEntryIds.size ? `${selectedDatasetEntryIds.size}/` : ""}
+              {matchingDatasetEntryIds?.length || 0})
+            </Text>
+          </HStack>
+        </Th>
+        <Th>Created At</Th>
+        <Th isNumeric>Input tokens</Th>
+        <Th isNumeric>Output tokens</Th>
+        <Th isNumeric>Type</Th>
+      </Tr>
+    </Thead>
+  );
+};
+
+export const TableRow = ({
+  datasetEntry,
+  onToggle,
+  showOptions,
+}: {
+  datasetEntry: DatasetEntry;
+  onToggle: () => void;
+  showOptions?: boolean;
+}) => {
+  const createdAt = dayjs(datasetEntry.createdAt).format("MMMM D h:mm A");
+  const fullTime = dayjs(datasetEntry.createdAt).toString();
+
+  const isChecked = useAppStore((s) => s.selectedDatasetEntries.selectedIds.has(datasetEntry.id));
+  const toggleChecked = useAppStore((s) => s.selectedDatasetEntries.toggleSelectedId);
+
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
+  return (
+    <Tr
+      onClick={onToggle}
+      key={datasetEntry.id}
+      _hover={{ bgColor: "gray.50", cursor: "pointer" }}
+      fontSize="sm"
+    >
+      {showOptions && (
+        <Td>
+          <Checkbox isChecked={isChecked} onChange={() => toggleChecked(datasetEntry.id)} />
+        </Td>
+      )}
+      <Td>
+        <Tooltip label={fullTime} placement="top">
+          <Box whiteSpace="nowrap" minW="120px">
+            {createdAt}
+          </Box>
+        </Tooltip>
+      </Td>
+      <Td isNumeric>{datasetEntry.inputTokens}</Td>
+      <Td isNumeric>{datasetEntry.outputTokens}</Td>
+      <Td isNumeric>{datasetEntry.type}</Td>
+    </Tr>
+  );
+};
+
+export const EmptyTableRow = ({ filtersApplied = true }: { filtersApplied?: boolean }) => {
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const filters = useAppStore((state) => state.logFilters.filters);
+  const { isLoading } = useDatasetEntries();
+
+  if (isLoading) return null;
+
+  if (filters.length && filtersApplied) {
+    return (
+      <Tr>
+        <Td w="full" colSpan={visibleColumns.size + 1}>
+          <Text color="gray.500" textAlign="center" w="full" p={4}>
+            No matching entries found. Try removing some filters.
+          </Text>
+        </Td>
+      </Tr>
+    );
+  }
+
+  return (
+    <Tr>
+      <Td w="full" colSpan={visibleColumns.size + 1}>
+        <Text color="gray.500" textAlign="center" w="full" p={4}>
+          This dataset has no entries. Add some logs in the{" "}
+          <Link href="/request-logs">
+            <Text as="span" color="blue.600">
+              Request Logs
+            </Text>
+          </Link>{" "}
+          tab.
+        </Text>
+      </Td>
+    </Tr>
+  );
+};

app/src/components/datasets/DatasetEntryPaginator.tsx

@@ -3,14 +3,14 @@ import { type StackProps } from "@chakra-ui/react";
 import { useDatasetEntries } from "~/utils/hooks";
 import Paginator from "../Paginator";
 
-const DatasetEntriesPaginator = (props: StackProps) => {
+const DatasetEntryPaginator = (props: StackProps) => {
   const { data } = useDatasetEntries();
 
   if (!data) return null;
 
-  const { count } = data;
+  const { matchingEntryIds } = data;
 
-  return <Paginator count={count} {...props} />;
+  return <Paginator count={matchingEntryIds.length} {...props} />;
 };
 
-export default DatasetEntriesPaginator;
+export default DatasetEntryPaginator;

app/src/components/datasets/DatasetHeaderButtons.tsx

@@ -0,0 +1,20 @@
+import { Button, HStack, Icon, Text } from "@chakra-ui/react";
+import { useDataset } from "~/utils/hooks";
+import { BsGearFill } from "react-icons/bs";
+
+export const DatasetHeaderButtons = ({ openDrawer }: { openDrawer: () => void }) => {
+  const dataset = useDataset();
+
+  if (dataset.isLoading) return null;
+
+  return (
+    <HStack spacing={0} mt={{ base: 2, md: 0 }}>
+      <Button variant={{ base: "solid", md: "ghost" }} onClick={openDrawer}>
+        <HStack>
+          <Icon as={BsGearFill} />
+          <Text>Configure</Text>
+        </HStack>
+      </Button>
+    </HStack>
+  );
+};

app/src/components/datasets/DatasetHeaderButtons/DatasetHeaderButtons.tsx

@@ -1,26 +0,0 @@
-import { Button, HStack, useDisclosure } from "@chakra-ui/react";
-import { BiImport } from "react-icons/bi";
-import { BsStars } from "react-icons/bs";
-
-import { GenerateDataModal } from "./GenerateDataModal";
-
-export const DatasetHeaderButtons = () => {
-  const generateModalDisclosure = useDisclosure();
-
-  return (
-    <>
-      <HStack>
-        <Button leftIcon={<BiImport />} colorScheme="blue" variant="ghost">
-          Import Data
-        </Button>
-        <Button leftIcon={<BsStars />} colorScheme="blue" onClick={generateModalDisclosure.onOpen}>
-          Generate Data
-        </Button>
-      </HStack>
-      <GenerateDataModal
-        isOpen={generateModalDisclosure.isOpen}
-        onClose={generateModalDisclosure.onClose}
-      />
-    </>
-  );
-};

app/src/components/datasets/DatasetHeaderButtons/GenerateDataModal.tsx

@@ -1,128 +0,0 @@
-import {
-  Modal,
-  ModalBody,
-  ModalCloseButton,
-  ModalContent,
-  ModalHeader,
-  ModalOverlay,
-  ModalFooter,
-  Text,
-  HStack,
-  VStack,
-  Icon,
-  NumberInput,
-  NumberInputField,
-  NumberInputStepper,
-  NumberIncrementStepper,
-  NumberDecrementStepper,
-  Button,
-} from "@chakra-ui/react";
-import { BsStars } from "react-icons/bs";
-import { useState } from "react";
-import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
-import { api } from "~/utils/api";
-import AutoResizeTextArea from "~/components/AutoResizeTextArea";
-
-export const GenerateDataModal = ({
-  isOpen,
-  onClose,
-}: {
-  isOpen: boolean;
-  onClose: () => void;
-}) => {
-  const utils = api.useContext();
-
-  const datasetId = useDataset().data?.id;
-
-  const [numToGenerate, setNumToGenerate] = useState<number>(20);
-  const [inputDescription, setInputDescription] = useState<string>(
-    "Each input should contain an email body. Half of the emails should contain event details, and the other half should not.",
-  );
-  const [outputDescription, setOutputDescription] = useState<string>(
-    `Each output should contain "true" or "false", where "true" indicates that the email contains event details.`,
-  );
-
-  const generateEntriesMutation = api.datasetEntries.autogenerateEntries.useMutation();
-
-  const [generateEntries, generateEntriesInProgress] = useHandledAsyncCallback(async () => {
-    if (!inputDescription || !outputDescription || !numToGenerate || !datasetId) return;
-    await generateEntriesMutation.mutateAsync({
-      datasetId,
-      inputDescription,
-      outputDescription,
-      numToGenerate,
-    });
-    await utils.datasetEntries.list.invalidate();
-    onClose();
-  }, [
-    generateEntriesMutation,
-    onClose,
-    inputDescription,
-    outputDescription,
-    numToGenerate,
-    datasetId,
-  ]);
-
-  return (
-    <Modal isOpen={isOpen} onClose={onClose} size={{ base: "xl", sm: "2xl", md: "3xl" }}>
-      <ModalOverlay />
-      <ModalContent w={1200}>
-        <ModalHeader>
-          <HStack>
-            <Icon as={BsStars} />
-            <Text>Generate Data</Text>
-          </HStack>
-        </ModalHeader>
-        <ModalCloseButton />
-        <ModalBody maxW="unset">
-          <VStack w="full" spacing={8} padding={8} alignItems="flex-start">
-            <VStack alignItems="flex-start" spacing={2}>
-              <Text fontWeight="bold">Number of Rows:</Text>
-              <NumberInput
-                step={5}
-                defaultValue={15}
-                min={0}
-                max={100}
-                onChange={(valueString) => setNumToGenerate(parseInt(valueString) || 0)}
-                value={numToGenerate}
-                w="24"
-              >
-                <NumberInputField />
-                <NumberInputStepper>
-                  <NumberIncrementStepper />
-                  <NumberDecrementStepper />
-                </NumberInputStepper>
-              </NumberInput>
-            </VStack>
-            <VStack alignItems="flex-start" w="full" spacing={2}>
-              <Text fontWeight="bold">Input Description:</Text>
-              <AutoResizeTextArea
-                value={inputDescription}
-                onChange={(e) => setInputDescription(e.target.value)}
-                placeholder="Each input should contain..."
-              />
-            </VStack>
-            <VStack alignItems="flex-start" w="full" spacing={2}>
-              <Text fontWeight="bold">Output Description (optional):</Text>
-              <AutoResizeTextArea
-                value={outputDescription}
-                onChange={(e) => setOutputDescription(e.target.value)}
-                placeholder="The output should contain..."
-              />
-            </VStack>
-          </VStack>
-        </ModalBody>
-        <ModalFooter>
-          <Button
-            colorScheme="blue"
-            isLoading={generateEntriesInProgress}
-            isDisabled={!numToGenerate || !inputDescription || !outputDescription}
-            onClick={generateEntries}
-          >
-            Generate
-          </Button>
-        </ModalFooter>
-      </ModalContent>
-    </Modal>
-  );
-};

app/src/components/datasets/DatasetsTable.tsx

@@ -0,0 +1,52 @@
+import { Card, Table, Thead, Tr, Th, Tbody, Td, VStack, Icon, Text } from "@chakra-ui/react";
+import { FaTable } from "react-icons/fa";
+import Link from "next/link";
+
+import dayjs from "~/utils/dayjs";
+import { useDatasets } from "~/utils/hooks";
+
+const DatasetsTable = ({}) => {
+  const { data } = useDatasets();
+
+  const datasets = data || [];
+
+  return (
+    <Card width="100%" overflowX="auto">
+      {datasets.length ? (
+        <Table>
+          <Thead>
+            <Tr>
+              <Th>Name</Th>
+              <Th>Created At</Th>
+              <Th>Size</Th>
+            </Tr>
+          </Thead>
+          <Tbody>
+            {datasets.map((dataset) => {
+              return (
+                <Tr key={dataset.id}>
+                  <Td>
+                    <Link href={{ pathname: "/datasets/[id]", query: { id: dataset.id } }}>
+                      <Text color="blue.600">{dataset.name}</Text>
+                    </Link>
+                  </Td>
+                  <Td>{dayjs(dataset.createdAt).format("MMMM D h:mm A")}</Td>
+                  <Td>{dataset._count.datasetEntries}</Td>
+                </Tr>
+              );
+            })}
+          </Tbody>
+        </Table>
+      ) : (
+        <VStack py={8}>
+          <Icon as={FaTable} boxSize={16} color="gray.300" />
+          <Text color="gray.400" fontSize="lg" fontWeight="bold">
+            No Datasets Found. Create your first dataset.
+          </Text>
+        </VStack>
+      )}
+    </Card>
+  );
+};
+
+export default DatasetsTable;

app/src/components/datasets/DeleteButton.tsx

@@ -0,0 +1,107 @@
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  useDisclosure,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import { BsTrash } from "react-icons/bs";
+
+import { useHandledAsyncCallback, useDataset } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import pluralize from "pluralize";
+
+const DeleteButton = () => {
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Delete"
+        icon={BsTrash}
+        isDisabled={selectedIds.size === 0}
+        requireBeta
+      />
+      <DeleteDatasetEntriesModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default DeleteButton;
+
+const DeleteDatasetEntriesModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const dataset = useDataset().data;
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+  const clearSelectedIds = useAppStore((s) => s.selectedDatasetEntries.clearSelectedIds);
+
+  const deleteRowsMutation = api.datasetEntries.delete.useMutation();
+
+  const utils = api.useContext();
+
+  const [deleteRows, deletionInProgress] = useHandledAsyncCallback(async () => {
+    if (!dataset?.id || !selectedIds.size) return;
+
+    // divide selectedIds into chunks of 15000 to reduce request size
+    const chunkSize = 15000;
+    const idsArray = Array.from(selectedIds);
+    for (let i = 0; i < idsArray.length; i += chunkSize) {
+      const response = await deleteRowsMutation.mutateAsync({
+        ids: idsArray.slice(i, i + chunkSize),
+      });
+
+      if (maybeReportError(response)) return;
+    }
+
+    await utils.datasetEntries.list.invalidate();
+    disclosure.onClose();
+    clearSelectedIds();
+  }, [deleteRowsMutation, dataset, selectedIds, utils]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={BsTrash} />
+            <Text>Delete Logs</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              Are you sure you want to delete the <b>{selectedIds.size}</b>{" "}
+              {pluralize("row", selectedIds.size)} rows you've selected?
+            </Text>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button colorScheme="red" onClick={deleteRows} isLoading={deletionInProgress} minW={24}>
+              Delete
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/datasets/DownloadButton.tsx

@@ -0,0 +1,182 @@
+import { useState, useEffect } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  Checkbox,
+  NumberInput,
+  NumberInputField,
+  NumberInputStepper,
+  NumberIncrementStepper,
+  NumberDecrementStepper,
+  Collapse,
+  Flex,
+  useDisclosure,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import { AiOutlineDownload } from "react-icons/ai";
+
+import { useHandledAsyncCallback, useDataset } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+import { FiChevronUp, FiChevronDown } from "react-icons/fi";
+import InfoCircle from "../InfoCircle";
+
+const ExportButton = () => {
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Download"
+        icon={AiOutlineDownload}
+        isDisabled={selectedIds.size === 0}
+        requireBeta
+      />
+      <ExportDatasetEntriesModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default ExportButton;
+
+const ExportDatasetEntriesModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const dataset = useDataset().data;
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+  const clearSelectedIds = useAppStore((s) => s.selectedDatasetEntries.clearSelectedIds);
+
+  const [testingSplit, setTestingSplit] = useState(10);
+  const [removeDuplicates, setRemoveDuplicates] = useState(false);
+  const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      setTestingSplit(10);
+      setRemoveDuplicates(false);
+    }
+  }, [disclosure.isOpen]);
+
+  const exportDataMutation = api.datasetEntries.export.useMutation();
+
+  const [exportData, exportInProgress] = useHandledAsyncCallback(async () => {
+    if (!dataset?.id || !selectedIds.size || !testingSplit) return;
+    const response = await exportDataMutation.mutateAsync({
+      datasetId: dataset.id,
+      datasetEntryIds: Array.from(selectedIds),
+      testingSplit,
+      removeDuplicates,
+    });
+
+    const dataUrl = `data:application/pdf;base64,${response}`;
+    const blob = await fetch(dataUrl).then((res) => res.blob());
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement("a");
+
+    a.href = url;
+    a.download = `data.zip`;
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+
+    disclosure.onClose();
+    clearSelectedIds();
+  }, [exportDataMutation, dataset, selectedIds, testingSplit, removeDuplicates]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={AiOutlineDownload} />
+            <Text>Export Logs</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              We'll export the <b>{selectedIds.size}</b> rows you have selected in the OpenAI
+              training format.
+            </Text>
+            <VStack alignItems="flex-start" spacing={4}>
+              <Flex
+                flexDir={{ base: "column", md: "row" }}
+                alignItems={{ base: "flex-start", md: "center" }}
+              >
+                <HStack w={48} alignItems="center" spacing={1}>
+                  <Text fontWeight="bold">Testing Split:</Text>
+                  <InfoCircle tooltipText="The percent of your logs that will be reserved for testing and saved in another file. Logs are split randomly." />
+                </HStack>
+                <HStack>
+                  <NumberInput
+                    defaultValue={10}
+                    onChange={(_, num) => setTestingSplit(num)}
+                    min={1}
+                    max={100}
+                    w={48}
+                  >
+                    <NumberInputField />
+                    <NumberInputStepper>
+                      <NumberIncrementStepper />
+                      <NumberDecrementStepper />
+                    </NumberInputStepper>
+                  </NumberInput>
+                </HStack>
+              </Flex>
+            </VStack>
+            <VStack alignItems="flex-start" spacing={0}>
+              <Button
+                variant="unstyled"
+                color="blue.600"
+                onClick={() => setShowAdvancedOptions(!showAdvancedOptions)}
+              >
+                <HStack>
+                  <Text>Advanced Options</Text>
+                  <Icon as={showAdvancedOptions ? FiChevronUp : FiChevronDown} />
+                </HStack>
+              </Button>
+              <Collapse in={showAdvancedOptions} unmountOnExit={true}>
+                <VStack align="stretch" pt={4}>
+                  <HStack>
+                    <Checkbox
+                      colorScheme="blue"
+                      isChecked={removeDuplicates}
+                      onChange={(e) => setRemoveDuplicates(e.target.checked)}
+                    >
+                      <Text>Remove duplicates</Text>
+                    </Checkbox>
+                    <InfoCircle tooltipText="To avoid overfitting and speed up training, automatically deduplicate logs with matching input and output." />
+                  </HStack>
+                </VStack>
+              </Collapse>
+            </VStack>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button colorScheme="blue" onClick={exportData} isLoading={exportInProgress} minW={24}>
+              Download
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/datasets/ExperimentButton.tsx

@@ -0,0 +1,21 @@
+import { RiFlaskLine } from "react-icons/ri";
+
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+
+const ExperimentButton = () => {
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+  return (
+    <ActionButton
+      onClick={() => {
+        console.log("experimenting with these ids", selectedIds);
+      }}
+      label="Experiment"
+      icon={RiFlaskLine}
+      isDisabled={selectedIds.size === 0}
+      requireBeta
+    />
+  );
+};
+
+export default ExperimentButton;

app/src/components/datasets/FileUploadsCard.tsx

@@ -0,0 +1,139 @@
+import { useState, useEffect } from "react";
+import { VStack, HStack, Button, Text, Progress, IconButton, Portal } from "@chakra-ui/react";
+import { BsX } from "react-icons/bs";
+
+import { type RouterOutputs, api } from "~/utils/api";
+import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
+import { formatFileSize } from "~/utils/utils";
+
+type FileUpload = RouterOutputs["datasets"]["listFileUploads"][0];
+
+const FileUploadsCard = () => {
+  const dataset = useDataset();
+  const [fileUploadsRefetchInterval, setFileUploadsRefetchInterval] = useState<number>(500);
+  const fileUploads = api.datasets.listFileUploads.useQuery(
+    { datasetId: dataset.data?.id as string },
+    { enabled: !!dataset.data?.id, refetchInterval: fileUploadsRefetchInterval },
+  );
+  useEffect(() => {
+    if (fileUploads?.data?.some((fu) => fu.status !== "COMPLETE" && fu.status !== "ERROR")) {
+      setFileUploadsRefetchInterval(500);
+    } else {
+      setFileUploadsRefetchInterval(15000);
+    }
+  }, [fileUploads]);
+
+  const utils = api.useContext();
+
+  const hideFileUploadsMutation = api.datasets.hideFileUploads.useMutation();
+  const [hideAllFileUploads] = useHandledAsyncCallback(async () => {
+    if (!fileUploads.data?.length) return;
+    await hideFileUploadsMutation.mutateAsync({
+      fileUploadIds: fileUploads.data.map((upload) => upload.id),
+    });
+    await utils.datasets.listFileUploads.invalidate();
+  }, [hideFileUploadsMutation, fileUploads.data, utils]);
+
+  if (!fileUploads.data?.length) return null;
+
+  return (
+    <Portal>
+      <VStack
+        w={72}
+        borderRadius={8}
+        position="fixed"
+        bottom={8}
+        right={8}
+        overflow="hidden"
+        borderWidth={1}
+        boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
+        minW={0}
+        bgColor="white"
+      >
+        <HStack p={4} w="full" bgColor="gray.200" justifyContent="space-between">
+          <Text fontWeight="bold">Uploads</Text>
+          <IconButton
+            aria-label="Close uploads"
+            as={BsX}
+            boxSize={6}
+            minW={0}
+            variant="ghost"
+            onClick={hideAllFileUploads}
+            cursor="pointer"
+          />
+        </HStack>
+        {fileUploads?.data?.map((upload) => <FileUploadRow key={upload.id} fileUpload={upload} />)}
+      </VStack>
+    </Portal>
+  );
+};
+
+export default FileUploadsCard;
+
+const FileUploadRow = ({ fileUpload }: { fileUpload: FileUpload }) => {
+  const { id, fileName, fileSize, progress, status, errorMessage } = fileUpload;
+
+  const utils = api.useContext();
+
+  const hideFileUploadsMutation = api.datasets.hideFileUploads.useMutation();
+  const [hideFileUpload, hidingInProgress] = useHandledAsyncCallback(async () => {
+    await hideFileUploadsMutation.mutateAsync({ fileUploadIds: [id] });
+    await utils.datasets.listFileUploads.invalidate();
+  }, [id, hideFileUploadsMutation, utils]);
+
+  useEffect(() => {
+    // Invalidate dataset entries list when upload is processed
+    if (status === "COMPLETE") void utils.datasetEntries.list.invalidate();
+  }, [status, utils]);
+
+  return (
+    <VStack w="full" alignItems="flex-start" p={4} borderBottomWidth={1}>
+      <HStack w="full" justifyContent="space-between" alignItems="flex-start">
+        <VStack alignItems="flex-start" spacing={0}>
+          <Text fontWeight="bold">{fileName}</Text>
+          <Text fontSize="xs">({formatFileSize(fileSize, 2)})</Text>
+        </VStack>
+        <Button
+          aria-label="Hide file upload"
+          minW={0}
+          variant="ghost"
+          isLoading={hidingInProgress}
+          onClick={hideFileUpload}
+          size="xs"
+        >
+          HIDE
+        </Button>
+      </HStack>
+
+      {errorMessage ? (
+        <Text alignSelf="center" pt={2}>
+          {errorMessage}
+        </Text>
+      ) : (
+        <>
+          <Text alignSelf="center" fontSize="xs">
+            {getStatusText(status)}
+          </Text>
+          <Progress w="full" value={progress} borderRadius={2} />
+        </>
+      )}
+    </VStack>
+  );
+};
+
+const getStatusText = (status: FileUpload["status"]) => {
+  switch (status) {
+    case "PENDING":
+      return "Pending";
+    case "DOWNLOADING":
+      return "Loading Data";
+    case "PROCESSING":
+      return "Processing";
+    case "SAVING":
+      return "Saving";
+    case "COMPLETE":
+      return "Complete";
+    case "ERROR":
+      return "Error";
+  }
+};

app/src/components/datasets/FineTuneButton.tsx

@@ -0,0 +1,161 @@
+import { useState, useEffect } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  useDisclosure,
+  type UseDisclosureReturn,
+  Input,
+} from "@chakra-ui/react";
+import { AiTwotoneThunderbolt } from "react-icons/ai";
+import humanId from "human-id";
+import { useRouter } from "next/router";
+
+import { useDataset, useDatasetEntries, useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import ActionButton from "../ActionButton";
+import InputDropdown from "../InputDropdown";
+// import { FiChevronDown } from "react-icons/fi";
+
+const SUPPORTED_BASE_MODELS = ["llama2-7b", "llama2-13b", "llama2-70b", "gpt-3.5-turbo"];
+
+const FineTuneButton = () => {
+  const datasetEntries = useDatasetEntries().data;
+
+  const numEntries = datasetEntries?.matchingEntryIds.length || 0;
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Fine Tune"
+        icon={AiTwotoneThunderbolt}
+        isDisabled={numEntries === 0}
+        requireBeta
+      />
+      <FineTuneModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default FineTuneButton;
+
+const FineTuneModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const dataset = useDataset().data;
+  const datasetEntries = useDatasetEntries().data;
+
+  const [selectedBaseModel, setSelectedBaseModel] = useState(SUPPORTED_BASE_MODELS[0]);
+  const [modelSlug, setModelSlug] = useState(humanId({ separator: "-", capitalize: false }));
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      setSelectedBaseModel(SUPPORTED_BASE_MODELS[0]);
+      setModelSlug(humanId({ separator: "-", capitalize: false }));
+    }
+  }, [disclosure.isOpen]);
+
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const createFineTuneMutation = api.fineTunes.create.useMutation();
+
+  const [createFineTune, creationInProgress] = useHandledAsyncCallback(async () => {
+    if (!modelSlug || !selectedBaseModel || !dataset) return;
+    await createFineTuneMutation.mutateAsync({
+      slug: modelSlug,
+      baseModel: selectedBaseModel,
+      datasetId: dataset.id,
+    });
+
+    await utils.fineTunes.list.invalidate();
+    await router.push({ pathname: "/fine-tunes" });
+    disclosure.onClose();
+  }, [createFineTuneMutation, modelSlug, selectedBaseModel]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={AiTwotoneThunderbolt} />
+            <Text>Fine Tune</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              We'll train on <b>{datasetEntries?.trainingCount}</b> and test on{" "}
+              <b>{datasetEntries?.testingCount}</b> entries in this dataset.
+            </Text>
+            <VStack>
+              <HStack spacing={2} w="full">
+                <Text fontWeight="bold" w={36}>
+                  Model ID:
+                </Text>
+                <Input
+                  value={modelSlug}
+                  onChange={(e) => setModelSlug(e.target.value)}
+                  w={48}
+                  placeholder="unique-id"
+                  onKeyDown={(e) => {
+                    // If the user types anything other than a-z, A-Z, or 0-9, replace it with -
+                    if (!/[a-zA-Z0-9]/.test(e.key)) {
+                      e.preventDefault();
+                      setModelSlug((s) => s && `${s}-`);
+                    }
+                  }}
+                />
+              </HStack>
+              <HStack spacing={2}>
+                <Text fontWeight="bold" w={36}>
+                  Base model:
+                </Text>
+                <InputDropdown
+                  options={SUPPORTED_BASE_MODELS}
+                  selectedOption={selectedBaseModel}
+                  onSelect={(option) => setSelectedBaseModel(option)}
+                  inputGroupProps={{ w: 48 }}
+                />
+              </HStack>
+            </VStack>
+            {/* <Button variant="unstyled" color="blue.600">
+              <HStack>
+                <Text>Advanced Options</Text>
+                <Icon as={FiChevronDown} />
+              </HStack>
+            </Button> */}
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="blue"
+              onClick={createFineTune}
+              isLoading={creationInProgress}
+              minW={24}
+              isDisabled={!modelSlug}
+            >
+              Start Training
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/datasets/TableRow.tsx

@@ -1,13 +0,0 @@
-import { Td, Tr } from "@chakra-ui/react";
-import { type DatasetEntry } from "@prisma/client";
-
-const TableRow = ({ entry }: { entry: DatasetEntry }) => {
-  return (
-    <Tr key={entry.id}>
-      <Td>{entry.input}</Td>
-      <Td>{entry.output}</Td>
-    </Tr>
-  );
-};
-
-export default TableRow;

app/src/components/datasets/UploadDataButton.tsx

@@ -0,0 +1,288 @@
+import { useState, useEffect, useRef, useCallback } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  Box,
+  useDisclosure,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import pluralize from "pluralize";
+import { AiOutlineCloudUpload, AiOutlineFile } from "react-icons/ai";
+
+import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import ActionButton from "../ActionButton";
+import { validateTrainingRows, type TrainingRow, parseJSONL } from "./validateTrainingRows";
+import { uploadDatasetEntryFile } from "~/utils/azure/website";
+import { formatFileSize } from "~/utils/utils";
+
+const UploadDataButton = () => {
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Upload Data"
+        icon={AiOutlineCloudUpload}
+        iconBoxSize={4}
+        requireBeta
+      />
+      <UploadDataModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default UploadDataButton;
+
+const UploadDataModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const dataset = useDataset().data;
+
+  const [validationError, setValidationError] = useState<string | null>(null);
+  const [trainingRows, setTrainingRows] = useState<TrainingRow[] | null>(null);
+  const [file, setFile] = useState<File | null>(null);
+
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileDrop = (e: React.DragEvent<HTMLDivElement>) => {
+    e.preventDefault();
+    const files = e.dataTransfer.files;
+    if (files.length > 0) {
+      processFile(files[0] as File);
+    }
+  };
+
+  const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const files = e.target.files;
+    if (files && files.length > 0) {
+      processFile(files[0] as File);
+    }
+  };
+
+  const processFile = (file: File) => {
+    setFile(file);
+
+    // skip reading if file is larger than 10MB
+    if (file.size > 10000000) {
+      setTrainingRows(null);
+      return;
+    }
+
+    const reader = new FileReader();
+    reader.onload = (e: ProgressEvent<FileReader>) => {
+      const content = e.target?.result as string;
+      // Process the content, e.g., set to state
+      let parsedJSONL;
+      try {
+        parsedJSONL = parseJSONL(content) as TrainingRow[];
+        const validationError = validateTrainingRows(parsedJSONL);
+        if (validationError) {
+          setValidationError(validationError);
+          setTrainingRows(null);
+          return;
+        }
+        setTrainingRows(parsedJSONL);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      } catch (e: any) {
+        setValidationError("Unable to parse JSONL file: " + (e.message as string));
+        setTrainingRows(null);
+        return;
+      }
+    };
+    reader.readAsText(file);
+  };
+
+  const resetState = useCallback(() => {
+    setValidationError(null);
+    setTrainingRows(null);
+    setFile(null);
+  }, [setValidationError, setTrainingRows, setFile]);
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      resetState();
+    }
+  }, [disclosure.isOpen, resetState]);
+
+  const triggerFileDownloadMutation = api.datasets.triggerFileDownload.useMutation();
+
+  const utils = api.useContext();
+
+  const [sendJSONL, sendingInProgress] = useHandledAsyncCallback(async () => {
+    if (!dataset || !file) return;
+
+    const blobName = await uploadDatasetEntryFile(file);
+
+    await triggerFileDownloadMutation.mutateAsync({
+      datasetId: dataset.id,
+      blobName,
+      fileName: file.name,
+      fileSize: file.size,
+    });
+
+    await utils.datasets.listFileUploads.invalidate();
+
+    disclosure.onClose();
+  }, [dataset, trainingRows, triggerFileDownloadMutation, file, utils]);
+
+  return (
+    <Modal
+      size={{ base: "xl", md: "2xl" }}
+      closeOnOverlayClick={false}
+      closeOnEsc={false}
+      {...disclosure}
+    >
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Text>Upload Training Logs</Text>
+          </HStack>
+        </ModalHeader>
+        {!sendingInProgress && <ModalCloseButton />}
+        <ModalBody maxW="unset" p={8}>
+          <Box w="full" aspectRatio={1.5}>
+            {validationError && (
+              <VStack w="full" h="full" justifyContent="center" spacing={8}>
+                <Icon as={AiOutlineFile} boxSize={24} color="gray.300" />
+                <VStack w="full">
+                  <Text fontSize={32} color="gray.500" fontWeight="bold">
+                    Error
+                  </Text>
+                  <Text color="gray.500">{validationError}</Text>
+                </VStack>
+                <Text
+                  as="span"
+                  textDecor="underline"
+                  color="gray.500"
+                  _hover={{ color: "orange.400" }}
+                  cursor="pointer"
+                  onClick={resetState}
+                >
+                  Try again
+                </Text>
+              </VStack>
+            )}
+            {!validationError && !file && (
+              <VStack
+                w="full"
+                h="full"
+                stroke="gray.300"
+                justifyContent="center"
+                borderRadius={8}
+                sx={{
+                  "background-image": `url("data:image/svg+xml,%3csvg width='100%25' height='100%25' xmlns='http://www.w3.org/2000/svg'%3e%3crect x='2%25' y='2%25' width='96%25' height='96%25' fill='none' stroke='%23eee' stroke-width='4' stroke-dasharray='6%2c 14' stroke-dashoffset='0' stroke-linecap='square' rx='8' ry='8'/%3e%3c/svg%3e")`,
+                }}
+                onDragOver={(e) => e.preventDefault()}
+                onDrop={handleFileDrop}
+              >
+                <JsonFileIcon />
+                <Icon as={AiOutlineCloudUpload} boxSize={24} color="gray.300" />
+
+                <Text fontSize={32} color="gray.500" fontWeight="bold">
+                  Drag & Drop
+                </Text>
+                <Text color="gray.500">
+                  your .jsonl file here, or{" "}
+                  <input
+                    type="file"
+                    ref={fileInputRef}
+                    onChange={handleFileChange}
+                    style={{ display: "none" }}
+                    accept=".jsonl"
+                  />
+                  <Text
+                    as="span"
+                    textDecor="underline"
+                    _hover={{ color: "orange.400" }}
+                    cursor="pointer"
+                    onClick={() => fileInputRef.current?.click()}
+                  >
+                    browse
+                  </Text>
+                </Text>
+              </VStack>
+            )}
+            {!validationError && file && (
+              <VStack w="full" h="full" justifyContent="center" spacing={8}>
+                <JsonFileIcon />
+                <VStack w="full">
+                  {trainingRows ? (
+                    <>
+                      <Text fontSize={32} color="gray.500" fontWeight="bold">
+                        Success
+                      </Text>
+                      <Text color="gray.500">
+                        We'll upload <b>{trainingRows.length}</b>{" "}
+                        {pluralize("row", trainingRows.length)} into <b>{dataset?.name}</b>.{" "}
+                      </Text>
+                    </>
+                  ) : (
+                    <>
+                      <Text fontSize={32} color="gray.500" fontWeight="bold">
+                        {file.name}
+                      </Text>
+                      <Text color="gray.500">{formatFileSize(file.size)}</Text>
+                    </>
+                  )}
+                </VStack>
+                {!sendingInProgress && (
+                  <Text
+                    as="span"
+                    textDecor="underline"
+                    color="gray.500"
+                    _hover={{ color: "orange.400" }}
+                    cursor="pointer"
+                    onClick={resetState}
+                  >
+                    Change file
+                  </Text>
+                )}
+              </VStack>
+            )}
+          </Box>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button
+              colorScheme="gray"
+              isDisabled={sendingInProgress}
+              onClick={disclosure.onClose}
+              minW={24}
+            >
+              Cancel
+            </Button>
+            <Button
+              colorScheme="orange"
+              onClick={sendJSONL}
+              isLoading={sendingInProgress}
+              minW={24}
+              isDisabled={!file || !!validationError}
+            >
+              Upload
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};
+
+const JsonFileIcon = () => (
+  <Box position="relative" display="flex" alignItems="center" justifyContent="center">
+    <Icon as={AiOutlineFile} boxSize={24} color="gray.300" />
+    <Text position="absolute" color="orange.400" fontWeight="bold" fontSize={12} pt={4}>
+      JSONL
+    </Text>
+  </Box>
+);

app/src/components/datasets/validateTrainingRows.ts

@@ -0,0 +1,71 @@
+import { type CreateChatCompletionRequestMessage } from "openai/resources/chat";
+
+export type TrainingRow = {
+  input: CreateChatCompletionRequestMessage[];
+  output?: CreateChatCompletionRequestMessage;
+};
+
+export const parseJSONL = (jsonlString: string): unknown[] => {
+  const lines = jsonlString.trim().split("\n");
+
+  let lineNumber = 0;
+  const parsedLines = [];
+
+  try {
+    for (const line of lines) {
+      lineNumber++;
+      parsedLines.push(JSON.parse(line));
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  } catch (e: any) {
+    throw new Error(`Error parsing line ${lineNumber}: ${e.message as string}`);
+  }
+  return parsedLines;
+};
+
+export const validateTrainingRows = (rows: unknown): string | null => {
+  if (!Array.isArray(rows)) return "training data is not an array";
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i] as TrainingRow;
+    let errorMessage: string | null = null;
+    try {
+      errorMessage = validateTrainingRow(row);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (error: any) {
+      errorMessage = error.message;
+    }
+    if (errorMessage) return `row ${i + 1}: ${errorMessage}`;
+  }
+
+  return null;
+};
+
+const validateTrainingRow = (row: TrainingRow): string | null => {
+  if (!row) return "empty row";
+  if (!row.input) return "missing input";
+
+  //   Validate input
+  if (!Array.isArray(row.input)) return "input is not an array";
+  if ((row.input as unknown[]).some((x) => typeof x !== "object"))
+    return "input contains invalid item";
+  if (row.input.some((x) => !x)) return "input contains empty item";
+  if (row.input.some((x) => !x.content && !x.function_call))
+    return "input contains item with no content or function_call";
+  if (row.input.some((x) => x.function_call && !x.function_call.arguments))
+    return "input contains item with function_call but no arguments";
+  if (row.input.some((x) => x.function_call && !x.function_call.name))
+    return "input contains item with function_call but no name";
+
+  //   Validate output
+  if (row.output) {
+    if (typeof row.output !== "object") return "output is not an object";
+    if (!row.output.content && !row.output.function_call)
+      return "output contains no content or function_call";
+    if (row.output.function_call && !row.output.function_call.arguments)
+      return "output contains function_call but no arguments";
+    if (row.output.function_call && !row.output.function_call.name)
+      return "output contains function_call but no name";
+  }
+
+  return null;
+};

app/src/components/experiments/DeleteExperimentDialog.tsx

@@ -1,36 +1,43 @@
+import { useRef } from "react";
 import {
-  Button,
+  type UseDisclosureReturn,
   AlertDialog,
+  AlertDialogOverlay,
+  AlertDialogContent,
+  AlertDialogHeader,
   AlertDialogBody,
   AlertDialogFooter,
-  AlertDialogHeader,
-  AlertDialogContent,
-  AlertDialogOverlay,
+  Button,
 } from "@chakra-ui/react";
-
-import { useRouter } from "next/router";
-import { useRef } from "react";
 import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 
-export const DeleteDialog = ({ onClose }: { onClose: () => void }) => {
-  const experiment = useExperiment();
-  const deleteMutation = api.experiments.delete.useMutation();
-  const utils = api.useContext();
-  const router = useRouter();
+import { useHandledAsyncCallback } from "~/utils/hooks";
 
+const DeleteExperimentDialog = ({
+  experimentId,
+  onDelete,
+  disclosure,
+}: {
+  experimentId?: string;
+  onDelete?: () => void;
+  disclosure: UseDisclosureReturn;
+}) => {
   const cancelRef = useRef<HTMLButtonElement>(null);
 
-  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id) return;
-    await deleteMutation.mutateAsync({ id: experiment.data.id });
+  const mutation = api.experiments.delete.useMutation();
+  const utils = api.useContext();
+
+  const [onDeleteConfirm, deletionInProgress] = useHandledAsyncCallback(async () => {
+    if (!experimentId) return;
+    await mutation.mutateAsync({ id: experimentId });
     await utils.experiments.list.invalidate();
-    await router.push({ pathname: "/experiments" });
-    onClose();
-  }, [deleteMutation, experiment.data?.id, router]);
+    onDelete?.();
+
+    disclosure.onClose();
+  }, [mutation, experimentId, disclosure.onClose]);
 
   return (
-    <AlertDialog isOpen leastDestructiveRef={cancelRef} onClose={onClose}>
+    <AlertDialog leastDestructiveRef={cancelRef} {...disclosure}>
       <AlertDialogOverlay>
         <AlertDialogContent>
           <AlertDialogHeader fontSize="lg" fontWeight="bold">
@@ -43,10 +50,15 @@ export const DeleteDialog = ({ onClose }: { onClose: () => void }) => {
           </AlertDialogBody>
 
           <AlertDialogFooter>
-            <Button ref={cancelRef} onClick={onClose}>
+            <Button ref={cancelRef} onClick={disclosure.onClose}>
               Cancel
             </Button>
-            <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
+            <Button
+              colorScheme="red"
+              isLoading={deletionInProgress}
+              onClick={onDeleteConfirm}
+              ml={3}
+            >
               Delete
             </Button>
           </AlertDialogFooter>
@@ -55,3 +67,5 @@ export const DeleteDialog = ({ onClose }: { onClose: () => void }) => {
     </AlertDialog>
   );
 };
+
+export default DeleteExperimentDialog;

app/src/components/experiments/ExperimentCard.tsx

@@ -1,3 +1,4 @@
+import { type MouseEvent, useState } from "react";
 import {
   HStack,
   Icon,
@@ -8,17 +9,29 @@ import {
   AspectRatio,
   SkeletonText,
   Card,
+  useDisclosure,
+  Box,
+  Menu,
+  MenuButton,
+  MenuList,
+  MenuItem,
+  IconButton,
+  useToast,
 } from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
-import { formatTimePast } from "~/utils/dayjs";
 import Link from "next/link";
 import { useRouter } from "next/router";
-import { BsPlusSquare } from "react-icons/bs";
-import { RouterOutputs, api } from "~/utils/api";
+import { BsPlusSquare, BsThreeDotsVertical, BsLink45Deg, BsTrash } from "react-icons/bs";
+
+import { formatTimePast } from "~/utils/dayjs";
+import { type RouterOutputs, api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
 import { useAppStore } from "~/state/store";
+import DeleteExperimentDialog from "./DeleteExperimentDialog";
 
 export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["list"][0] }) => {
+  const [isMenuHovered, setIsMenuHovered] = useState(false);
+
   return (
     <Card
       w="full"
@@ -27,7 +40,7 @@ export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["lis
       p={4}
       bg="white"
       borderRadius={4}
-      _hover={{ bg: "gray.100" }}
+      _hover={{ bg: isMenuHovered ? undefined : "gray.100" }}
       transition="background 0.2s"
       aspectRatio={1.2}
     >
@@ -38,9 +51,17 @@ export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["lis
         href={{ pathname: "/experiments/[experimentSlug]", query: { experimentSlug: exp.slug } }}
         justify="space-between"
       >
-        <HStack w="full" color="gray.700" justify="center">
-          <Icon as={RiFlaskLine} boxSize={4} />
-          <Text fontWeight="bold">{exp.label}</Text>
+        <HStack w="full" justify="space-between" spacing={0}>
+          <Box w={6} />
+          <HStack color="gray.700" justify="center">
+            <Icon as={RiFlaskLine} boxSize={4} />
+            <Text fontWeight="bold">{exp.label}</Text>
+          </HStack>
+          <CardMenu
+            experimentId={exp.id}
+            experimentSlug={exp.slug}
+            setIsMenuHovered={setIsMenuHovered}
+          />
         </HStack>
         <HStack h="full" spacing={4} flex={1} align="center">
           <CountLabel label="Variants" count={exp.promptVariantCount} />
@@ -57,6 +78,75 @@ export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["lis
   );
 };
 
+const CardMenu = ({
+  experimentId,
+  experimentSlug,
+  setIsMenuHovered,
+}: {
+  experimentId: string;
+  experimentSlug: string;
+  setIsMenuHovered: (isHovered: boolean) => void;
+}) => {
+  const deleteDisclosure = useDisclosure();
+  const menuDisclosure = useDisclosure();
+  const toast = useToast();
+  const [copyShareLink] = useHandledAsyncCallback(
+    async (e: MouseEvent<HTMLButtonElement>) => {
+      if (typeof window === "undefined") return;
+      e.preventDefault();
+      e.stopPropagation();
+      const shareLink = `${window.location.origin}/experiments/${experimentSlug}`;
+      await navigator.clipboard.writeText(shareLink);
+      toast({
+        title: "Share link copied to clipboard",
+        status: "success",
+        duration: 2000,
+        isClosable: true,
+      });
+      menuDisclosure.onClose();
+    },
+    [toast, menuDisclosure.onClose, experimentSlug],
+  );
+  return (
+    <>
+      <Menu isLazy {...menuDisclosure}>
+        <MenuButton
+          as={IconButton}
+          aria-label="Options"
+          icon={<BsThreeDotsVertical />}
+          variant="ghost"
+          onClick={(e) => {
+            e.preventDefault();
+            e.stopPropagation();
+            menuDisclosure.onOpen();
+          }}
+          onMouseEnter={() => setIsMenuHovered(true)}
+          onMouseLeave={() => setIsMenuHovered(false)}
+          boxSize={6}
+          minW={0}
+        />
+        <MenuList>
+          <MenuItem icon={<Icon as={BsLink45Deg} boxSize={5} />} onClick={copyShareLink}>
+            Copy Link
+          </MenuItem>
+          <MenuItem
+            icon={<Icon as={BsTrash} boxSize={5} />}
+            onClick={(e) => {
+              e.preventDefault();
+              e.stopPropagation();
+              deleteDisclosure.onOpen();
+            }}
+            color="red.500"
+          >
+            Delete
+          </MenuItem>
+        </MenuList>
+      </Menu>
+      <DeleteExperimentDialog experimentId={experimentId} disclosure={deleteDisclosure} />
+    </>
+  );
+};
+
 const CountLabel = ({ label, count }: { label: string; count: number }) => {
   return (
     <VStack alignItems="center" flex={1}>
@@ -98,9 +188,7 @@ export const NewExperimentCard = () => {
     >
       <VStack align="center" justify="center" w="full" h="full" p={4} onClick={createExperiment}>
         <Icon as={isLoading ? Spinner : BsPlusSquare} boxSize={8} />
-        <Text display={{ base: "none", md: "block" }} ml={2}>
-          New Experiment
-        </Text>
+        <Text ml={2}>New Experiment</Text>
       </VStack>
     </Card>
   );

app/src/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons.tsx

@@ -3,17 +3,14 @@ import { useOnForkButtonPressed } from "./useOnForkButtonPressed";
 import { useExperiment } from "~/utils/hooks";
 import { BsGearFill } from "react-icons/bs";
 import { TbGitFork } from "react-icons/tb";
-import { useAppStore } from "~/state/store";
 
-export const ExperimentHeaderButtons = () => {
+export const ExperimentHeaderButtons = ({ openDrawer }: { openDrawer: () => void }) => {
   const experiment = useExperiment();
 
   const canModify = experiment.data?.access.canModify ?? false;
 
   const { onForkButtonPressed, isForking } = useOnForkButtonPressed();
 
-  const openDrawer = useAppStore((s) => s.openDrawer);
-
   if (experiment.isLoading) return null;
 
   return (

app/src/components/experiments/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -0,0 +1,39 @@
+import { Button, Icon, useDisclosure, Text } from "@chakra-ui/react";
+import { useRouter } from "next/router";
+import { BsTrash } from "react-icons/bs";
+
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import DeleteExperimentDialog from "../DeleteExperimentDialog";
+
+export const DeleteButton = ({ closeDrawer }: { closeDrawer: () => void }) => {
+  const experiment = useExperiment();
+  const router = useRouter();
+
+  const disclosure = useDisclosure();
+
+  const [onDelete] = useHandledAsyncCallback(async () => {
+    await router.push({ pathname: "/experiments" });
+    closeDrawer();
+  }, [router, closeDrawer]);
+
+  return (
+    <>
+      <Button
+        size="sm"
+        variant="ghost"
+        colorScheme="red"
+        fontWeight="normal"
+        onClick={disclosure.onOpen}
+      >
+        <Icon as={BsTrash} boxSize={4} />
+        <Text ml={2}>Delete Experiment</Text>
+      </Button>
+
+      <DeleteExperimentDialog
+        experimentId={experiment.data?.id}
+        onDelete={onDelete}
+        disclosure={disclosure}
+      />
+    </>
+  );
+};

app/src/components/experiments/ExperimentSettingsDrawer/EditEvaluations.tsx

@@ -19,7 +19,7 @@ import { useCallback, useState } from "react";
 import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-import AutoResizeTextArea from "../AutoResizeTextArea";
+import AutoResizeTextArea from "~/components/AutoResizeTextArea";
 
 type EvalValues = Pick<Evaluation, "label" | "value" | "evalType">;
 

app/src/components/experiments/ExperimentSettingsDrawer/EditScenarioVars.tsx

@@ -5,7 +5,7 @@ import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
 import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
-import { FloatingLabelInput } from "./FloatingLabelInput";
+import { FloatingLabelInput } from "~/components/OutputsTable/FloatingLabelInput";
 
 export const ScenarioVar = ({
   variable,

app/src/components/experiments/ExperimentSettingsDrawer/ExperimentSettingsDrawer.tsx

@@ -7,18 +7,19 @@ import {
   DrawerOverlay,
   Heading,
   VStack,
+  type UseDisclosureReturn,
 } from "@chakra-ui/react";
-import EditScenarioVars from "../OutputsTable/EditScenarioVars";
-import EditEvaluations from "../OutputsTable/EditEvaluations";
-import { useAppStore } from "~/state/store";
+import EditScenarioVars from "./EditScenarioVars";
+import EditEvaluations from "./EditEvaluations";
 import { DeleteButton } from "./DeleteButton";
 
-export default function ExperimentSettingsDrawer() {
-  const isOpen = useAppStore((state) => state.drawerOpen);
-  const closeDrawer = useAppStore((state) => state.closeDrawer);
-
+export default function ExperimentSettingsDrawer({
+  disclosure,
+}: {
+  disclosure: UseDisclosureReturn;
+}) {
   return (
-    <Drawer isOpen={isOpen} placement="right" onClose={closeDrawer} size="md">
+    <Drawer placement="right" size="md" {...disclosure}>
       <DrawerOverlay />
       <DrawerContent>
         <DrawerCloseButton />
@@ -31,7 +32,7 @@ export default function ExperimentSettingsDrawer() {
               <EditScenarioVars />
               <EditEvaluations />
             </VStack>
-            <DeleteButton />
+            <DeleteButton closeDrawer={disclosure.onClose} />
           </VStack>
         </DrawerBody>
       </DrawerContent>

app/src/components/fineTunes/FineTunesTable.tsx

@@ -0,0 +1,65 @@
+import { Card, Table, Thead, Tr, Th, Tbody, Td, VStack, Icon, Text } from "@chakra-ui/react";
+import { FaTable } from "react-icons/fa";
+import { type FineTuneStatus } from "@prisma/client";
+
+import dayjs from "~/utils/dayjs";
+import { useFineTunes } from "~/utils/hooks";
+
+const FineTunesTable = ({}) => {
+  const { data } = useFineTunes();
+
+  const fineTunes = data?.fineTunes || [];
+
+  return (
+    <Card width="100%" overflowX="auto">
+      {fineTunes.length ? (
+        <Table>
+          <Thead>
+            <Tr>
+              <Th>ID</Th>
+              <Th>Created At</Th>
+              <Th>Base Model</Th>
+              <Th>Dataset Size</Th>
+              <Th>Status</Th>
+            </Tr>
+          </Thead>
+          <Tbody>
+            {fineTunes.map((fineTune) => {
+              return (
+                <Tr key={fineTune.id}>
+                  <Td>{fineTune.slug}</Td>
+                  <Td>{dayjs(fineTune.createdAt).format("MMMM D h:mm A")}</Td>
+                  <Td>{fineTune.baseModel}</Td>
+                  <Td>{fineTune.dataset._count.datasetEntries}</Td>
+                  <Td fontSize="sm" fontWeight="bold">
+                    <Text color={getStatusColor(fineTune.status)}>{fineTune.status}</Text>
+                  </Td>
+                </Tr>
+              );
+            })}
+          </Tbody>
+        </Table>
+      ) : (
+        <VStack py={8}>
+          <Icon as={FaTable} boxSize={16} color="gray.300" />
+          <Text color="gray.400" fontSize="lg" fontWeight="bold">
+            No Fine Tunes Found
+          </Text>
+        </VStack>
+      )}
+    </Card>
+  );
+};
+
+export default FineTunesTable;
+
+const getStatusColor = (status: FineTuneStatus) => {
+  switch (status) {
+    case "DEPLOYED":
+      return "green.500";
+    case "ERROR":
+      return "red.500";
+    default:
+      return "yellow.500";
+  }
+};

app/src/components/nav/AppShell.tsx

@@ -13,14 +13,19 @@ import {
 } from "@chakra-ui/react";
 import Head from "next/head";
 import Link from "next/link";
+import { useRouter } from "next/router";
 import { BsGearFill, BsGithub, BsPersonCircle } from "react-icons/bs";
 import { IoStatsChartOutline } from "react-icons/io5";
-import { RiHome3Line, RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
+import { RiHome3Line, RiFlaskLine } from "react-icons/ri";
+import { AiOutlineThunderbolt, AiOutlineDatabase } from "react-icons/ai";
+import { FaReadme } from "react-icons/fa";
 import { signIn, useSession } from "next-auth/react";
-import { env } from "~/env.mjs";
+
 import ProjectMenu from "./ProjectMenu";
 import NavSidebarOption from "./NavSidebarOption";
 import IconLink from "./IconLink";
+import { BetaModal } from "../BetaModal";
+import { useAppStore } from "~/state/store";
 
 const Divider = () => <Box h="1px" bgColor="gray.300" w="full" />;
 
@@ -71,21 +76,11 @@ const NavSidebar = () => {
             <ProjectMenu />
             <Divider />
 
-            {env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS && (
-              <>
-                <IconLink icon={RiHome3Line} label="Dashboard" href="/dashboard" beta />
-                <IconLink
-                  icon={IoStatsChartOutline}
-                  label="Request Logs"
-                  href="/request-logs"
-                  beta
-                />
-              </>
-            )}
+            <IconLink icon={RiHome3Line} label="Dashboard" href="/dashboard" />
+            <IconLink icon={IoStatsChartOutline} label="Request Logs" href="/request-logs" />
+            <IconLink icon={AiOutlineDatabase} label="Datasets" href="/datasets" beta />
+            <IconLink icon={AiOutlineThunderbolt} label="Fine Tunes" href="/fine-tunes" beta />
             <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
-            {env.NEXT_PUBLIC_SHOW_DATA && (
-              <IconLink icon={RiDatabase2Line} label="Data" href="/data" />
-            )}
             <VStack w="full" alignItems="flex-start" spacing={0} pt={8}>
               <Text
                 pl={2}
@@ -105,7 +100,7 @@ const NavSidebar = () => {
           <NavSidebarOption>
             <HStack
               w="full"
-              p={4}
+              p={{ base: 2, md: 4 }}
               as={ChakraLink}
               justifyContent="start"
               onClick={() => {
@@ -120,7 +115,22 @@ const NavSidebar = () => {
           </NavSidebarOption>
         )}
       </VStack>
-
+      <HStack
+        w="full"
+        px={{ base: 3, md: 4 }}
+        py={{ base: 0, md: 1 }}
+        as={ChakraLink}
+        justifyContent="start"
+        href="https://docs.openpipe.ai"
+        target="_blank"
+        color="gray.500"
+        spacing={1}
+      >
+        <Icon as={FaReadme} boxSize={4} mr={2} />
+        <Text fontWeight="bold" fontSize="sm" display={{ base: "none", md: "flex" }}>
+          Open Documentation
+        </Text>
+      </HStack>
       <Divider />
       <VStack spacing={0} align="center">
         <ChakraLink
@@ -141,12 +151,15 @@ export default function AppShell({
   children,
   title,
   requireAuth,
+  requireBeta,
 }: {
   children: React.ReactNode;
   title?: string;
   requireAuth?: boolean;
+  requireBeta?: boolean;
 }) {
   const [vh, setVh] = useState("100vh"); // Default height to prevent flicker on initial render
+  const router = useRouter();
 
   useEffect(() => {
     const setHeight = () => {
@@ -174,15 +187,21 @@ export default function AppShell({
     }
   }, [requireAuth, user, authLoading]);
 
+  const flags = useAppStore((s) => s.featureFlags.featureFlags);
+  const flagsLoaded = useAppStore((s) => s.featureFlags.flagsLoaded);
+
   return (
-    <Flex h={vh} w="100vw">
-      <Head>
-        <title>{title ? `${title} | OpenPipe` : "OpenPipe"}</title>
-      </Head>
-      <NavSidebar />
-      <Box h="100%" flex={1} overflowY="auto" bgColor="gray.50">
-        {children}
-      </Box>
-    </Flex>
+    <>
+      <Flex h={vh} w="100vw">
+        <Head>
+          <title>{title ? `${title} | OpenPipe` : "OpenPipe"}</title>
+        </Head>
+        <NavSidebar />
+        <Box h="100%" flex={1} overflowY="auto" bgColor="gray.50">
+          {children}
+        </Box>
+      </Flex>
+      <BetaModal isOpen={!!requireBeta && flagsLoaded && !flags.betaAccess} onClose={router.back} />
+    </>
   );
 }

app/src/components/nav/ProjectMenu.tsx

@@ -14,6 +14,7 @@ import {
   Link as ChakraLink,
   Image,
   Box,
+  Portal,
 } from "@chakra-ui/react";
 import { useEffect } from "react";
 import Link from "next/link";
@@ -56,6 +57,7 @@ export default function ProjectMenu() {
     await utils.projects.list.invalidate();
     setSelectedProjectId(newProj.id);
     await router.push({ pathname: "/project/settings" });
+    popover.onClose();
   }, [createMutation, router]);
 
   const user = useSession().data;
@@ -109,64 +111,66 @@ export default function ProjectMenu() {
             </HStack>
           </NavSidebarOption>
         </PopoverTrigger>
-        <PopoverContent
-          _focusVisible={{ outline: "unset" }}
-          w={220}
-          ml={{ base: 2, md: 0 }}
-          boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
-          fontSize="sm"
-        >
-          <VStack alignItems="flex-start" spacing={1} py={1}>
-            <Text px={3} py={2}>
-              {user?.user.email}
-            </Text>
-            <Divider />
-            <Text alignSelf="flex-start" fontWeight="bold" px={3} pt={2}>
-              Your Projects
-            </Text>
-            <VStack spacing={0} w="full" px={1}>
-              {projects?.map((proj) => (
-                <ProjectOption
-                  key={proj.id}
-                  proj={proj}
-                  isActive={proj.id === selectedProjectId}
-                  onClose={popover.onClose}
-                />
-              ))}
-              <HStack
-                as={Button}
-                variant="ghost"
-                colorScheme="blue"
-                color="blue.400"
-                fontSize="sm"
-                justifyContent="flex-start"
-                onClick={createProject}
-                w="full"
-                borderRadius={4}
-                spacing={0}
-              >
-                <Text>Add project</Text>
-                <Icon as={isLoading ? Spinner : BsPlus} boxSize={4} strokeWidth={0.5} />
-              </HStack>
-            </VStack>
+        <Portal>
+          <PopoverContent
+            _focusVisible={{ outline: "unset" }}
+            w={220}
+            ml={{ base: 2, md: 0 }}
+            boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
+            fontSize="sm"
+          >
+            <VStack alignItems="flex-start" spacing={1} py={1}>
+              <Text px={3} py={2}>
+                {user?.user.email}
+              </Text>
+              <Divider />
+              <Text alignSelf="flex-start" fontWeight="bold" px={3} pt={2}>
+                Your Projects
+              </Text>
+              <VStack spacing={0} w="full" px={1}>
+                {projects?.map((proj) => (
+                  <ProjectOption
+                    key={proj.id}
+                    proj={proj}
+                    isActive={proj.id === selectedProjectId}
+                    onClose={popover.onClose}
+                  />
+                ))}
+                <HStack
+                  as={Button}
+                  variant="ghost"
+                  colorScheme="blue"
+                  color="blue.400"
+                  fontSize="sm"
+                  justifyContent="flex-start"
+                  onClick={createProject}
+                  w="full"
+                  borderRadius={4}
+                  spacing={0}
+                >
+                  <Text>Add project</Text>
+                  <Icon as={isLoading ? Spinner : BsPlus} boxSize={4} strokeWidth={0.5} />
+                </HStack>
+              </VStack>
 
-            <Divider />
-            <VStack w="full" px={1}>
-              <ChakraLink
-                onClick={() => {
-                  signOut().catch(console.error);
-                }}
-                _hover={{ bgColor: "gray.200", textDecoration: "none" }}
-                w="full"
-                py={2}
-                px={2}
-                borderRadius={4}
-              >
-                <Text>Sign out</Text>
-              </ChakraLink>
+              <Divider />
+              <VStack w="full" px={1}>
+                <ChakraLink
+                  onClick={() => {
+                    signOut().catch(console.error);
+                  }}
+                  _hover={{ bgColor: "gray.200", textDecoration: "none" }}
+                  w="full"
+                  py={2}
+                  px={2}
+                  borderRadius={4}
+                >
+                  <Text>Sign out</Text>
+                </ChakraLink>
+              </VStack>
             </VStack>
-          </VStack>
-        </PopoverContent>
+          </PopoverContent>
+        </Portal>
       </Popover>
     </VStack>
   );

app/src/components/nav/UserMenu.tsx

@@ -23,50 +23,48 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
   );
 
   return (
-    <>
-      <Popover placement="right">
-        <PopoverTrigger>
-          <NavSidebarOption>
-            <HStack
-              // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
-              py={2}
-              px={1}
-              spacing={3}
-              {...rest}
-            >
-              {profileImage}
-              <VStack spacing={0} align="start" flex={1} flexShrink={1}>
-                <Text fontWeight="bold" fontSize="sm">
-                  {user.user.name}
-                </Text>
-                <Text color="gray.500" fontSize="xs">
-                  {/* {user.user.email} */}
-                </Text>
-              </VStack>
-              <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
-            </HStack>
-          </NavSidebarOption>
-        </PopoverTrigger>
-        <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} minW={48} w="full">
-          <VStack align="stretch" spacing={0}>
-            {/* sign out */}
-            <HStack
-              as={Link}
-              onClick={() => {
-                signOut().catch(console.error);
-              }}
-              px={4}
-              py={2}
-              spacing={4}
-              color="gray.500"
-              fontSize="sm"
-            >
-              <Icon as={BsBoxArrowRight} boxSize={6} />
-              <Text>Sign out</Text>
-            </HStack>
-          </VStack>
-        </PopoverContent>
-      </Popover>
-    </>
+    <Popover placement="right">
+      <PopoverTrigger>
+        <NavSidebarOption>
+          <HStack
+            // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
+            py={2}
+            px={1}
+            spacing={3}
+            {...rest}
+          >
+            {profileImage}
+            <VStack spacing={0} align="start" flex={1} flexShrink={1}>
+              <Text fontWeight="bold" fontSize="sm">
+                {user.user.name}
+              </Text>
+              <Text color="gray.500" fontSize="xs">
+                {/* {user.user.email} */}
+              </Text>
+            </VStack>
+            <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+          </HStack>
+        </NavSidebarOption>
+      </PopoverTrigger>
+      <PopoverContent _focusVisible={{ outline: "unset" }} ml={-1} minW={48} w="full">
+        <VStack align="stretch" spacing={0}>
+          {/* sign out */}
+          <HStack
+            as={Link}
+            onClick={() => {
+              signOut().catch(console.error);
+            }}
+            px={4}
+            py={2}
+            spacing={4}
+            color="gray.500"
+            fontSize="sm"
+          >
+            <Icon as={BsBoxArrowRight} boxSize={6} />
+            <Text>Sign out</Text>
+          </HStack>
+        </VStack>
+      </PopoverContent>
+    </Popover>
   );
 }

app/src/components/requestLogs/ActionButton.tsx

@@ -1,30 +0,0 @@
-import { Button, HStack, type ButtonProps, Icon, Text } from "@chakra-ui/react";
-import { type IconType } from "react-icons";
-
-const ActionButton = ({
-  icon,
-  label,
-  ...buttonProps
-}: { icon: IconType; label: string } & ButtonProps) => {
-  return (
-    <Button
-      colorScheme="blue"
-      color="black"
-      bgColor="white"
-      borderColor="gray.300"
-      borderRadius={4}
-      variant="outline"
-      size="sm"
-      fontSize="sm"
-      fontWeight="normal"
-      {...buttonProps}
-    >
-      <HStack spacing={1}>
-        {icon && <Icon as={icon} />}
-        <Text>{label}</Text>
-      </HStack>
-    </Button>
-  );
-};
-
-export default ActionButton;

app/src/components/requestLogs/AddToDatasetButton.tsx

@@ -0,0 +1,194 @@
+import { useState, useEffect, useMemo } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  Flex,
+  Input,
+  useDisclosure,
+  type UseDisclosureReturn,
+  Checkbox,
+} from "@chakra-ui/react";
+import { FiPlusSquare } from "react-icons/fi";
+
+import { useDatasets, useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+import InputDropdown from "../InputDropdown";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { useRouter } from "next/router";
+
+const AddToDatasetButton = () => {
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Add to Dataset"
+        icon={FiPlusSquare}
+        isDisabled={selectedLogIds.size === 0}
+        requireBeta
+      />
+      <AddToDatasetModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default AddToDatasetButton;
+
+const AddToDatasetModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const clearSelectedLogIds = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
+  const router = useRouter();
+
+  const datasets = useDatasets().data;
+
+  const existingDatasetOptions = useMemo(
+    () =>
+      datasets?.length
+        ? datasets.map((d) => ({ label: d.name, id: d.id }))
+        : [{ label: "", id: "" }],
+    [datasets],
+  );
+
+  const [selectedDatasetOption, setSelectedDatasetOption] = useState(existingDatasetOptions?.[0]);
+  const [newDatasetName, setNewDatasetName] = useState("");
+  const [createNewDataset, setCreateNewDataset] = useState(false);
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      setSelectedDatasetOption(existingDatasetOptions?.[0]);
+      setCreateNewDataset(!existingDatasetOptions[0]?.id);
+    }
+  }, [disclosure.isOpen, existingDatasetOptions]);
+
+  const createDatasetEntriesMutation = api.datasetEntries.create.useMutation();
+
+  const [addToDataset, addingInProgress] = useHandledAsyncCallback(async () => {
+    if (
+      !selectedProjectId ||
+      !selectedLogIds.size ||
+      !(createNewDataset ? newDatasetName : selectedDatasetOption?.id)
+    )
+      return;
+    const datasetParams = createNewDataset
+      ? { newDatasetParams: { projectId: selectedProjectId, name: newDatasetName } }
+      : { datasetId: selectedDatasetOption?.id };
+    const response = await createDatasetEntriesMutation.mutateAsync({
+      loggedCallIds: Array.from(selectedLogIds),
+      ...datasetParams,
+    });
+
+    if (maybeReportError(response)) return;
+
+    const datasetId = response.payload;
+
+    await router.push({ pathname: "/datasets/[id]", query: { id: datasetId } });
+
+    disclosure.onClose();
+    clearSelectedLogIds();
+  }, [
+    selectedProjectId,
+    selectedLogIds,
+    createNewDataset,
+    selectedDatasetOption?.id,
+    newDatasetName,
+    router,
+  ]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={FiPlusSquare} />
+            <Text>Add to Dataset</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              We'll add the <b>{selectedLogIds.size}</b> logs you have selected to the dataset you
+              choose.
+            </Text>
+            <VStack alignItems="flex-start" spacing={4}>
+              {existingDatasetOptions?.length && selectedDatasetOption && (
+                <Flex
+                  flexDir={{ base: "column", md: "row" }}
+                  alignItems={{ base: "flex-start", md: "center" }}
+                >
+                  <Text fontWeight="bold" w={48}>
+                    Dataset:
+                  </Text>
+                  <InputDropdown
+                    options={existingDatasetOptions}
+                    selectedOption={selectedDatasetOption}
+                    getDisplayLabel={(option) => option.label}
+                    onSelect={(option) => setSelectedDatasetOption(option)}
+                    inputGroupProps={{ w: 48 }}
+                    isDisabled={createNewDataset}
+                  />
+                  <Checkbox
+                    isChecked={createNewDataset}
+                    onChange={(e) => setCreateNewDataset(e.target.checked)}
+                    paddingLeft={4}
+                    isDisabled={!existingDatasetOptions[0]?.id}
+                  >
+                    <Text>Create New Dataset</Text>
+                  </Checkbox>
+                </Flex>
+              )}
+
+              {createNewDataset && (
+                <Flex
+                  flexDir={{ base: "column", md: "row" }}
+                  alignItems={{ base: "flex-start", md: "center" }}
+                >
+                  <Text w={48} fontWeight="bold">
+                    Dataset Name:
+                  </Text>
+                  <Input
+                    w={48}
+                    value={newDatasetName}
+                    onChange={(e) => setNewDatasetName(e.target.value)}
+                  />
+                </Flex>
+              )}
+            </VStack>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="blue"
+              onClick={addToDataset}
+              isLoading={addingInProgress}
+              minW={24}
+            >
+              Add
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/requestLogs/ColumnVisiblityDropdown.tsx

@@ -0,0 +1,117 @@
+import {
+  Icon,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  VStack,
+  HStack,
+  Button,
+  Text,
+  useDisclosure,
+  Box,
+} from "@chakra-ui/react";
+import { BiCheck } from "react-icons/bi";
+import { BsToggles } from "react-icons/bs";
+import { useMemo } from "react";
+
+import { useIsClientRehydrated, useTagNames } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
+import { StaticColumnKeys } from "~/state/columnVisiblitySlice";
+import ActionButton from "../ActionButton";
+
+const ColumnVisiblityDropdown = () => {
+  const tagNames = useTagNames().data;
+
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const toggleColumnVisibility = useAppStore((s) => s.columnVisibility.toggleColumnVisibility);
+  const totalColumns = Object.keys(StaticColumnKeys).length + (tagNames?.length ?? 0);
+
+  const popover = useDisclosure();
+
+  const columnVisiblityOptions = useMemo(() => {
+    const options: { label: string; key: string }[] = [
+      {
+        label: "Sent At",
+        key: StaticColumnKeys.SENT_AT,
+      },
+      {
+        label: "Model",
+        key: StaticColumnKeys.MODEL,
+      },
+      {
+        label: "Duration",
+        key: StaticColumnKeys.DURATION,
+      },
+      {
+        label: "Input Tokens",
+        key: StaticColumnKeys.INPUT_TOKENS,
+      },
+      {
+        label: "Output Tokens",
+        key: StaticColumnKeys.OUTPUT_TOKENS,
+      },
+      {
+        label: "Status Code",
+        key: StaticColumnKeys.STATUS_CODE,
+      },
+    ];
+    for (const tagName of tagNames ?? []) {
+      options.push({
+        label: tagName,
+        key: tagName,
+      });
+    }
+    return options;
+  }, [tagNames]);
+
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
+  return (
+    <Popover
+      placement="bottom-start"
+      isOpen={popover.isOpen}
+      onOpen={popover.onOpen}
+      onClose={popover.onClose}
+    >
+      <PopoverTrigger>
+        <Box>
+          <ActionButton
+            label={`Columns (${visibleColumns.size}/${totalColumns})`}
+            icon={BsToggles}
+          />
+        </Box>
+      </PopoverTrigger>
+      <PopoverContent boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);" minW={0} w="auto">
+        <VStack spacing={0} maxH={400} overflowY="auto">
+          {columnVisiblityOptions?.map((option, index) => (
+            <HStack
+              key={index}
+              as={Button}
+              onClick={() => toggleColumnVisibility(option.key)}
+              w="full"
+              minH={10}
+              variant="ghost"
+              justifyContent="space-between"
+              fontWeight="semibold"
+              borderRadius={0}
+              colorScheme="blue"
+              color="black"
+              fontSize="sm"
+              borderBottomWidth={1}
+            >
+              <Text mr={16}>{option.label}</Text>
+              <Box w={5}>
+                {visibleColumns.has(option.key) && (
+                  <Icon as={BiCheck} color="blue.500" boxSize={5} />
+                )}
+              </Box>
+            </HStack>
+          ))}
+        </VStack>
+      </PopoverContent>
+    </Popover>
+  );
+};
+
+export default ColumnVisiblityDropdown;

app/src/components/requestLogs/ExportButton.tsx

@@ -0,0 +1,211 @@
+import { useState, useEffect } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  Checkbox,
+  NumberInput,
+  NumberInputField,
+  NumberInputStepper,
+  NumberIncrementStepper,
+  NumberDecrementStepper,
+  Collapse,
+  Flex,
+  useDisclosure,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import { BiExport } from "react-icons/bi";
+
+import { useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+import InputDropdown from "../InputDropdown";
+import { FiChevronUp, FiChevronDown } from "react-icons/fi";
+import InfoCircle from "../InfoCircle";
+
+const SUPPORTED_EXPORT_FORMATS = ["alpaca-finetune", "openai-fine-tune", "unformatted"];
+
+const ExportButton = () => {
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Export"
+        icon={BiExport}
+        isDisabled={selectedLogIds.size === 0}
+        requireBeta
+      />
+      <ExportLogsModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default ExportButton;
+
+const ExportLogsModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const clearSelectedLogIds = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
+
+  const [selectedExportFormat, setSelectedExportFormat] = useState(SUPPORTED_EXPORT_FORMATS[0]);
+  const [testingSplit, setTestingSplit] = useState(10);
+  const [removeDuplicates, setRemoveDuplicates] = useState(true);
+  const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      setSelectedExportFormat(SUPPORTED_EXPORT_FORMATS[0]);
+      setTestingSplit(10);
+      setRemoveDuplicates(true);
+    }
+  }, [disclosure.isOpen]);
+
+  const exportLogsMutation = api.loggedCalls.export.useMutation();
+
+  const [exportLogs, exportInProgress] = useHandledAsyncCallback(async () => {
+    if (!selectedProjectId || !selectedLogIds.size || !testingSplit || !selectedExportFormat)
+      return;
+    const response = await exportLogsMutation.mutateAsync({
+      projectId: selectedProjectId,
+      loggedCallIds: Array.from(selectedLogIds),
+      testingSplit,
+      selectedExportFormat,
+      removeDuplicates,
+    });
+
+    const dataUrl = `data:application/pdf;base64,${response}`;
+    const blob = await fetch(dataUrl).then((res) => res.blob());
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement("a");
+
+    a.href = url;
+    a.download = `data.zip`;
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+
+    disclosure.onClose();
+    clearSelectedLogIds();
+  }, [
+    exportLogsMutation,
+    selectedProjectId,
+    selectedLogIds,
+    testingSplit,
+    selectedExportFormat,
+    removeDuplicates,
+  ]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={BiExport} />
+            <Text>Export Logs</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              We'll export the <b>{selectedLogIds.size}</b> logs you have selected in the format of
+              your choice.
+            </Text>
+            <VStack alignItems="flex-start" spacing={4}>
+              <Flex
+                flexDir={{ base: "column", md: "row" }}
+                alignItems={{ base: "flex-start", md: "center" }}
+              >
+                <HStack w={48} alignItems="center" spacing={1}>
+                  <Text fontWeight="bold">Format:</Text>
+                  <InfoCircle tooltipText="Format logs for for fine tuning or export them without formatting." />
+                </HStack>
+                <InputDropdown
+                  options={SUPPORTED_EXPORT_FORMATS}
+                  selectedOption={selectedExportFormat}
+                  onSelect={(option) => setSelectedExportFormat(option)}
+                  inputGroupProps={{ w: 48 }}
+                />
+              </Flex>
+              <Flex
+                flexDir={{ base: "column", md: "row" }}
+                alignItems={{ base: "flex-start", md: "center" }}
+              >
+                <HStack w={48} alignItems="center" spacing={1}>
+                  <Text fontWeight="bold">Testing Split:</Text>
+                  <InfoCircle tooltipText="The percent of your logs that will be reserved for testing and saved in another file. Logs are split randomly." />
+                </HStack>
+                <HStack>
+                  <NumberInput
+                    defaultValue={10}
+                    onChange={(_, num) => setTestingSplit(num)}
+                    min={1}
+                    max={100}
+                    w={48}
+                  >
+                    <NumberInputField />
+                    <NumberInputStepper>
+                      <NumberIncrementStepper />
+                      <NumberDecrementStepper />
+                    </NumberInputStepper>
+                  </NumberInput>
+                </HStack>
+              </Flex>
+            </VStack>
+            <VStack alignItems="flex-start" spacing={0}>
+              <Button
+                variant="unstyled"
+                color="blue.600"
+                onClick={() => setShowAdvancedOptions(!showAdvancedOptions)}
+              >
+                <HStack>
+                  <Text>Advanced Options</Text>
+                  <Icon as={showAdvancedOptions ? FiChevronUp : FiChevronDown} />
+                </HStack>
+              </Button>
+              <Collapse in={showAdvancedOptions} unmountOnExit={true}>
+                <VStack align="stretch" pt={4}>
+                  <HStack>
+                    <Checkbox
+                      colorScheme="blue"
+                      isChecked={removeDuplicates}
+                      onChange={(e) => setRemoveDuplicates(e.target.checked)}
+                    >
+                      <Text>Remove duplicates</Text>
+                    </Checkbox>
+                    <InfoCircle tooltipText="To avoid overfitting and speed up training, automatically deduplicate logs with matching input and output." />
+                  </HStack>
+                </VStack>
+              </Collapse>
+            </VStack>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button colorScheme="blue" onClick={exportLogs} isLoading={exportInProgress} minW={24}>
+              Export
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/requestLogs/LoggedCallsTable.tsx

@@ -1,7 +1,7 @@
 import { Card, Table, Tbody } from "@chakra-ui/react";
 import { useState } from "react";
 import { useLoggedCalls } from "~/utils/hooks";
-import { TableHeader, TableRow } from "./TableRow";
+import { TableHeader, TableRow, EmptyTableRow } from "./TableRow";
 
 export default function LoggedCallsTable() {
   const [expandedRow, setExpandedRow] = useState<string | null>(null);
@@ -10,25 +10,29 @@ export default function LoggedCallsTable() {
   return (
     <Card width="100%" overflowX="auto">
       <Table>
-        <TableHeader showCheckbox />
+        <TableHeader showOptions />
         <Tbody>
-          {loggedCalls?.calls?.map((loggedCall) => {
-            return (
-              <TableRow
-                key={loggedCall.id}
-                loggedCall={loggedCall}
-                isExpanded={loggedCall.id === expandedRow}
-                onToggle={() => {
-                  if (loggedCall.id === expandedRow) {
-                    setExpandedRow(null);
-                  } else {
-                    setExpandedRow(loggedCall.id);
-                  }
-                }}
-                showCheckbox
-              />
-            );
-          })}
+          {loggedCalls?.calls.length ? (
+            loggedCalls?.calls?.map((loggedCall) => {
+              return (
+                <TableRow
+                  key={loggedCall.id}
+                  loggedCall={loggedCall}
+                  isExpanded={loggedCall.id === expandedRow}
+                  onToggle={() => {
+                    if (loggedCall.id === expandedRow) {
+                      setExpandedRow(null);
+                    } else {
+                      setExpandedRow(loggedCall.id);
+                    }
+                  }}
+                  showOptions
+                />
+              );
+            })
+          ) : (
+            <EmptyTableRow />
+          )}
         </Tbody>
       </Table>
     </Card>

app/src/components/requestLogs/TableRow.tsx

@@ -9,26 +9,22 @@ import {
   Collapse,
   HStack,
   VStack,
-  Button,
-  ButtonGroup,
   Text,
   Checkbox,
+  Link as ChakraLink,
 } from "@chakra-ui/react";
-import dayjs from "dayjs";
-import relativeTime from "dayjs/plugin/relativeTime";
-import Link from "next/link";
 
+import dayjs from "~/utils/dayjs";
 import { type RouterOutputs } from "~/utils/api";
-import { FormattedJson } from "./FormattedJson";
+import { FormattedJson } from "../FormattedJson";
 import { useAppStore } from "~/state/store";
-import { useLoggedCalls, useTagNames } from "~/utils/hooks";
+import { useIsClientRehydrated, useLoggedCalls, useTagNames } from "~/utils/hooks";
 import { useMemo } from "react";
-
-dayjs.extend(relativeTime);
+import { StaticColumnKeys } from "~/state/columnVisiblitySlice";
 
 type LoggedCall = RouterOutputs["loggedCalls"]["list"]["calls"][0];
 
-export const TableHeader = ({ showCheckbox }: { showCheckbox?: boolean }) => {
+export const TableHeader = ({ showOptions }: { showOptions?: boolean }) => {
   const matchingLogIds = useLoggedCalls().data?.matchingLogIds;
   const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
   const addAll = useAppStore((s) => s.selectedLogs.addSelectedLogIds);
@@ -38,10 +34,14 @@ export const TableHeader = ({ showCheckbox }: { showCheckbox?: boolean }) => {
     return matchingLogIds.every((id) => selectedLogIds.has(id));
   }, [selectedLogIds, matchingLogIds]);
   const tagNames = useTagNames().data;
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
   return (
     <Thead>
       <Tr>
-        {showCheckbox && (
+        {showOptions && (
           <Th pr={0}>
             <HStack minW={16}>
               <Checkbox
@@ -57,13 +57,19 @@ export const TableHeader = ({ showCheckbox }: { showCheckbox?: boolean }) => {
             </HStack>
           </Th>
         )}
-        <Th>Sent At</Th>
-        <Th>Model</Th>
-        {tagNames?.map((tagName) => <Th key={tagName}>{tagName}</Th>)}
-        <Th isNumeric>Duration</Th>
-        <Th isNumeric>Input tokens</Th>
-        <Th isNumeric>Output tokens</Th>
-        <Th isNumeric>Status</Th>
+        {visibleColumns.has(StaticColumnKeys.SENT_AT) && <Th>Sent At</Th>}
+        {visibleColumns.has(StaticColumnKeys.MODEL) && <Th>Model</Th>}
+        {tagNames
+          ?.filter((tagName) => visibleColumns.has(tagName))
+          .map((tagName) => (
+            <Th key={tagName} textTransform={"none"}>
+              {tagName}
+            </Th>
+          ))}
+        {visibleColumns.has(StaticColumnKeys.DURATION) && <Th isNumeric>Duration</Th>}
+        {visibleColumns.has(StaticColumnKeys.INPUT_TOKENS) && <Th isNumeric>Input tokens</Th>}
+        {visibleColumns.has(StaticColumnKeys.OUTPUT_TOKENS) && <Th isNumeric>Output tokens</Th>}
+        {visibleColumns.has(StaticColumnKeys.STATUS_CODE) && <Th isNumeric>Status</Th>}
       </Tr>
     </Thead>
   );
@@ -73,12 +79,12 @@ export const TableRow = ({
   loggedCall,
   isExpanded,
   onToggle,
-  showCheckbox,
+  showOptions,
 }: {
   loggedCall: LoggedCall;
   isExpanded: boolean;
   onToggle: () => void;
-  showCheckbox?: boolean;
+  showOptions?: boolean;
 }) => {
   const isError = loggedCall.modelResponse?.statusCode !== 200;
   const requestedAt = dayjs(loggedCall.requestedAt).format("MMMM D h:mm A");
@@ -88,6 +94,14 @@ export const TableRow = ({
   const toggleChecked = useAppStore((s) => s.selectedLogs.toggleSelectedLogId);
 
   const tagNames = useTagNames().data;
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+
+  const visibleTagNames = useMemo(() => {
+    return tagNames?.filter((tagName) => visibleColumns.has(tagName)) ?? [];
+  }, [tagNames, visibleColumns]);
+
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
 
   return (
     <>
@@ -100,71 +114,116 @@ export const TableRow = ({
         }}
         fontSize="sm"
       >
-        {showCheckbox && (
+        {showOptions && (
           <Td>
             <Checkbox isChecked={isChecked} onChange={() => toggleChecked(loggedCall.id)} />
           </Td>
         )}
-        <Td>
-          <Tooltip label={fullTime} placement="top">
-            <Box whiteSpace="nowrap" minW="120px">
-              {requestedAt}
-            </Box>
-          </Tooltip>
-        </Td>
-        <Td>
-          <HStack justifyContent="flex-start">
-            <Text
-              colorScheme="purple"
-              color="purple.500"
-              borderColor="purple.500"
-              px={1}
-              borderRadius={4}
-              borderWidth={1}
-              fontSize="xs"
-              whiteSpace="nowrap"
-            >
-              {loggedCall.model}
-            </Text>
-          </HStack>
-        </Td>
-        {tagNames?.map((tagName) => <Td key={tagName}>{loggedCall.tags[tagName]}</Td>)}
-        <Td isNumeric>
-          {loggedCall.cacheHit ? (
-            <Text color="gray.500">Cached</Text>
-          ) : (
-            ((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2) + "s"
-          )}
-        </Td>
-        <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
-        <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
-        <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
-          {loggedCall.modelResponse?.statusCode ?? "No response"}
-        </Td>
+        {visibleColumns.has(StaticColumnKeys.SENT_AT) && (
+          <Td>
+            <Tooltip label={fullTime} placement="top">
+              <Box whiteSpace="nowrap" minW="120px">
+                {requestedAt}
+              </Box>
+            </Tooltip>
+          </Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.MODEL) && (
+          <Td>
+            <HStack justifyContent="flex-start">
+              <Text
+                colorScheme="purple"
+                color="purple.500"
+                borderColor="purple.500"
+                px={1}
+                borderRadius={4}
+                borderWidth={1}
+                fontSize="xs"
+                whiteSpace="nowrap"
+              >
+                {loggedCall.model}
+              </Text>
+            </HStack>
+          </Td>
+        )}
+        {visibleTagNames.map((tagName) => (
+          <Td key={tagName}>{loggedCall.tags[tagName]}</Td>
+        ))}
+        {visibleColumns.has(StaticColumnKeys.DURATION) && (
+          <Td isNumeric>
+            {loggedCall.cacheHit ? (
+              <Text color="gray.500">Cached</Text>
+            ) : (
+              ((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2) + "s"
+            )}
+          </Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.INPUT_TOKENS) && (
+          <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.OUTPUT_TOKENS) && (
+          <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
+        )}
+        {visibleColumns.has(StaticColumnKeys.STATUS_CODE) && (
+          <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
+            {loggedCall.modelResponse?.statusCode ?? "No response"}
+          </Td>
+        )}
       </Tr>
       <Tr>
-        <Td colSpan={8} p={0}>
+        <Td colSpan={visibleColumns.size + 1} w="full" p={0}>
           <Collapse in={isExpanded} unmountOnExit={true}>
-            <VStack p={4} align="stretch">
-              <HStack align="stretch">
-                <VStack flex={1} align="stretch">
-                  <Heading size="sm">Input</Heading>
-                  <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
-                </VStack>
-                <VStack flex={1} align="stretch">
-                  <Heading size="sm">Output</Heading>
-                  <FormattedJson json={loggedCall.modelResponse?.respPayload} />
-                </VStack>
-              </HStack>
-              <ButtonGroup alignSelf="flex-end">
-                <Button as={Link} colorScheme="blue" href={{ pathname: "/experiments" }}>
-                  Experiments
-                </Button>
-              </ButtonGroup>
-            </VStack>
+            <HStack align="stretch" p={4}>
+              <VStack flex={1} align="stretch">
+                <Heading size="sm">Input</Heading>
+                <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
+              </VStack>
+              <VStack flex={1} align="stretch">
+                <Heading size="sm">Output</Heading>
+                <FormattedJson json={loggedCall.modelResponse?.respPayload} />
+              </VStack>
+            </HStack>
           </Collapse>
         </Td>
       </Tr>
     </>
   );
 };
+
+export const EmptyTableRow = ({ filtersApplied = true }: { filtersApplied?: boolean }) => {
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const filters = useAppStore((state) => state.logFilters.filters);
+  const { isLoading } = useLoggedCalls();
+
+  if (isLoading) return null;
+
+  if (filters.length && filtersApplied) {
+    return (
+      <Tr>
+        <Td w="full" colSpan={visibleColumns.size + 1}>
+          <Text color="gray.500" textAlign="center" w="full" p={4}>
+            No matching request logs found. Try removing some filters.
+          </Text>
+        </Td>
+      </Tr>
+    );
+  }
+
+  return (
+    <Tr>
+      <Td w="full" colSpan={visibleColumns.size + 1}>
+        <Text color="gray.500" textAlign="center" w="full" p={4}>
+          This project has no request logs. Learn how to add request logs to your project in our{" "}
+          <ChakraLink
+            href="https://docs.openpipe.ai/getting-started/quick-start"
+            target="_blank"
+            color="blue.600"
+          >
+            Quick Start
+          </ChakraLink>{" "}
+          guide.
+        </Text>
+      </Td>
+    </Tr>
+  );
+};

app/src/env.mjs

@@ -26,10 +26,17 @@ export const env = createEnv({
     SMTP_PORT: z.string().default("placeholder"),
     SMTP_LOGIN: z.string().default("placeholder"),
     SMTP_PASSWORD: z.string().default("placeholder"),
+    AZURE_STORAGE_ACCOUNT_NAME: z.string().default("placeholder"),
+    AZURE_STORAGE_ACCOUNT_KEY: z.string().default("placeholder"),
+    AZURE_STORAGE_CONTAINER_NAME: z.string().default("placeholder"),
     WORKER_CONCURRENCY: z
       .string()
       .default("10")
       .transform((val) => parseInt(val)),
+    WORKER_MAX_POOL_SIZE: z
+      .string()
+      .default("10")
+      .transform((val) => parseInt(val)),
   },
 
   /**
@@ -42,8 +49,6 @@ export const env = createEnv({
     NEXT_PUBLIC_SOCKET_URL: z.string().url().default("http://localhost:3318"),
     NEXT_PUBLIC_HOST: z.string().url().default("http://localhost:3000"),
     NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
-    NEXT_PUBLIC_SHOW_DATA: z.string().optional(),
-    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: z.string().optional(),
   },
 
   /**
@@ -58,7 +63,6 @@ export const env = createEnv({
     NEXT_PUBLIC_POSTHOG_KEY: process.env.NEXT_PUBLIC_POSTHOG_KEY,
     NEXT_PUBLIC_SOCKET_URL: process.env.NEXT_PUBLIC_SOCKET_URL,
     NEXT_PUBLIC_HOST: process.env.NEXT_PUBLIC_HOST,
-    NEXT_PUBLIC_SHOW_DATA: process.env.NEXT_PUBLIC_SHOW_DATA,
     GITHUB_CLIENT_ID: process.env.GITHUB_CLIENT_ID,
     GITHUB_CLIENT_SECRET: process.env.GITHUB_CLIENT_SECRET,
     REPLICATE_API_TOKEN: process.env.REPLICATE_API_TOKEN,
@@ -66,13 +70,16 @@ export const env = createEnv({
     NEXT_PUBLIC_SENTRY_DSN: process.env.NEXT_PUBLIC_SENTRY_DSN,
     SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
     OPENPIPE_API_KEY: process.env.OPENPIPE_API_KEY,
-    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: process.env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS,
     SENDER_EMAIL: process.env.SENDER_EMAIL,
     SMTP_HOST: process.env.SMTP_HOST,
     SMTP_PORT: process.env.SMTP_PORT,
     SMTP_LOGIN: process.env.SMTP_LOGIN,
     SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+    AZURE_STORAGE_ACCOUNT_NAME: process.env.AZURE_STORAGE_ACCOUNT_NAME,
+    AZURE_STORAGE_ACCOUNT_KEY: process.env.AZURE_STORAGE_ACCOUNT_KEY,
+    AZURE_STORAGE_CONTAINER_NAME: process.env.AZURE_STORAGE_CONTAINER_NAME,
     WORKER_CONCURRENCY: process.env.WORKER_CONCURRENCY,
+    WORKER_MAX_POOL_SIZE: process.env.WORKER_MAX_POOL_SIZE,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/modelProviders/openai-ChatCompletion/getCompletion.ts

@@ -2,7 +2,7 @@
 import { isArray, isString } from "lodash-es";
 import { APIError } from "openai";
 import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
-import mergeChunks from "openpipe/src/openai/mergeChunks";
+import mergeChunks from "openpipe/openai/mergeChunks";
 import { openai } from "~/server/utils/openai";
 import { type CompletionResponse } from "../types";
 
@@ -14,9 +14,18 @@ export async function getCompletion(
   let finalCompletion: ChatCompletion | null = null;
 
   try {
-    if (onStream) {
+    if (onStream && !input.function_call) {
       const resp = await openai.chat.completions.create(
-        { ...input, stream: true },
+        {
+          ...input,
+          stream: true,
+          openpipe: {
+            tags: {
+              prompt_id: "getCompletion",
+              stream: "true",
+            },
+          },
+        },
         {
           maxRetries: 0,
         },
@@ -34,7 +43,16 @@ export async function getCompletion(
       }
     } else {
       const resp = await openai.chat.completions.create(
-        { ...input, stream: false },
+        {
+          ...input,
+          stream: false,
+          openpipe: {
+            tags: {
+              prompt_id: "getCompletion",
+              stream: "false",
+            },
+          },
+        },
         {
           maxRetries: 0,
         },

app/src/modelProviders/openai-ChatCompletion/index.ts

@@ -42,24 +42,21 @@ const modelProvider: OpenaiChatModelProvider = {
   canStream: true,
   getCompletion,
   getUsage: (input, output) => {
-    if (output.choices.length === 0) return null;
-
     const model = modelProvider.getModel(input);
     if (!model) return null;
 
     let inputTokens: number;
     let outputTokens: number;
 
-    if (output.usage) {
+    if (output?.usage) {
       inputTokens = output.usage.prompt_tokens;
       outputTokens = output.usage.completion_tokens;
     } else {
       try {
         inputTokens = countOpenAIChatTokens(model, input.messages);
-        outputTokens = countOpenAIChatTokens(
-          model,
-          output.choices.map((c) => c.message).filter(truthyFilter),
-        );
+        outputTokens = output
+          ? countOpenAIChatTokens(model, output.choices.map((c) => c.message).filter(truthyFilter))
+          : 0;
       } catch (err) {
         inputTokens = 0;
         outputTokens = 0;

app/src/modelProviders/openpipe-chat/getCompletion.ts

@@ -17,10 +17,23 @@ const modelEndpoints: Record<OpenpipeChatInput["model"], string> = {
   "NousResearch/Nous-Hermes-llama-2-7b": "https://ua1bpc6kv3dgge-8000.proxy.runpod.net/v1",
 };
 
+const CUSTOM_MODELS_ENABLED = false;
+
 export async function getCompletion(
   input: OpenpipeChatInput,
   onStream: ((partialOutput: OpenpipeChatOutput) => void) | null,
 ): Promise<CompletionResponse<OpenpipeChatOutput>> {
+  // Temporarily disable these models because of GPU constraints
+
+  if (!CUSTOM_MODELS_ENABLED) {
+    return {
+      type: "error",
+      message:
+        "We've disabled this model temporarily because of GPU capacity constraints. Check back later.",
+      autoRetry: false,
+    };
+  }
+
   const { model, messages, ...rest } = input;
 
   const templatedPrompt = frontendModelProvider.models[model].templatePrompt?.(messages);

app/src/modelProviders/replicate-llama2/getCompletion.ts

@@ -8,9 +8,9 @@ const replicate = new Replicate({
 });
 
 const modelIds: Record<ReplicateLlama2Input["model"], string> = {
-  "7b-chat": "7b0bfc9aff140d5b75bacbed23e91fd3c34b01a1e958d32132de6e0a19796e2c",
-  "13b-chat": "2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
-  "70b-chat": "2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1",
+  "7b-chat": "658b64a1e83d7caaba4ef10d5ee9c12c40770003f45852f05c2564962f921d8e",
+  "13b-chat": "7457c09004773f9f9710f7eb3b270287ffcebcfb23a13c8ec30cfb98f6bff9b2",
+  "70b-chat": "4dfd64cc207097970659087cf5670e3c1fbe02f83aa0f751e079cfba72ca790a",
 };
 
 export async function getCompletion(

app/src/modelProviders/types.ts

@@ -59,7 +59,7 @@ export type ModelProvider<SupportedModels extends string, InputSchema, OutputSch
   ) => Promise<CompletionResponse<OutputSchema>>;
   getUsage: (
     input: InputSchema,
-    output: OutputSchema,
+    output?: OutputSchema,
   ) => { gpuRuntime?: number; inputTokens?: number; outputTokens?: number; cost?: number } | null;
 
   // This is just a convenience for type inference, don't use it at runtime

app/src/pages/data/[id].tsx

@@ -1,97 +0,0 @@
-import {
-  Box,
-  Breadcrumb,
-  BreadcrumbItem,
-  Center,
-  Flex,
-  Icon,
-  Input,
-  VStack,
-} from "@chakra-ui/react";
-import Link from "next/link";
-
-import { useRouter } from "next/router";
-import { useState, useEffect } from "react";
-import { RiDatabase2Line } from "react-icons/ri";
-import AppShell from "~/components/nav/AppShell";
-import { api } from "~/utils/api";
-import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
-import DatasetEntriesTable from "~/components/datasets/DatasetEntriesTable";
-import { DatasetHeaderButtons } from "~/components/datasets/DatasetHeaderButtons/DatasetHeaderButtons";
-import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
-import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
-
-export default function Dataset() {
-  const router = useRouter();
-  const utils = api.useContext();
-
-  const dataset = useDataset();
-  const datasetId = router.query.id as string;
-
-  const [name, setName] = useState(dataset.data?.name || "");
-  useEffect(() => {
-    setName(dataset.data?.name || "");
-  }, [dataset.data?.name]);
-
-  const updateMutation = api.datasets.update.useMutation();
-  const [onSaveName] = useHandledAsyncCallback(async () => {
-    if (name && name !== dataset.data?.name && dataset.data?.id) {
-      await updateMutation.mutateAsync({
-        id: dataset.data.id,
-        updates: { name: name },
-      });
-      await Promise.all([utils.datasets.list.invalidate(), utils.datasets.get.invalidate()]);
-    }
-  }, [updateMutation, dataset.data?.id, dataset.data?.name, name]);
-
-  if (!dataset.isLoading && !dataset.data) {
-    return (
-      <AppShell title="Dataset not found">
-        <Center h="100%">
-          <div>Dataset not found 😕</div>
-        </Center>
-      </AppShell>
-    );
-  }
-
-  return (
-    <AppShell title={dataset.data?.name}>
-      <VStack h="full">
-        <PageHeaderContainer>
-          <Breadcrumb>
-            <BreadcrumbItem>
-              <ProjectBreadcrumbContents projectName={dataset.data?.project?.name} />
-            </BreadcrumbItem>
-            <BreadcrumbItem>
-              <Link href="/data">
-                <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
-                  <Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
-                </Flex>
-              </Link>
-            </BreadcrumbItem>
-            <BreadcrumbItem isCurrentPage>
-              <Input
-                size="sm"
-                value={name}
-                onChange={(e) => setName(e.target.value)}
-                onBlur={onSaveName}
-                borderWidth={1}
-                borderColor="transparent"
-                fontSize={16}
-                px={0}
-                minW={{ base: 100, lg: 300 }}
-                flex={1}
-                _hover={{ borderColor: "gray.300" }}
-                _focus={{ borderColor: "blue.500", outline: "none" }}
-              />
-            </BreadcrumbItem>
-          </Breadcrumb>
-          <DatasetHeaderButtons />
-        </PageHeaderContainer>
-        <Box w="full" overflowX="auto" flex={1} px={8} pt={8} pb={16}>
-          {datasetId && <DatasetEntriesTable />}
-        </Box>
-      </VStack>
-    </AppShell>
-  );
-}

app/src/pages/data/index.tsx

@@ -1,49 +0,0 @@
-import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
-import AppShell from "~/components/nav/AppShell";
-import { RiDatabase2Line } from "react-icons/ri";
-import {
-  DatasetCard,
-  DatasetCardSkeleton,
-  NewDatasetCard,
-} from "~/components/datasets/DatasetCard";
-import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
-import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
-import { useDatasets } from "~/utils/hooks";
-
-export default function DatasetsPage() {
-  const datasets = useDatasets();
-
-  return (
-    <AppShell title="Data" requireAuth>
-      <PageHeaderContainer>
-        <Breadcrumb>
-          <BreadcrumbItem>
-            <ProjectBreadcrumbContents />
-          </BreadcrumbItem>
-          <BreadcrumbItem minH={8}>
-            <Flex alignItems="center">
-              <Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
-            </Flex>
-          </BreadcrumbItem>
-        </Breadcrumb>
-      </PageHeaderContainer>
-      <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py={4} px={8}>
-        <NewDatasetCard />
-        {datasets.data && !datasets.isLoading ? (
-          datasets?.data?.map((dataset) => (
-            <DatasetCard
-              key={dataset.id}
-              dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
-            />
-          ))
-        ) : (
-          <>
-            <DatasetCardSkeleton />
-            <DatasetCardSkeleton />
-            <DatasetCardSkeleton />
-          </>
-        )}
-      </SimpleGrid>
-    </AppShell>
-  );
-}

app/src/pages/datasets/[id].tsx

@@ -0,0 +1,121 @@
+import {
+  Breadcrumb,
+  BreadcrumbItem,
+  Center,
+  Flex,
+  Icon,
+  Input,
+  VStack,
+  HStack,
+  useDisclosure,
+} from "@chakra-ui/react";
+import Link from "next/link";
+import { useState, useEffect } from "react";
+import { AiOutlineDatabase } from "react-icons/ai";
+
+import AppShell from "~/components/nav/AppShell";
+import { api } from "~/utils/api";
+import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import DatasetConfigurationDrawer from "~/components/datasets/DatasetConfigurationDrawer/DatasetConfigurationDrawer";
+import { DatasetHeaderButtons } from "~/components/datasets/DatasetHeaderButtons";
+import DatasetEntriesTable from "~/components/datasets/DatasetEntriesTable/DatasetEntriesTable";
+import DatasetEntryPaginator from "~/components/datasets/DatasetEntryPaginator";
+import { useAppStore } from "~/state/store";
+import FineTuneButton from "~/components/datasets/FineTuneButton";
+// import ExperimentButton from "~/components/datasets/ExperimentButton";
+import UploadDataButton from "~/components/datasets/UploadDataButton";
+// import DownloadButton from "~/components/datasets/DownloadButton";
+import DeleteButton from "~/components/datasets/DeleteButton";
+import FileUploadsCard from "~/components/datasets/FileUploadsCard";
+
+export default function Dataset() {
+  const utils = api.useContext();
+
+  const dataset = useDataset();
+
+  const drawerDisclosure = useDisclosure();
+  const [name, setName] = useState(dataset.data?.name || "");
+  useEffect(() => {
+    setName(dataset.data?.name || "");
+  }, [dataset.data?.name]);
+
+  useEffect(() => {
+    useAppStore.getState().sharedArgumentsEditor.loadMonaco().catch(console.error);
+  }, []);
+
+  const updateMutation = api.datasets.update.useMutation();
+  const [onSaveName] = useHandledAsyncCallback(async () => {
+    if (name && name !== dataset.data?.name && dataset.data?.id) {
+      await updateMutation.mutateAsync({
+        id: dataset.data.id,
+        name,
+      });
+      await Promise.all([utils.datasets.list.invalidate(), utils.datasets.get.invalidate()]);
+    }
+  }, [updateMutation, dataset.data?.id, dataset.data?.name, name]);
+
+  if (!dataset.isLoading && !dataset.data) {
+    return (
+      <AppShell title="Dataset not found">
+        <Center h="100%">
+          <div>Dataset not found 😕</div>
+        </Center>
+      </AppShell>
+    );
+  }
+
+  return (
+    <>
+      <AppShell title={dataset.data?.name}>
+        <VStack h="full" overflowY="scroll">
+          <PageHeaderContainer>
+            <Breadcrumb>
+              <BreadcrumbItem>
+                <ProjectBreadcrumbContents projectName={dataset.data?.project?.name} />
+              </BreadcrumbItem>
+              <BreadcrumbItem>
+                <Link href="/datasets">
+                  <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
+                    <Icon as={AiOutlineDatabase} boxSize={4} mr={2} /> Datasets
+                  </Flex>
+                </Link>
+              </BreadcrumbItem>
+              <BreadcrumbItem isCurrentPage>
+                <Input
+                  size="sm"
+                  value={name}
+                  onChange={(e) => setName(e.target.value)}
+                  onBlur={onSaveName}
+                  borderWidth={1}
+                  borderColor="transparent"
+                  fontSize={16}
+                  px={0}
+                  minW={{ base: 100, lg: 300 }}
+                  flex={1}
+                  _hover={{ borderColor: "gray.300" }}
+                  _focus={{ borderColor: "blue.500", outline: "none" }}
+                />
+              </BreadcrumbItem>
+            </Breadcrumb>
+            <DatasetHeaderButtons openDrawer={drawerDisclosure.onOpen} />
+          </PageHeaderContainer>
+          <VStack px={8} py={8} alignItems="flex-start" spacing={4} w="full">
+            <HStack w="full" justifyContent="flex-end">
+              <FineTuneButton />
+              <UploadDataButton />
+              {/* <ExperimentButton /> */}
+              {/* <DownloadButton /> */}
+              <DeleteButton />
+            </HStack>
+            <DatasetEntriesTable />
+            <DatasetEntryPaginator />
+          </VStack>
+        </VStack>
+        <FileUploadsCard />
+      </AppShell>
+      <DatasetConfigurationDrawer disclosure={drawerDisclosure} />
+    </>
+  );
+}

app/src/pages/datasets/index.tsx

@@ -0,0 +1,17 @@
+import { VStack, Text, Divider } from "@chakra-ui/react";
+import AppShell from "~/components/nav/AppShell";
+import DatasetsTable from "~/components/datasets/DatasetsTable";
+
+export default function DatasetsPage() {
+  return (
+    <AppShell title="Datasets" requireAuth>
+      <VStack w="full" py={8} px={8} spacing={4} alignItems="flex-start">
+        <Text fontSize="2xl" fontWeight="bold">
+          Datasets
+        </Text>
+        <Divider />
+        <DatasetsTable />
+      </VStack>
+    </AppShell>
+  );
+}

app/src/pages/experiments/[experimentSlug].tsx

@@ -8,26 +8,25 @@ import {
   Input,
   Text,
   VStack,
+  useDisclosure,
 } from "@chakra-ui/react";
 import Link from "next/link";
 
-import { useRouter } from "next/router";
 import { useState, useEffect } from "react";
 import { RiFlaskLine } from "react-icons/ri";
 import OutputsTable from "~/components/OutputsTable";
-import ExperimentSettingsDrawer from "~/components/ExperimentSettingsDrawer/ExperimentSettingsDrawer";
+import ExperimentSettingsDrawer from "~/components/experiments/ExperimentSettingsDrawer/ExperimentSettingsDrawer";
+import { ExperimentHeaderButtons } from "~/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons";
 import AppShell from "~/components/nav/AppShell";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import { useAppStore } from "~/state/store";
 import { useSyncVariantEditor } from "~/state/sync";
-import { ExperimentHeaderButtons } from "~/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons";
 import Head from "next/head";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
 import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
 export default function Experiment() {
-  const router = useRouter();
   const utils = api.useContext();
   useSyncVariantEditor();
 
@@ -44,6 +43,7 @@ export default function Experiment() {
     useAppStore.getState().sharedVariantEditor.loadMonaco().catch(console.error);
   }, []);
 
+  const drawerDisclosure = useDisclosure();
   const [label, setLabel] = useState(experiment.data?.label || "");
   useEffect(() => {
     setLabel(experiment.data?.label || "");
@@ -121,11 +121,11 @@ export default function Experiment() {
                 )}
               </BreadcrumbItem>
             </Breadcrumb>
-            <ExperimentHeaderButtons />
+            <ExperimentHeaderButtons openDrawer={drawerDisclosure.onOpen} />
           </PageHeaderContainer>
-          <ExperimentSettingsDrawer />
+          <ExperimentSettingsDrawer disclosure={drawerDisclosure} />
           <Box w="100%" overflowX="auto" flex={1} id="output-container">
-            <OutputsTable experimentId={experiment.data?.id} />
+            <OutputsTable experimentId={experiment.data?.id} openDrawer={drawerDisclosure.onOpen} />
           </Box>
         </VStack>
       </AppShell>

app/src/pages/fine-tunes/index.tsx

@@ -0,0 +1,18 @@
+import { Text, VStack, Divider } from "@chakra-ui/react";
+import FineTunesTable from "~/components/fineTunes/FineTunesTable";
+
+import AppShell from "~/components/nav/AppShell";
+
+export default function FineTunes() {
+  return (
+    <AppShell title="Fine Tunes" requireAuth requireBeta>
+      <VStack px={8} py={8} alignItems="flex-start" spacing={4} w="full">
+        <Text fontSize="2xl" fontWeight="bold">
+          Fine Tunes
+        </Text>
+        <Divider />
+        <FineTunesTable />
+      </VStack>
+    </AppShell>
+  );
+}

app/src/pages/request-logs/index.tsx

@@ -1,14 +1,16 @@
 import { useState } from "react";
-import { Text, VStack, Divider, HStack } from "@chakra-ui/react";
+import { Text, VStack, Divider, HStack, Box } from "@chakra-ui/react";
 
 import AppShell from "~/components/nav/AppShell";
 import LoggedCallTable from "~/components/requestLogs/LoggedCallsTable";
 import LoggedCallsPaginator from "~/components/requestLogs/LoggedCallsPaginator";
-import ActionButton from "~/components/requestLogs/ActionButton";
+import ActionButton from "~/components/ActionButton";
 import { useAppStore } from "~/state/store";
-import { RiFlaskLine } from "react-icons/ri";
 import { FiFilter } from "react-icons/fi";
 import LogFilters from "~/components/requestLogs/LogFilters/LogFilters";
+import ColumnVisiblityDropdown from "~/components/requestLogs/ColumnVisiblityDropdown";
+import ExportButton from "~/components/requestLogs/ExportButton";
+import AddToDatasetButton from "~/components/requestLogs/AddToDatasetButton";
 
 export default function LoggedCalls() {
   const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
@@ -17,32 +19,29 @@ export default function LoggedCalls() {
 
   return (
     <AppShell title="Request Logs" requireAuth>
-      <VStack px={8} py={8} alignItems="flex-start" spacing={4} w="full">
-        <Text fontSize="2xl" fontWeight="bold">
-          Request Logs
-        </Text>
-        <Divider />
-        <HStack w="full" justifyContent="flex-end">
-          <ActionButton
-            onClick={() => {
-              setFiltersShown(!filtersShown);
-            }}
-            label={filtersShown ? "Hide Filters" : "Show Filters"}
-            icon={FiFilter}
-          />
-          <ActionButton
-            onClick={() => {
-              console.log("experimenting with these ids", selectedLogIds);
-            }}
-            label="Experiment"
-            icon={RiFlaskLine}
-            isDisabled={selectedLogIds.size === 0}
-          />
-        </HStack>
-        {filtersShown && <LogFilters />}
-        <LoggedCallTable />
-        <LoggedCallsPaginator />
-      </VStack>
+      <Box h="100vh" overflowY="scroll">
+        <VStack px={8} py={8} alignItems="flex-start" spacing={4} w="full">
+          <Text fontSize="2xl" fontWeight="bold">
+            Request Logs
+          </Text>
+          <Divider />
+          <HStack w="full" justifyContent="flex-end">
+            <AddToDatasetButton />
+            <ExportButton />
+            <ColumnVisiblityDropdown />
+            <ActionButton
+              onClick={() => {
+                setFiltersShown(!filtersShown);
+              }}
+              label={filtersShown ? "Hide Filters" : "Show Filters"}
+              icon={FiFilter}
+            />
+          </HStack>
+          {filtersShown && <LogFilters />}
+          <LoggedCallTable />
+          <LoggedCallsPaginator />
+        </VStack>
+      </Box>
     </AppShell>
   );
 }

app/src/server/api/autogenerate/autogenerateDatasetEntries.ts

@@ -1,108 +0,0 @@
-import { type ChatCompletion } from "openai/resources/chat";
-import { openai } from "../../utils/openai";
-import { isAxiosError } from "./utils";
-import { type APIResponse } from "openai/core";
-import { sleep } from "~/server/utils/sleep";
-
-const MAX_AUTO_RETRIES = 50;
-const MIN_DELAY = 500; // milliseconds
-const MAX_DELAY = 15000; // milliseconds
-
-function calculateDelay(numPreviousTries: number): number {
-  const baseDelay = Math.min(MAX_DELAY, MIN_DELAY * Math.pow(2, numPreviousTries));
-  const jitter = Math.random() * baseDelay;
-  return baseDelay + jitter;
-}
-
-const getCompletionWithBackoff = async (
-  getCompletion: () => Promise<APIResponse<ChatCompletion>>,
-) => {
-  let completion;
-  let tries = 0;
-  while (tries < MAX_AUTO_RETRIES) {
-    try {
-      completion = await getCompletion();
-      break;
-    } catch (e) {
-      if (isAxiosError(e)) {
-        console.error(e?.response?.data?.error?.message);
-      } else {
-        await sleep(calculateDelay(tries));
-        console.error(e);
-      }
-    }
-    tries++;
-  }
-  return completion;
-};
-// TODO: Add seeds to ensure batches don't contain duplicate data
-const MAX_BATCH_SIZE = 5;
-
-export const autogenerateDatasetEntries = async (
-  numToGenerate: number,
-  inputDescription: string,
-  outputDescription: string,
-): Promise<{ input: string; output: string }[]> => {
-  const batchSizes = Array.from({ length: Math.ceil(numToGenerate / MAX_BATCH_SIZE) }, (_, i) =>
-    i === Math.ceil(numToGenerate / MAX_BATCH_SIZE) - 1 && numToGenerate % MAX_BATCH_SIZE
-      ? numToGenerate % MAX_BATCH_SIZE
-      : MAX_BATCH_SIZE,
-  );
-
-  const getCompletion = (batchSize: number) =>
-    openai.chat.completions.create({
-      model: "gpt-4",
-      messages: [
-        {
-          role: "system",
-          content: `The user needs ${batchSize} rows of data, each with an input and an output.\n---\n The input should follow these requirements: ${inputDescription}\n---\n The output should follow these requirements: ${outputDescription}`,
-        },
-      ],
-      functions: [
-        {
-          name: "add_list_of_data",
-          description: "Add a list of data to the database",
-          parameters: {
-            type: "object",
-            properties: {
-              rows: {
-                type: "array",
-                description: "The rows of data that match the description",
-                items: {
-                  type: "object",
-                  properties: {
-                    input: {
-                      type: "string",
-                      description: "The input for this row",
-                    },
-                    output: {
-                      type: "string",
-                      description: "The output for this row",
-                    },
-                  },
-                },
-              },
-            },
-          },
-        },
-      ],
-
-      function_call: { name: "add_list_of_data" },
-      temperature: 0.5,
-    });
-
-  const completionCallbacks = batchSizes.map((batchSize) =>
-    getCompletionWithBackoff(() => getCompletion(batchSize)),
-  );
-
-  const completions = await Promise.all(completionCallbacks);
-
-  const rows = completions.flatMap((completion) => {
-    const parsed = JSON.parse(
-      completion?.choices[0]?.message?.function_call?.arguments ?? "{rows: []}",
-    ) as { rows: { input: string; output: string }[] };
-    return parsed.rows;
-  });
-
-  return rows;
-};

app/src/server/api/autogenerate/autogenerateScenarioValues.ts

@@ -98,6 +98,11 @@ export const autogenerateScenarioValues = async (
 
       function_call: { name: "add_scenario" },
       temperature: 0.5,
+      openpipe: {
+        tags: {
+          prompt_id: "autogenerateScenarioValues",
+        },
+      },
     });
 
     const parsed = JSON.parse(

app/src/server/api/external/v1Api.router.ts

@@ -119,10 +119,10 @@ export const v1ApiRouter = createOpenApiRouter({
 
       let usage;
       let model;
-      if (reqPayload.success && respPayload.success) {
+      if (reqPayload.success) {
         usage = modelProvider.getUsage(
           input.reqPayload as CompletionCreateParams,
-          input.respPayload as ChatCompletion,
+          respPayload.success ? (input.respPayload as ChatCompletion) : undefined,
         );
         model = reqPayload.data.model;
       }

app/src/server/api/root.router.ts

@@ -6,11 +6,12 @@ import { scenarioVariantCellsRouter } from "./routers/scenarioVariantCells.route
 import { scenarioVarsRouter } from "./routers/scenarioVariables.router";
 import { evaluationsRouter } from "./routers/evaluations.router";
 import { worldChampsRouter } from "./routers/worldChamps.router";
-import { datasetsRouter } from "./routers/datasets.router";
-import { datasetEntries } from "./routers/datasetEntries.router";
 import { projectsRouter } from "./routers/projects.router";
 import { dashboardRouter } from "./routers/dashboard.router";
 import { loggedCallsRouter } from "./routers/loggedCalls.router";
+import { datasetsRouter } from "./routers/datasets.router";
+import { datasetEntriesRouter } from "./routers/datasetEntries.router";
+import { fineTunesRouter } from "./routers/fineTunes.router";
 import { usersRouter } from "./routers/users.router";
 import { adminJobsRouter } from "./routers/adminJobs.router";
 
@@ -27,11 +28,12 @@ export const appRouter = createTRPCRouter({
   scenarioVars: scenarioVarsRouter,
   evaluations: evaluationsRouter,
   worldChamps: worldChampsRouter,
-  datasets: datasetsRouter,
-  datasetEntries: datasetEntries,
   projects: projectsRouter,
   dashboard: dashboardRouter,
   loggedCalls: loggedCallsRouter,
+  datasets: datasetsRouter,
+  datasetEntries: datasetEntriesRouter,
+  fineTunes: fineTunesRouter,
   users: usersRouter,
   adminJobs: adminJobsRouter,
 });

app/src/server/api/routers/datasetEntries.router.ts

@@ -1,145 +1,337 @@
 import { z } from "zod";
+import { v4 as uuidv4 } from "uuid";
+import {
+  type ChatCompletion,
+  type CompletionCreateParams,
+  type CreateChatCompletionRequestMessage,
+} from "openai/resources/chat";
+import { TRPCError } from "@trpc/server";
+import archiver from "archiver";
+import { WritableStreamBuffer } from "stream-buffers";
+
 import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
-import { requireCanModifyDataset, requireCanViewDataset } from "~/utils/accessControl";
-import { autogenerateDatasetEntries } from "../autogenerate/autogenerateDatasetEntries";
+import { requireCanModifyProject, requireCanViewProject } from "~/utils/accessControl";
+import { error, success } from "~/utils/errorHandling/standardResponses";
+import { countOpenAIChatTokens } from "~/utils/countTokens";
+import { type TrainingRow } from "~/components/datasets/validateTrainingRows";
+import hashObject from "~/server/utils/hashObject";
+import { type JsonValue } from "type-fest";
+import { formatEntriesFromTrainingRows } from "~/server/utils/createEntriesFromTrainingRows";
 
-export const datasetEntries = createTRPCRouter({
+export const datasetEntriesRouter = createTRPCRouter({
   list: protectedProcedure
     .input(z.object({ datasetId: z.string(), page: z.number(), pageSize: z.number() }))
     .query(async ({ input, ctx }) => {
-      await requireCanViewDataset(input.datasetId, ctx);
-
       const { datasetId, page, pageSize } = input;
 
-      const entries = await prisma.datasetEntry.findMany({
-        where: {
-          datasetId,
-        },
-        orderBy: { createdAt: "desc" },
-        skip: (page - 1) * pageSize,
-        take: pageSize,
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: datasetId },
       });
+      await requireCanViewProject(projectId, ctx);
 
-      const count = await prisma.datasetEntry.count({
-        where: {
-          datasetId,
-        },
-      });
+      const [entries, matchingEntries, trainingCount, testingCount] = await prisma.$transaction([
+        prisma.datasetEntry.findMany({
+          where: {
+            datasetId: datasetId,
+          },
+          orderBy: [{ createdAt: "desc" }, { id: "desc" }],
+          skip: (page - 1) * pageSize,
+          take: pageSize,
+        }),
+        prisma.datasetEntry.findMany({
+          where: {
+            datasetId: datasetId,
+          },
+          select: {
+            id: true,
+          },
+        }),
+        prisma.datasetEntry.count({
+          where: {
+            datasetId: datasetId,
+            type: "TRAIN",
+          },
+        }),
+        prisma.datasetEntry.count({
+          where: {
+            datasetId: datasetId,
+            type: "TEST",
+          },
+        }),
+      ]);
 
       return {
         entries,
-        count,
+        matchingEntryIds: matchingEntries.map((entry) => entry.id),
+        trainingCount,
+        testingCount,
       };
     }),
-  createOne: protectedProcedure
+  get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
+    const entry = await prisma.datasetEntry.findUniqueOrThrow({
+      where: { id: input.id },
+      include: {
+        dataset: true,
+      },
+    });
+
+    if (!entry.dataset) {
+      throw new TRPCError({ message: "Dataset not found for dataset entry", code: "NOT_FOUND" });
+    }
+
+    await requireCanViewProject(entry.dataset.projectId, ctx);
+
+    if (!entry) {
+      throw new TRPCError({ message: "Dataset entry not found", code: "NOT_FOUND" });
+    }
+
+    return entry;
+  }),
+  create: protectedProcedure
     .input(
       z.object({
-        datasetId: z.string(),
-        input: z.string(),
-        output: z.string().optional(),
+        datasetId: z.string().optional(),
+        newDatasetParams: z
+          .object({
+            projectId: z.string(),
+            name: z.string(),
+          })
+          .optional(),
+        loggedCallIds: z.string().array().optional(),
       }),
     )
     .mutation(async ({ input, ctx }) => {
-      await requireCanModifyDataset(input.datasetId, ctx);
-
-      return await prisma.datasetEntry.create({
-        data: {
-          datasetId: input.datasetId,
-          input: input.input,
-          output: input.output,
-        },
-      });
-    }),
-
-  autogenerateEntries: protectedProcedure
-    .input(
-      z.object({
-        datasetId: z.string(),
-        numToGenerate: z.number(),
-        inputDescription: z.string(),
-        outputDescription: z.string(),
-      }),
-    )
-    .mutation(async ({ input, ctx }) => {
-      await requireCanModifyDataset(input.datasetId, ctx);
-
-      const dataset = await prisma.dataset.findUnique({
-        where: {
-          id: input.datasetId,
-        },
-      });
-
-      if (!dataset) {
-        throw new Error(`Dataset with id ${input.datasetId} does not exist`);
+      let datasetId: string;
+      let trainingRatio = 0.8;
+      if (input.datasetId) {
+        datasetId = input.datasetId;
+        const { projectId, trainingRatio: datasetTrainingRatio } =
+          await prisma.dataset.findUniqueOrThrow({
+            where: { id: input.datasetId },
+          });
+        trainingRatio = datasetTrainingRatio;
+        await requireCanModifyProject(projectId, ctx);
+      } else if (input.newDatasetParams) {
+        await requireCanModifyProject(input.newDatasetParams.projectId, ctx);
+        datasetId = uuidv4();
+      } else {
+        return error("No datasetId or newDatasetParams provided");
       }
 
-      const entries = await autogenerateDatasetEntries(
-        input.numToGenerate,
-        input.inputDescription,
-        input.outputDescription,
-      );
+      if (!input.loggedCallIds) {
+        return error("No loggedCallIds provided");
+      }
 
-      const createdEntries = await prisma.datasetEntry.createMany({
-        data: entries.map((entry) => ({
-          datasetId: input.datasetId,
-          input: entry.input,
-          output: entry.output,
-        })),
-      });
-
-      return createdEntries;
-    }),
-
-  delete: protectedProcedure
-    .input(z.object({ id: z.string() }))
-    .mutation(async ({ input, ctx }) => {
-      const datasetId = (
-        await prisma.datasetEntry.findUniqueOrThrow({
-          where: { id: input.id },
-        })
-      ).datasetId;
-
-      await requireCanModifyDataset(datasetId, ctx);
-
-      return await prisma.datasetEntry.delete({
+      const loggedCalls = await prisma.loggedCall.findMany({
         where: {
-          id: input.id,
+          id: {
+            in: input.loggedCallIds,
+          },
+          modelResponse: {
+            isNot: null,
+          },
         },
+        include: {
+          modelResponse: {
+            select: {
+              reqPayload: true,
+              respPayload: true,
+              inputTokens: true,
+              outputTokens: true,
+            },
+          },
+        },
+        orderBy: { createdAt: "desc" },
       });
-    }),
 
+      const trainingRows = loggedCalls.map((loggedCall) => {
+        const inputMessages = (
+          loggedCall.modelResponse?.reqPayload as unknown as CompletionCreateParams
+        ).messages;
+        let output: ChatCompletion.Choice.Message | undefined = undefined;
+        const resp = loggedCall.modelResponse?.respPayload as unknown as ChatCompletion | undefined;
+        if (resp && resp.choices?.[0]) {
+          output = resp.choices[0].message;
+        }
+        return {
+          input: inputMessages as unknown as CreateChatCompletionRequestMessage[],
+          output: output as unknown as CreateChatCompletionRequestMessage,
+        };
+      });
+
+      const datasetEntriesToCreate = await formatEntriesFromTrainingRows(datasetId, trainingRows);
+
+      // Ensure dataset and dataset entries are created atomically
+      await prisma.$transaction([
+        prisma.dataset.upsert({
+          where: { id: datasetId },
+          update: {},
+          create: {
+            id: datasetId,
+            projectId: input.newDatasetParams?.projectId ?? "",
+            name: input.newDatasetParams?.name ?? "",
+            trainingRatio,
+          },
+        }),
+        prisma.datasetEntry.createMany({
+          data: datasetEntriesToCreate,
+        }),
+      ]);
+
+      return success(datasetId);
+    }),
   update: protectedProcedure
     .input(
       z.object({
         id: z.string(),
         updates: z.object({
-          input: z.string(),
+          type: z.enum(["TRAIN", "TEST"]).optional(),
+          input: z.string().optional(),
           output: z.string().optional(),
         }),
       }),
     )
     .mutation(async ({ input, ctx }) => {
-      const existing = await prisma.datasetEntry.findUnique({
-        where: {
-          id: input.id,
+      const { dataset } = await prisma.datasetEntry.findUniqueOrThrow({
+        where: { id: input.id },
+        include: {
+          dataset: true,
         },
       });
 
-      if (!existing) {
-        throw new Error(`dataEntry with id ${input.id} does not exist`);
+      if (!dataset) {
+        return error("Dataset not found for dataset entry");
       }
 
-      await requireCanModifyDataset(existing.datasetId, ctx);
+      await requireCanModifyProject(dataset.projectId, ctx);
 
-      return await prisma.datasetEntry.update({
-        where: {
-          id: input.id,
-        },
+      let parsedInput = undefined;
+      let inputTokens = undefined;
+      if (input.updates.input) {
+        parsedInput = JSON.parse(input.updates.input);
+        inputTokens = countOpenAIChatTokens(
+          "gpt-4-0613",
+          parsedInput as unknown as CreateChatCompletionRequestMessage[],
+        );
+      }
+
+      let parsedOutput = undefined;
+      let outputTokens = undefined;
+      // The client might send "null" as a string, so we need to check for that
+      if (input.updates.output && input.updates.output !== "null") {
+        parsedOutput = JSON.parse(input.updates.output);
+        outputTokens = countOpenAIChatTokens("gpt-4-0613", [
+          parsedOutput as unknown as ChatCompletion.Choice.Message,
+        ]);
+      }
+
+      await prisma.datasetEntry.update({
+        where: { id: input.id },
         data: {
-          input: input.updates.input,
-          output: input.updates.output,
+          type: input.updates.type,
+          input: parsedInput,
+          output: parsedOutput,
+          inputTokens,
+          outputTokens,
         },
       });
+
+      return success("Dataset entry updated");
+    }),
+
+  delete: protectedProcedure
+    .input(z.object({ ids: z.string().array() }))
+    .mutation(async ({ input, ctx }) => {
+      if (input.ids.length === 0) {
+        return error("No ids provided");
+      }
+      const { dataset } = await prisma.datasetEntry.findUniqueOrThrow({
+        where: { id: input.ids[0] },
+        include: {
+          dataset: true,
+        },
+      });
+
+      if (!dataset) {
+        return error("Dataset not found for dataset entry");
+      }
+
+      await requireCanModifyProject(dataset.projectId, ctx);
+
+      await prisma.datasetEntry.deleteMany({
+        where: {
+          id: {
+            in: input.ids,
+          },
+          datasetId: dataset?.id,
+        },
+      });
+
+      return success("Dataset entries deleted");
+    }),
+
+  export: protectedProcedure
+    .input(
+      z.object({
+        datasetId: z.string(),
+        datasetEntryIds: z.string().array(),
+        testingSplit: z.number(),
+        removeDuplicates: z.boolean(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.datasetId },
+      });
+      await requireCanViewProject(projectId, ctx);
+
+      const datasetEntries = await ctx.prisma.datasetEntry.findMany({
+        where: {
+          id: {
+            in: input.datasetEntryIds,
+          },
+        },
+      });
+
+      let rows: TrainingRow[] = datasetEntries.map((entry) => ({
+        input: entry.input as unknown as CreateChatCompletionRequestMessage[],
+        output: entry.output as unknown as CreateChatCompletionRequestMessage,
+      }));
+
+      if (input.removeDuplicates) {
+        const deduplicatedRows = [];
+        const rowHashSet = new Set<string>();
+        for (const row of rows) {
+          const rowHash = hashObject(row as unknown as JsonValue);
+          if (!rowHashSet.has(rowHash)) {
+            rowHashSet.add(rowHash);
+            deduplicatedRows.push(row);
+          }
+        }
+        rows = deduplicatedRows;
+      }
+
+      const splitIndex = Math.floor((rows.length * input.testingSplit) / 100);
+
+      const testingData = rows.slice(0, splitIndex);
+      const trainingData = rows.slice(splitIndex);
+
+      // Convert arrays to JSONL format
+      const trainingDataJSONL = trainingData.map((item) => JSON.stringify(item)).join("\n");
+      const testingDataJSONL = testingData.map((item) => JSON.stringify(item)).join("\n");
+
+      const output = new WritableStreamBuffer();
+      const archive = archiver("zip");
+
+      archive.pipe(output);
+      archive.append(trainingDataJSONL, { name: "train.jsonl" });
+      archive.append(testingDataJSONL, { name: "test.jsonl" });
+      await archive.finalize();
+
+      // Convert buffer to base64
+      const base64 = output.getContents().toString("base64");
+
+      return base64;
     }),
 });

app/src/server/api/routers/datasets.router.ts

@@ -1,87 +1,182 @@
 import { z } from "zod";
-import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
-import {
-  requireCanModifyDataset,
-  requireCanModifyProject,
-  requireCanViewDataset,
-  requireCanViewProject,
-} from "~/utils/accessControl";
+import { requireCanModifyProject, requireCanViewProject } from "~/utils/accessControl";
+import { error, success } from "~/utils/errorHandling/standardResponses";
+import { generateServiceClientUrl } from "~/utils/azure/server";
+import { queueImportDatasetEntries } from "~/server/tasks/importDatasetEntries.task";
 
 export const datasetsRouter = createTRPCRouter({
-  list: protectedProcedure
-    .input(z.object({ projectId: z.string() }))
-    .query(async ({ input, ctx }) => {
-      await requireCanViewProject(input.projectId, ctx);
-
-      const datasets = await prisma.dataset.findMany({
-        where: {
-          projectId: input.projectId,
-        },
-        orderBy: {
-          createdAt: "desc",
-        },
-        include: {
-          _count: {
-            select: { datasetEntries: true },
-          },
-        },
-      });
-
-      return datasets;
-    }),
-
-  get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
-    await requireCanViewDataset(input.id, ctx);
-    return await prisma.dataset.findFirstOrThrow({
+  get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
+    const dataset = await prisma.dataset.findUniqueOrThrow({
       where: { id: input.id },
       include: {
         project: true,
       },
     });
+
+    await requireCanViewProject(dataset.projectId, ctx);
+
+    return dataset;
   }),
-
-  create: protectedProcedure
+  list: protectedProcedure
     .input(z.object({ projectId: z.string() }))
-    .mutation(async ({ input, ctx }) => {
-      await requireCanModifyProject(input.projectId, ctx);
+    .query(async ({ input, ctx }) => {
+      await requireCanViewProject(input.projectId, ctx);
 
-      const numDatasets = await prisma.dataset.count({
+      return await prisma.dataset.findMany({
         where: {
           projectId: input.projectId,
         },
-      });
-
-      return await prisma.dataset.create({
-        data: {
-          name: `Dataset ${numDatasets + 1}`,
-          projectId: input.projectId,
+        include: {
+          _count: {
+            select: {
+              datasetEntries: true,
+            },
+          },
         },
+        orderBy: { createdAt: "desc" },
       });
     }),
 
-  update: protectedProcedure
-    .input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))
+  create: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        name: z.string(),
+      }),
+    )
     .mutation(async ({ input, ctx }) => {
-      await requireCanModifyDataset(input.id, ctx);
-      return await prisma.dataset.update({
-        where: {
-          id: input.id,
-        },
+      await requireCanModifyProject(input.projectId, ctx);
+
+      const dataset = await prisma.dataset.create({
         data: {
-          name: input.updates.name,
+          projectId: input.projectId,
+          name: input.name,
         },
       });
+
+      return success(dataset);
+    }),
+
+  update: protectedProcedure
+    .input(
+      z.object({
+        id: z.string(),
+        name: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+      await requireCanModifyProject(projectId, ctx);
+
+      await prisma.dataset.update({
+        where: { id: input.id },
+        data: {
+          name: input.name,
+        },
+      });
+
+      return success("Dataset updated");
     }),
 
   delete: protectedProcedure
     .input(z.object({ id: z.string() }))
     .mutation(async ({ input, ctx }) => {
-      await requireCanModifyDataset(input.id, ctx);
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+      await requireCanModifyProject(projectId, ctx);
 
       await prisma.dataset.delete({
+        where: { id: input.id },
+      });
+
+      return success("Dataset deleted");
+    }),
+  getServiceClientUrl: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      // The user must at least be authenticated to get a SAS token
+      await requireCanModifyProject(input.projectId, ctx);
+      return generateServiceClientUrl();
+    }),
+  triggerFileDownload: protectedProcedure
+    .input(
+      z.object({
+        datasetId: z.string(),
+        blobName: z.string(),
+        fileName: z.string(),
+        fileSize: z.number(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.datasetId },
+      });
+      await requireCanViewProject(projectId, ctx);
+
+      const { id } = await prisma.datasetFileUpload.create({
+        data: {
+          datasetId: input.datasetId,
+          blobName: input.blobName,
+          status: "PENDING",
+          fileName: input.fileName,
+          fileSize: input.fileSize,
+          uploadedAt: new Date(),
+        },
+      });
+
+      await queueImportDatasetEntries(id);
+    }),
+  listFileUploads: protectedProcedure
+    .input(z.object({ datasetId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.datasetId },
+      });
+      await requireCanViewProject(projectId, ctx);
+
+      return await prisma.datasetFileUpload.findMany({
         where: {
-          id: input.id,
+          datasetId: input.datasetId,
+          visible: true,
+        },
+        orderBy: { createdAt: "desc" },
+      });
+    }),
+  hideFileUploads: protectedProcedure
+    .input(z.object({ fileUploadIds: z.string().array() }))
+    .mutation(async ({ input, ctx }) => {
+      if (!input.fileUploadIds.length) return error("No file upload ids provided");
+
+      const {
+        dataset: { projectId, id: datasetId },
+      } = await prisma.datasetFileUpload.findUniqueOrThrow({
+        where: { id: input.fileUploadIds[0] },
+        select: {
+          dataset: {
+            select: {
+              id: true,
+              projectId: true,
+            },
+          },
+        },
+      });
+      await requireCanModifyProject(projectId, ctx);
+
+      await prisma.datasetFileUpload.updateMany({
+        where: {
+          id: {
+            in: input.fileUploadIds,
+          },
+          datasetId,
+        },
+        data: {
+          visible: false,
         },
       });
     }),

app/src/server/api/routers/fineTunes.router.ts

@@ -0,0 +1,90 @@
+import { z } from "zod";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { requireCanViewProject, requireCanModifyProject } from "~/utils/accessControl";
+import { error, success } from "~/utils/errorHandling/standardResponses";
+
+export const fineTunesRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        page: z.number(),
+        pageSize: z.number(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      const { projectId, page, pageSize } = input;
+
+      await requireCanViewProject(projectId, ctx);
+
+      const fineTunes = await prisma.fineTune.findMany({
+        where: {
+          projectId,
+        },
+        include: {
+          dataset: {
+            include: {
+              _count: {
+                select: {
+                  datasetEntries: true,
+                },
+              },
+            },
+          },
+        },
+        orderBy: { createdAt: "asc" },
+        skip: (page - 1) * pageSize,
+        take: pageSize,
+      });
+
+      const count = await prisma.fineTune.count({
+        where: {
+          projectId,
+        },
+      });
+
+      return {
+        fineTunes,
+        count,
+      };
+    }),
+  create: protectedProcedure
+    .input(
+      z.object({
+        datasetId: z.string(),
+        slug: z.string(),
+        baseModel: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: {
+          id: input.datasetId,
+        },
+      });
+      await requireCanModifyProject(projectId, ctx);
+
+      const existingFineTune = await prisma.fineTune.findFirst({
+        where: {
+          slug: input.slug,
+        },
+      });
+
+      if (existingFineTune) {
+        return error("A fine tune with that slug already exists");
+      }
+
+      await prisma.fineTune.create({
+        data: {
+          projectId,
+          slug: input.slug,
+          baseModel: input.baseModel,
+          datasetId: input.datasetId,
+        },
+      });
+
+      return success();
+    }),
+});

app/src/server/api/routers/loggedCalls.router.ts

@@ -1,11 +1,16 @@
 import { z } from "zod";
 import { type Expression, type SqlBool, sql, type RawBuilder } from "kysely";
 import { jsonArrayFrom } from "kysely/helpers/postgres";
+import archiver from "archiver";
+import { WritableStreamBuffer } from "stream-buffers";
+import { type JsonValue } from "type-fest";
+import { shuffle } from "lodash-es";
 
 import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
 import { kysely, prisma } from "~/server/db";
 import { comparators, defaultFilterableFields } from "~/state/logFiltersSlice";
 import { requireCanViewProject } from "~/utils/accessControl";
+import hashObject from "~/server/utils/hashObject";
 
 // create comparator type based off of comparators
 const comparatorToSqlExpression = (comparator: (typeof comparators)[number], value: string) => {
@@ -180,4 +185,102 @@ export const loggedCallsRouter = createTRPCRouter({
 
       return tags.map((tag) => tag.name);
     }),
+  export: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        loggedCallIds: z.string().array(),
+        testingSplit: z.number(),
+        selectedExportFormat: z.string(),
+        removeDuplicates: z.boolean(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      await requireCanViewProject(input.projectId, ctx);
+
+      // Fetch the real data using Prisma
+      const loggedCallsFromDb = await ctx.prisma.loggedCallModelResponse.findMany({
+        where: {
+          originalLoggedCall: {
+            projectId: input.projectId,
+            id: { in: input.loggedCallIds },
+          },
+          statusCode: 200,
+        },
+      });
+
+      // Convert the database data into the desired format
+      let formattedLoggedCalls: { instruction: JsonValue[]; output: JsonValue }[] =
+        loggedCallsFromDb.map((call) => ({
+          instruction: (call.reqPayload as unknown as Record<string, unknown>)
+            .messages as JsonValue[],
+          output: (call.respPayload as unknown as { choices: { message: unknown }[] }).choices[0]
+            ?.message as JsonValue,
+        }));
+
+      if (input.removeDuplicates) {
+        const deduplicatedLoggedCalls = [];
+        const loggedCallHashSet = new Set<string>();
+        for (const loggedCall of formattedLoggedCalls) {
+          const loggedCallHash = hashObject(loggedCall);
+          if (!loggedCallHashSet.has(loggedCallHash)) {
+            loggedCallHashSet.add(loggedCallHash);
+            deduplicatedLoggedCalls.push(loggedCall);
+          }
+        }
+        formattedLoggedCalls = deduplicatedLoggedCalls;
+      }
+
+      // Remove duplicate messages from instructions
+      const instructionMessageHashMap = new Map<string, number>();
+      for (const loggedCall of formattedLoggedCalls) {
+        for (const message of loggedCall.instruction) {
+          const hash = hashObject(message);
+          if (instructionMessageHashMap.has(hash)) {
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            instructionMessageHashMap.set(hash, instructionMessageHashMap.get(hash)! + 1);
+          } else {
+            instructionMessageHashMap.set(hash, 0);
+          }
+        }
+      }
+      for (const loggedCall of formattedLoggedCalls) {
+        loggedCall.instruction = loggedCall.instruction.filter((message) => {
+          const hash = hashObject(message);
+          // If the same message appears in a single instruction multiple times, there is some danger of
+          // it being removed from all logged calls. This is enough of an edge case that we don't
+          // need to worry about it for now.
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          return instructionMessageHashMap.get(hash)! < formattedLoggedCalls.length;
+        });
+      }
+
+      // Stringify instructions and outputs
+      const stringifiedLoggedCalls = shuffle(formattedLoggedCalls).map((loggedCall) => ({
+        instruction: JSON.stringify(loggedCall.instruction),
+        output: JSON.stringify(loggedCall.output),
+      }));
+
+      const splitIndex = Math.floor((stringifiedLoggedCalls.length * input.testingSplit) / 100);
+
+      const testingData = stringifiedLoggedCalls.slice(0, splitIndex);
+      const trainingData = stringifiedLoggedCalls.slice(splitIndex);
+
+      // Convert arrays to JSONL format
+      const trainingDataJSONL = trainingData.map((item) => JSON.stringify(item)).join("\n");
+      const testingDataJSONL = testingData.map((item) => JSON.stringify(item)).join("\n");
+
+      const output = new WritableStreamBuffer();
+      const archive = archiver("zip");
+
+      archive.pipe(output);
+      archive.append(trainingDataJSONL, { name: "train.jsonl" });
+      archive.append(testingDataJSONL, { name: "test.jsonl" });
+      await archive.finalize();
+
+      // Convert buffer to base64
+      const base64 = output.getContents().toString("base64");
+
+      return base64;
+    }),
 });

app/src/server/api/routers/promptVariants.router.ts

@@ -93,17 +93,12 @@ export const promptVariantsRouter = createTRPCRouter({
           visible: true,
         },
       });
-      const outputCount = await prisma.scenarioVariantCell.count({
+      const finishedCount = await prisma.scenarioVariantCell.count({
         where: {
           promptVariantId: input.variantId,
           testScenario: { visible: true },
-          modelResponses: {
-            some: {
-              outdated: false,
-              respPayload: {
-                not: Prisma.AnyNull,
-              },
-            },
+          retrievalStatus: {
+            in: ["COMPLETE", "ERROR"],
           },
         },
       });
@@ -131,7 +126,7 @@ export const promptVariantsRouter = createTRPCRouter({
       const inputTokens = overallTokens._sum?.inputTokens ?? 0;
       const outputTokens = overallTokens._sum?.outputTokens ?? 0;
 
-      const awaitingCompletions = outputCount < scenarioCount;
+      const awaitingCompletions = finishedCount < scenarioCount;
 
       const awaitingEvals = !!evalResults.find(
         (result) => result.totalCount < scenarioCount * evals.length,
@@ -143,7 +138,7 @@ export const promptVariantsRouter = createTRPCRouter({
         outputTokens,
         overallCost: overallTokens._sum?.cost ?? 0,
         scenarioCount,
-        outputCount,
+        finishedCount,
         awaitingCompletions,
         awaitingEvals,
       };
@@ -196,7 +191,10 @@ export const promptVariantsRouter = createTRPCRouter({
           ? `${originalVariant?.label} Copy`
           : `Prompt Variant ${largestSortIndex + 2}`;
 
-      const newConstructFn = await deriveNewConstructFn(originalVariant);
+      const newConstructFn = await deriveNewConstructFn(
+        originalVariant,
+        originalVariant?.promptConstructor,
+      );
 
       const createNewVariantAction = prisma.promptVariant.create({
         data: {
@@ -298,6 +296,7 @@ export const promptVariantsRouter = createTRPCRouter({
     .input(
       z.object({
         id: z.string(),
+        originalPromptFn: z.string(),
         instructions: z.string().optional(),
         newModel: z
           .object({
@@ -315,22 +314,21 @@ export const promptVariantsRouter = createTRPCRouter({
       });
       await requireCanModifyExperiment(existing.experimentId, ctx);
 
-      const constructedPrompt = await parsePromptConstructor(existing.promptConstructor);
-
-      if ("error" in constructedPrompt) {
-        return error(constructedPrompt.error);
-      }
-
       const model = input.newModel
         ? modelProviders[input.newModel.provider].models[input.newModel.model]
         : undefined;
 
-      const promptConstructionFn = await deriveNewConstructFn(existing, model, input.instructions);
+      const promptConstructionFn = await deriveNewConstructFn(
+        existing,
+        input.originalPromptFn,
+        model,
+        input.instructions,
+      );
 
       // TODO: Validate promptConstructionFn
       // TODO: Record in some sort of history
 
-      return promptConstructionFn;
+      return success(promptConstructionFn);
     }),
 
   replaceVariant: protectedProcedure

app/src/server/tasks/importDatasetEntries.task.ts

@@ -0,0 +1,152 @@
+import { type DatasetFileUpload } from "@prisma/client";
+import { prisma } from "~/server/db";
+import defineTask from "./defineTask";
+import { downloadBlobToString } from "~/utils/azure/server";
+import {
+  type TrainingRow,
+  validateTrainingRows,
+  parseJSONL,
+} from "~/components/datasets/validateTrainingRows";
+import { formatEntriesFromTrainingRows } from "~/server/utils/createEntriesFromTrainingRows";
+
+export type ImportDatasetEntriesJob = {
+  datasetFileUploadId: string;
+};
+
+export const importDatasetEntries = defineTask<ImportDatasetEntriesJob>(
+  "importDatasetEntries",
+  async (task) => {
+    const { datasetFileUploadId } = task;
+    const datasetFileUpload = await prisma.datasetFileUpload.findUnique({
+      where: { id: datasetFileUploadId },
+    });
+    if (!datasetFileUpload) {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          errorMessage: "Dataset File Upload not found",
+          status: "ERROR",
+        },
+      });
+      return;
+    }
+    await prisma.datasetFileUpload.update({
+      where: { id: datasetFileUploadId },
+      data: {
+        status: "DOWNLOADING",
+        progress: 5,
+      },
+    });
+
+    const onBlobDownloadProgress = async (progress: number) => {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          progress: 5 + Math.floor((progress / datasetFileUpload.fileSize) * 25),
+        },
+      });
+    };
+
+    const jsonlStr = await downloadBlobToString(datasetFileUpload.blobName, onBlobDownloadProgress);
+
+    let trainingRows: TrainingRow[] = [];
+    let validationError: string | null = null;
+    try {
+      trainingRows = parseJSONL(jsonlStr) as TrainingRow[];
+      validationError = validateTrainingRows(trainingRows);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (e: any) {
+      validationError = e.message;
+    }
+
+    if (validationError) {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          errorMessage: `Invalid JSONL: ${validationError}`,
+          status: "ERROR",
+        },
+      });
+      return;
+    }
+
+    await prisma.datasetFileUpload.update({
+      where: { id: datasetFileUploadId },
+      data: {
+        status: "PROCESSING",
+        progress: 30,
+      },
+    });
+
+    const updatePromises: Promise<DatasetFileUpload>[] = [];
+
+    const updateCallback = async (progress: number) => {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          progress: 30 + Math.floor((progress / trainingRows.length) * 69),
+        },
+      });
+    };
+
+    let datasetEntriesToCreate;
+    try {
+      datasetEntriesToCreate = await formatEntriesFromTrainingRows(
+        datasetFileUpload.datasetId,
+        trainingRows,
+        updateCallback,
+        500,
+      );
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (e: any) {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          errorMessage: `Error formatting rows: ${e.message as string}`,
+          status: "ERROR",
+          visible: true,
+        },
+      });
+      return;
+    }
+
+    await Promise.all(updatePromises);
+
+    await prisma.datasetFileUpload.update({
+      where: { id: datasetFileUploadId },
+      data: {
+        status: "SAVING",
+        progress: 99,
+      },
+    });
+
+    await prisma.datasetEntry.createMany({
+      data: datasetEntriesToCreate,
+    });
+
+    await prisma.datasetFileUpload.update({
+      where: { id: datasetFileUploadId },
+      data: {
+        status: "COMPLETE",
+        progress: 100,
+        visible: true,
+      },
+    });
+  },
+);
+
+export const queueImportDatasetEntries = async (datasetFileUploadId: string) => {
+  await Promise.all([
+    prisma.datasetFileUpload.update({
+      where: {
+        id: datasetFileUploadId,
+      },
+      data: {
+        errorMessage: null,
+        status: "PENDING",
+      },
+    }),
+
+    importDatasetEntries.enqueue({ datasetFileUploadId }),
+  ]);
+};