Add support for CommonJS in sdk (#222)

* trying rollup

* Imports mostly working

* Remove yalc.lock

* Use tsup to build

* Disable code splitting

* Update cjs exports

* Use openpipe from npm registry in app

* Remove npmignore, fix lint

* Copy the README to dist

* Update openpipe version

* Remove second openai entrypoint

* Update openpipe version

---------

Co-authored-by: Kyle Corbitt <kyle@corbt.com>

.github/ISSUE_TEMPLATE/sweep-fast-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Fast Issue
+title: 'Sweep (fast): '
+description: For few-line fixes to be handled by Sweep, an AI-powered junior developer. Sweep will use GPT-3.5 to quickly create a PR for very small changes.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

.github/ISSUE_TEMPLATE/sweep-slow-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Slow Issue
+title: 'Sweep (slow): '
+description: For larger bugs, features, refactors, and tests to be handled by Sweep, an AI-powered junior developer. Sweep will perform a deeper search and more self-reviews but will take longer.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

.github/ISSUE_TEMPLATE/sweep-template.yml

@@ -0,0 +1,14 @@
+name: Sweep Issue
+title: 'Sweep: '
+description: For small bugs, features, refactors, and tests to be handled by Sweep, an AI-powered junior developer.
+labels: sweep
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Details
+      description: Tell Sweep where and what to edit and provide enough context for a new developer to the codebase
+      placeholder: |
+        Bugs: The bug might be in ... file. Here are the logs: ...
+        Features: the new endpoint should use the ... class from ... file because it contains ... logic.
+        Refactors: We are migrating this function to ... version because ...

README.md

@@ -16,6 +16,7 @@
   <a href='http://makeapullrequest.com'><img alt='PRs Welcome' src='https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square'/></a>
   <a href="https://github.com/openpipe/openpipe/graphs/commit-activity"><img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/openpipe/openpipe?style=flat-square"/></a>
   <a href="https://github.com/openpipe/openpipe/issues"><img alt="GitHub closed issues" src="https://img.shields.io/github/issues-closed/openpipe/openpipe?style=flat-square"/></a>
+ <img src="https://img.shields.io/badge/Y%20Combinator-S23-orange?style=flat-square" alt="Y Combinator S23">
 </p>
 
 <p align="center">
@@ -27,7 +28,7 @@ Use powerful but expensive LLMs to fine-tune smaller and cheaper models suited t
 <br>
 
 
-## 🪛 Features
+## Features
 
  * <b>Experiment</b>
    * Bulk-test wide-reaching scenarios using code templating.

app/.env.example

@@ -40,3 +40,8 @@ SMTP_HOST="placeholder"
 SMTP_PORT="placeholder"
 SMTP_LOGIN="placeholder"
 SMTP_PASSWORD="placeholder"
+
+# Azure credentials are necessary for uploading large training data files
+AZURE_STORAGE_ACCOUNT_NAME="placeholder"
+AZURE_STORAGE_ACCOUNT_KEY="placeholder"
+AZURE_STORAGE_CONTAINER_NAME="placeholder"

app/.gitignore

@@ -47,3 +47,7 @@ yarn-error.log*
 
 # custom openai intialization
 src/server/utils/openaiCustomConfig.json
+
+# yalc
+.yalc
+yalc.lock

app/@types/nextjs-routes.d.ts

@@ -19,6 +19,8 @@ declare module "nextjs-routes" {
     | DynamicRoute<"/api/v1/[...trpc]", { "trpc": string[] }>
     | StaticRoute<"/api/v1/openapi">
     | StaticRoute<"/dashboard">
+    | DynamicRoute<"/datasets/[id]", { "id": string }>
+    | StaticRoute<"/datasets">
     | DynamicRoute<"/experiments/[experimentSlug]", { "experimentSlug": string }>
     | StaticRoute<"/experiments">
     | StaticRoute<"/fine-tunes">

app/package.json

@@ -26,6 +26,8 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.5.8",
     "@apidevtools/json-schema-ref-parser": "^10.1.0",
+    "@azure/identity": "^3.3.0",
+    "@azure/storage-blob": "12.15.0",
     "@babel/standalone": "^7.22.9",
     "@chakra-ui/anatomy": "^2.2.0",
     "@chakra-ui/next-js": "^2.1.4",
@@ -69,6 +71,7 @@
     "jsonschema": "^1.4.1",
     "kysely": "^0.26.1",
     "kysely-codegen": "^0.10.1",
+    "llama-tokenizer-js": "^1.1.3",
     "lodash-es": "^4.17.21",
     "lucide-react": "^0.265.0",
     "marked": "^7.0.3",
@@ -79,7 +82,8 @@
     "nextjs-routes": "^2.0.1",
     "nodemailer": "^6.9.4",
     "openai": "4.0.0-beta.7",
-    "openpipe": "workspace:*",
+    "openpipe": "0.4.0-beta.1",
+    "openpipe-dev": "workspace:^",
     "pg": "^8.11.2",
     "pluralize": "^8.0.0",
     "posthog-js": "^1.75.3",

app/prisma/migrations/20230905233758_revamp_dataset_entry/migration.sql

@@ -0,0 +1,26 @@
+/*
+  Warnings:
+
+  - Added the required column `inputTokens` to the `DatasetEntry` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `outputTokens` to the `DatasetEntry` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `type` to the `DatasetEntry` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- CreateEnum
+CREATE TYPE "DatasetEntryType" AS ENUM ('TRAIN', 'TEST');
+
+-- AlterTable
+ALTER TABLE "Dataset" ADD COLUMN     "trainingRatio" DOUBLE PRECISION NOT NULL DEFAULT 0.8;
+
+-- AlterTable
+ALTER TABLE "DatasetEntry" ADD COLUMN     "input" JSONB NOT NULL DEFAULT '[]',
+ADD COLUMN     "inputTokens" INTEGER NOT NULL DEFAULT 0,
+ADD COLUMN     "output" JSONB,
+ADD COLUMN     "outputTokens" INTEGER NOT NULL DEFAULT 0,
+ADD COLUMN     "type" "DatasetEntryType" NOT NULL DEFAULT 'TRAIN';
+
+-- CreateIndex
+CREATE INDEX "DatasetEntry_datasetId_createdAt_id_idx" ON "DatasetEntry"("datasetId", "createdAt", "id");
+
+-- CreateIndex
+CREATE INDEX "DatasetEntry_datasetId_type_idx" ON "DatasetEntry"("datasetId", "type");

app/prisma/migrations/20230906052922_make_dataset_entry_logged_call_optional/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "DatasetEntry" ALTER COLUMN "loggedCallId" DROP NOT NULL,
+ALTER COLUMN "inputTokens" DROP DEFAULT,
+ALTER COLUMN "outputTokens" DROP DEFAULT,
+ALTER COLUMN "type" DROP DEFAULT;

app/prisma/migrations/20230907120707_add_dataset_file_upload/migration.sql

@@ -0,0 +1,23 @@
+-- CreateEnum
+CREATE TYPE "DatasetFileUploadStatus" AS ENUM ('PENDING', 'DOWNLOADING', 'PROCESSING', 'SAVING', 'COMPLETE', 'ERROR');
+
+-- CreateTable
+CREATE TABLE "DatasetFileUpload" (
+    "id" UUID NOT NULL,
+    "datasetId" UUID NOT NULL,
+    "blobName" TEXT NOT NULL,
+    "fileName" TEXT NOT NULL,
+    "fileSize" INTEGER NOT NULL,
+    "progress" INTEGER NOT NULL DEFAULT 0,
+    "status" "DatasetFileUploadStatus" NOT NULL DEFAULT 'PENDING',
+    "uploadedAt" TIMESTAMP(3) NOT NULL,
+    "visible" BOOLEAN NOT NULL DEFAULT true,
+    "errorMessage" TEXT,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "DatasetFileUpload_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "DatasetFileUpload" ADD CONSTRAINT "DatasetFileUpload_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/schema.prisma

@@ -176,12 +176,42 @@ model OutputEvaluation {
     @@unique([modelResponseId, evaluationId])
 }
 
+
+enum DatasetFileUploadStatus {
+    PENDING
+    DOWNLOADING
+    PROCESSING
+    SAVING
+    COMPLETE
+    ERROR
+}
+
+model DatasetFileUpload {
+    id String @id @default(uuid()) @db.Uuid
+
+    datasetId String   @db.Uuid
+    dataset   Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    blobName  String
+    fileName String
+    fileSize Int
+    progress Int @default(0) // Percentage
+    status   DatasetFileUploadStatus @default(PENDING)
+    uploadedAt DateTime
+    visible  Boolean @default(true)
+    errorMessage String?
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
 model Dataset {
     id String @id @default(uuid()) @db.Uuid
 
-    name           String
+    name                String
-    datasetEntries DatasetEntry[]
+    datasetEntries      DatasetEntry[]
-    fineTunes      FineTune[]
+    fineTunes           FineTune[]
+    datasetFileUploads  DatasetFileUpload[]
+    trainingRatio  Float @default(0.8)
 
     projectId String  @db.Uuid
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -190,17 +220,32 @@ model Dataset {
     updatedAt DateTime @updatedAt
 }
 
+enum DatasetEntryType {
+    TRAIN
+    TEST
+}
+
 model DatasetEntry {
     id String @id @default(uuid()) @db.Uuid
 
-    loggedCallId String @db.Uuid
+    loggedCallId String? @db.Uuid
-    loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
+    loggedCall   LoggedCall? @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
+
+    input        Json  @default("[]")
+    output       Json?
+    inputTokens  Int
+    outputTokens Int
+
+    type DatasetEntryType
 
     datasetId String   @db.Uuid
     dataset   Dataset? @relation(fields: [datasetId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
+
+    @@index([datasetId, createdAt, id])
+    @@index([datasetId, type])
 }
 
 model Project {
@@ -452,7 +497,7 @@ model FineTune {
     deploymentFinishedAt DateTime?
 
     datasetId String   @db.Uuid
-    dataset   Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    dataset    Dataset @relation(fields: [datasetId], references: [id], onDelete: Cascade)
 
     projectId String  @db.Uuid
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)

app/prisma/seedAgiEval.ts

@@ -1,5 +1,4 @@
 import { prisma } from "~/server/db";
-import { generateNewCell } from "~/server/utils/generateNewCell";
 import dedent from "dedent";
 import { execSync } from "child_process";
 import fs from "fs";

app/prisma/seedDashboard.ts

@@ -108,7 +108,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 236,
     outputTokens: 5,
     finishReason: "stop",
-    tags: [{ name: "prompt_id", value: "define_func" }],
+    tags: [{ name: "prompt_id", value: "add_scenario" }],
   },
   {
     reqPayload: {
@@ -311,7 +311,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     outputTokens: 108,
     finishReason: "stop",
     tags: [
-      { name: "prompt_id", value: "chatcmpl-7" },
+      { name: "prompt_id", value: "define_func" },
       { name: "some_other_tag", value: "some_other_value" },
     ],
   },

app/src/components/ActionButton.tsx

@@ -0,0 +1,55 @@
+import { useState } from "react";
+
+import { Button, HStack, type ButtonProps, Icon, Text } from "@chakra-ui/react";
+import { type IconType } from "react-icons";
+import { useAppStore } from "~/state/store";
+import { BetaModal } from "./BetaModal";
+
+const ActionButton = ({
+  icon,
+  iconBoxSize = 3.5,
+  label,
+  requireBeta = false,
+  onClick,
+  ...buttonProps
+}: {
+  icon: IconType;
+  iconBoxSize?: number;
+  label: string;
+  requireBeta?: boolean;
+  onClick?: () => void;
+} & ButtonProps) => {
+  const flags = useAppStore((s) => s.featureFlags.featureFlags);
+  const flagsLoaded = useAppStore((s) => s.featureFlags.flagsLoaded);
+
+  const [betaModalOpen, setBetaModalOpen] = useState(false);
+
+  const isBetaBlocked = requireBeta && flagsLoaded && !flags.betaAccess;
+  return (
+    <>
+      <Button
+        colorScheme="blue"
+        color="black"
+        bgColor="white"
+        borderColor="gray.300"
+        borderRadius={4}
+        variant="outline"
+        size="sm"
+        fontSize="sm"
+        fontWeight="normal"
+        onClick={isBetaBlocked ? () => setBetaModalOpen(true) : onClick}
+        {...buttonProps}
+      >
+        <HStack spacing={1}>
+          {icon && (
+            <Icon as={icon} boxSize={iconBoxSize} color={requireBeta ? "orange.400" : undefined} />
+          )}
+          <Text display={{ base: "none", md: "flex" }}>{label}</Text>
+        </HStack>
+      </Button>
+      <BetaModal isOpen={betaModalOpen} onClose={() => setBetaModalOpen(false)} />
+    </>
+  );
+};
+
+export default ActionButton;

app/src/components/BetaModal.tsx

@@ -13,19 +13,17 @@ import {
   Link,
 } from "@chakra-ui/react";
 import { BsStars } from "react-icons/bs";
-import { useRouter } from "next/router";
 import { useSession } from "next-auth/react";
 
-export const BetaModal = () => {
+export const BetaModal = ({ isOpen, onClose }: { isOpen: boolean; onClose: () => void }) => {
-  const router = useRouter();
   const session = useSession();
 
   const email = session.data?.user.email ?? "";
 
   return (
     <Modal
-      isOpen
+      isOpen={isOpen}
-      onClose={router.back}
+      onClose={onClose}
       closeOnOverlayClick={false}
       size={{ base: "xl", md: "2xl" }}
     >
@@ -56,7 +54,7 @@ export const BetaModal = () => {
             >
               Join Waitlist
             </Button>
-            <Button colorScheme="blue" onClick={router.back}>
+            <Button colorScheme="blue" onClick={onClose}>
               Done
             </Button>
           </HStack>

app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -1,74 +0,0 @@
-import {
-  Button,
-  Icon,
-  AlertDialog,
-  AlertDialogBody,
-  AlertDialogFooter,
-  AlertDialogHeader,
-  AlertDialogContent,
-  AlertDialogOverlay,
-  useDisclosure,
-  Text,
-} from "@chakra-ui/react";
-
-import { useRouter } from "next/router";
-import { useRef } from "react";
-import { BsTrash } from "react-icons/bs";
-import { useAppStore } from "~/state/store";
-import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-
-export const DeleteButton = () => {
-  const experiment = useExperiment();
-  const mutation = api.experiments.delete.useMutation();
-  const utils = api.useContext();
-  const router = useRouter();
-
-  const closeDrawer = useAppStore((s) => s.closeDrawer);
-
-  const { isOpen, onOpen, onClose } = useDisclosure();
-  const cancelRef = useRef<HTMLButtonElement>(null);
-
-  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id) return;
-    await mutation.mutateAsync({ id: experiment.data.id });
-    await utils.experiments.list.invalidate();
-    await router.push({ pathname: "/experiments" });
-    closeDrawer();
-
-    onClose();
-  }, [mutation, experiment.data?.id, router]);
-
-  return (
-    <>
-      <Button size="sm" variant="ghost" colorScheme="red" fontWeight="normal" onClick={onOpen}>
-        <Icon as={BsTrash} boxSize={4} />
-        <Text ml={2}>Delete Experiment</Text>
-      </Button>
-
-      <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
-        <AlertDialogOverlay>
-          <AlertDialogContent>
-            <AlertDialogHeader fontSize="lg" fontWeight="bold">
-              Delete Experiment
-            </AlertDialogHeader>
-
-            <AlertDialogBody>
-              If you delete this experiment all the associated prompts and scenarios will be deleted
-              as well. Are you sure?
-            </AlertDialogBody>
-
-            <AlertDialogFooter>
-              <Button ref={cancelRef} onClick={onClose}>
-                Cancel
-              </Button>
-              <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
-                Delete
-              </Button>
-            </AlertDialogFooter>
-          </AlertDialogContent>
-        </AlertDialogOverlay>
-      </AlertDialog>
-    </>
-  );
-};

app/src/components/InputDropdown.tsx

@@ -16,12 +16,16 @@ import {
 
 import { FiChevronDown } from "react-icons/fi";
 import { BiCheck } from "react-icons/bi";
+import { isEqual } from "lodash-es";
+import React from "react";
 
 type InputDropdownProps<T> = {
   options: ReadonlyArray<T>;
   selectedOption: T;
   onSelect: (option: T) => void;
   inputGroupProps?: InputGroupProps;
+  getDisplayLabel?: (option: T) => string;
+  isDisabled?: boolean;
 };
 
 const InputDropdown = <T,>({
@@ -29,19 +33,21 @@ const InputDropdown = <T,>({
   selectedOption,
   onSelect,
   inputGroupProps,
+  getDisplayLabel = (option) => option as string,
+  isDisabled,
 }: InputDropdownProps<T>) => {
-  const popover = useDisclosure();
+  const { onOpen, ...popover } = useDisclosure();
 
   return (
-    <Popover placement="bottom-start" {...popover}>
+    <Popover placement="bottom-start" onOpen={isDisabled ? undefined : onOpen} {...popover}>
       <PopoverTrigger>
         <InputGroup
           cursor="pointer"
-          w={(selectedOption as string).length * 14 + 180}
+          w={getDisplayLabel(selectedOption).length * 14 + 180}
           {...inputGroupProps}
         >
           <Input
-            value={selectedOption as string}
+            value={getDisplayLabel(selectedOption)}
             // eslint-disable-next-line @typescript-eslint/no-empty-function -- controlled input requires onChange
             onChange={() => {}}
             cursor="pointer"
@@ -52,9 +58,10 @@ const InputDropdown = <T,>({
             onFocus={(e) => {
               e.target.blur();
             }}
+            isDisabled={isDisabled}
           />
           <InputRightElement>
-            <Icon as={FiChevronDown} />
+            <Icon as={FiChevronDown} color={isDisabled ? "gray.300" : undefined} />
           </InputRightElement>
         </InputGroup>
       </PopoverTrigger>
@@ -78,8 +85,10 @@ const InputDropdown = <T,>({
               fontSize="sm"
               borderBottomWidth={1}
             >
-              <Text mr={16}>{option as string}</Text>
+              <Text mr={16}>{getDisplayLabel(option)}</Text>
-              {option === selectedOption && <Icon as={BiCheck} color="blue.500" boxSize={5} />}
+              {isEqual(option, selectedOption) && (
+                <Icon as={BiCheck} color="blue.500" boxSize={5} />
+              )}
             </HStack>
           ))}
         </VStack>

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -19,15 +19,13 @@ import {
   useScenarios,
 } from "~/utils/hooks";
 import { BsGear, BsPencil, BsPlus, BsStars } from "react-icons/bs";
-import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 
 export const ActionButton = (props: ButtonProps) => (
   <Button size="sm" variant="ghost" color="gray.600" {...props} />
 );
 
-export const ScenariosHeader = () => {
+export const ScenariosHeader = ({ openDrawer }: { openDrawer: () => void }) => {
-  const openDrawer = useAppStore((s) => s.openDrawer);
   const { canModify } = useExperimentAccess();
   const scenarios = useScenarios();
 

app/src/components/OutputsTable/VariantStats.tsx

@@ -20,7 +20,7 @@ export default function VariantStats(props: { variant: PromptVariant }) {
         inputTokens: 0,
         outputTokens: 0,
         scenarioCount: 0,
-        outputCount: 0,
+        finishedCount: 0,
         awaitingCompletions: false,
         awaitingEvals: false,
       },
@@ -42,7 +42,7 @@ export default function VariantStats(props: { variant: PromptVariant }) {
 
   const scale = chroma.scale([failColor, neutralColor, passColor]).domain([0, 0.5, 1]);
 
-  const showNumFinished = data.scenarioCount > 0 && data.scenarioCount !== data.outputCount;
+  const showNumFinished = data.scenarioCount > 0 && data.scenarioCount !== data.finishedCount;
 
   return (
     <HStack
@@ -55,7 +55,7 @@ export default function VariantStats(props: { variant: PromptVariant }) {
       <HStack px={cellPadding.x} flexWrap="wrap">
         {showNumFinished && (
           <Text>
-            {data.outputCount} / {data.scenarioCount}
+            {data.finishedCount} / {data.scenarioCount}
           </Text>
         )}
         {data.evalResults.map((result) => {

app/src/components/OutputsTable/index.tsx

@@ -12,7 +12,13 @@ import ScenarioPaginator from "./ScenarioPaginator";
 import { Fragment } from "react";
 import useScrolledPast from "./useHasScrolledPast";
 
-export default function OutputsTable({ experimentId }: { experimentId: string | undefined }) {
+export default function OutputsTable({
+  experimentId,
+  openDrawer,
+}: {
+  experimentId: string | undefined;
+  openDrawer: () => void;
+}) {
   const variants = api.promptVariants.list.useQuery(
     { experimentId: experimentId as string },
     { enabled: !!experimentId },
@@ -91,7 +97,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
         colStart={1}
         borderRightWidth={0}
       >
-        <ScenariosHeader />
+        <ScenariosHeader openDrawer={openDrawer} />
       </GridItem>
 
       {scenarios.data.scenarios.map((scenario, i) => (

app/src/components/StatsCard.tsx

@@ -1,26 +0,0 @@
-import { VStack, HStack, type StackProps, Text, Divider } from "@chakra-ui/react";
-import Link, { type LinkProps } from "next/link";
-
-const StatsCard = ({
-  title,
-  href,
-  children,
-  ...rest
-}: { title: string; href: string } & StackProps & LinkProps) => {
-  return (
-    <VStack flex={1} borderWidth={1} padding={4} borderRadius={4} borderColor="gray.300" {...rest}>
-      <HStack w="full" justifyContent="space-between">
-        <Text fontSize="md" fontWeight="bold">
-          {title}
-        </Text>
-        <Link href={href}>
-          <Text color="blue">View all</Text>
-        </Link>
-      </HStack>
-      <Divider />
-      {children}
-    </VStack>
-  );
-};
-
-export default StatsCard;

app/src/components/dashboard/LoggedCallsTable.tsx

@@ -2,11 +2,12 @@ import { Card, CardHeader, Heading, Table, Tbody, HStack, Button, Text } from "@
 import { useState } from "react";
 import Link from "next/link";
 import { useLoggedCalls } from "~/utils/hooks";
-import { TableHeader, TableRow } from "../requestLogs/TableRow";
+import { EmptyTableRow, TableHeader, TableRow } from "../requestLogs/TableRow";
 
 export default function LoggedCallsTable() {
+  const { data: loggedCalls } = useLoggedCalls(false);
+
   const [expandedRow, setExpandedRow] = useState<string | null>(null);
-  const { data: loggedCalls } = useLoggedCalls();
 
   return (
     <Card width="100%" overflow="hidden">
@@ -23,22 +24,26 @@ export default function LoggedCallsTable() {
       <Table>
         <TableHeader />
         <Tbody>
-          {loggedCalls?.calls.map((loggedCall) => {
+          {loggedCalls?.calls.length ? (
-            return (
+            loggedCalls?.calls.map((loggedCall) => {
-              <TableRow
+              return (
-                key={loggedCall.id}
+                <TableRow
-                loggedCall={loggedCall}
+                  key={loggedCall.id}
-                isExpanded={loggedCall.id === expandedRow}
+                  loggedCall={loggedCall}
-                onToggle={() => {
+                  isExpanded={loggedCall.id === expandedRow}
-                  if (loggedCall.id === expandedRow) {
+                  onToggle={() => {
-                    setExpandedRow(null);
+                    if (loggedCall.id === expandedRow) {
-                  } else {
+                      setExpandedRow(null);
-                    setExpandedRow(loggedCall.id);
+                    } else {
-                  }
+                      setExpandedRow(loggedCall.id);
-                }}
+                    }
-              />
+                  }}
-            );
+                />
-          })}
+              );
+            })
+          ) : (
+            <EmptyTableRow filtersApplied={false} />
+          )}
         </Tbody>
       </Table>
     </Card>

app/src/components/datasets/DatasetConfigurationDrawer/DatasetConfigurationDrawer.tsx

@@ -0,0 +1,37 @@
+import {
+  Drawer,
+  DrawerBody,
+  DrawerCloseButton,
+  DrawerContent,
+  DrawerHeader,
+  DrawerOverlay,
+  Heading,
+  VStack,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+
+import { DeleteButton } from "./DeleteButton";
+
+export default function DatasetConfigurationDrawer({
+  disclosure,
+}: {
+  disclosure: UseDisclosureReturn;
+}) {
+  return (
+    <Drawer placement="right" size="md" {...disclosure}>
+      <DrawerOverlay />
+      <DrawerContent>
+        <DrawerCloseButton />
+        <DrawerHeader>
+          <Heading size="md">Dataset Configuration</Heading>
+        </DrawerHeader>
+        <DrawerBody h="full" pb={4}>
+          <VStack h="full" justifyContent="space-between">
+            <VStack spacing={6}></VStack>
+            <DeleteButton closeDrawer={disclosure.onClose} />
+          </VStack>
+        </DrawerBody>
+      </DrawerContent>
+    </Drawer>
+  );
+}

app/src/components/datasets/DatasetConfigurationDrawer/DeleteButton.tsx

@@ -0,0 +1,39 @@
+import { Button, Icon, useDisclosure, Text } from "@chakra-ui/react";
+import { useRouter } from "next/router";
+import { BsTrash } from "react-icons/bs";
+
+import { useHandledAsyncCallback, useDataset } from "~/utils/hooks";
+import DeleteDatasetDialog from "./DeleteDatasetDialog";
+
+export const DeleteButton = ({ closeDrawer }: { closeDrawer: () => void }) => {
+  const dataset = useDataset();
+  const router = useRouter();
+
+  const disclosure = useDisclosure();
+
+  const [onDelete] = useHandledAsyncCallback(async () => {
+    await router.push({ pathname: "/datasets" });
+    closeDrawer();
+  }, [router, closeDrawer]);
+
+  return (
+    <>
+      <Button
+        size="sm"
+        variant="ghost"
+        colorScheme="red"
+        fontWeight="normal"
+        onClick={disclosure.onOpen}
+      >
+        <Icon as={BsTrash} boxSize={4} />
+        <Text ml={2}>Delete Dataset</Text>
+      </Button>
+
+      <DeleteDatasetDialog
+        datasetId={dataset.data?.id}
+        onDelete={onDelete}
+        disclosure={disclosure}
+      />
+    </>
+  );
+};

app/src/components/datasets/DatasetConfigurationDrawer/DeleteDatasetDialog.tsx

@@ -0,0 +1,73 @@
+import { useRef } from "react";
+import {
+  type UseDisclosureReturn,
+  AlertDialog,
+  AlertDialogOverlay,
+  AlertDialogContent,
+  AlertDialogHeader,
+  AlertDialogBody,
+  AlertDialogFooter,
+  Button,
+} from "@chakra-ui/react";
+import { api } from "~/utils/api";
+
+import { useHandledAsyncCallback } from "~/utils/hooks";
+
+const DeleteDatasetDialog = ({
+  datasetId,
+  onDelete,
+  disclosure,
+}: {
+  datasetId?: string;
+  onDelete?: () => void;
+  disclosure: UseDisclosureReturn;
+}) => {
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const mutation = api.datasets.delete.useMutation();
+  const utils = api.useContext();
+
+  const [onDeleteConfirm, deletionInProgress] = useHandledAsyncCallback(async () => {
+    if (!datasetId) return;
+    await mutation.mutateAsync({ id: datasetId });
+    await utils.datasets.list.invalidate();
+    onDelete?.();
+
+    disclosure.onClose();
+  }, [mutation, datasetId, disclosure.onClose]);
+
+  console.log("dataset id", datasetId);
+
+  return (
+    <AlertDialog leastDestructiveRef={cancelRef} {...disclosure}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Delete Dataset
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            If you delete this dataset all the associated dataset entries will be deleted as well.
+            Are you sure?
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={disclosure.onClose}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="red"
+              isLoading={deletionInProgress}
+              onClick={onDeleteConfirm}
+              ml={3}
+            >
+              Delete
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};
+
+export default DeleteDatasetDialog;

app/src/components/datasets/DatasetEntriesTable/DatasetEntriesTable.tsx

@@ -0,0 +1,46 @@
+import { Card, Table, Tbody } from "@chakra-ui/react";
+import { useState } from "react";
+import { useDatasetEntries } from "~/utils/hooks";
+import { TableHeader, TableRow, EmptyTableRow } from "./TableRow";
+import DatasetEntryEditorDrawer from "./DatasetEntryEditorDrawer";
+
+export default function DatasetEntriesTable() {
+  const [expandedDatasetEntryId, setExpandedDatasetEntryId] = useState<string | null>(null);
+  const datasetEntries = useDatasetEntries().data?.entries;
+
+  return (
+    <>
+      <Card width="100%" overflowX="auto">
+        <Table>
+          <TableHeader />
+          <Tbody>
+            {datasetEntries?.length ? (
+              datasetEntries?.map((entry) => {
+                return (
+                  <TableRow
+                    key={entry.id}
+                    datasetEntry={entry}
+                    onToggle={() => {
+                      if (entry.id === expandedDatasetEntryId) {
+                        setExpandedDatasetEntryId(null);
+                      } else {
+                        setExpandedDatasetEntryId(entry.id);
+                      }
+                    }}
+                    showOptions
+                  />
+                );
+              })
+            ) : (
+              <EmptyTableRow />
+            )}
+          </Tbody>
+        </Table>
+      </Card>
+      <DatasetEntryEditorDrawer
+        datasetEntryId={expandedDatasetEntryId}
+        clearDatasetEntryId={() => setExpandedDatasetEntryId(null)}
+      />
+    </>
+  );
+}

app/src/components/datasets/DatasetEntriesTable/DatasetEntryEditorDrawer.tsx

@@ -0,0 +1,174 @@
+import { useState, useEffect, useMemo } from "react";
+import {
+  Drawer,
+  DrawerBody,
+  DrawerCloseButton,
+  DrawerContent,
+  DrawerHeader,
+  DrawerOverlay,
+  DrawerFooter,
+  Heading,
+  VStack,
+  HStack,
+  Button,
+  Text,
+  Divider,
+  Icon,
+} from "@chakra-ui/react";
+import { type CreateChatCompletionRequestMessage } from "openai/resources/chat";
+import { BsPlus } from "react-icons/bs";
+import { type DatasetEntryType } from "@prisma/client";
+
+import { api } from "~/utils/api";
+import { useDatasetEntry, useHandledAsyncCallback } from "~/utils/hooks";
+import EditableMessage from "./EditableMessage";
+import EntryTypeDropdown from "./EntryTypeDropdown";
+
+export default function DatasetDentryEditorDrawer({
+  datasetEntryId,
+  clearDatasetEntryId,
+}: {
+  datasetEntryId: string | null;
+  clearDatasetEntryId: () => void;
+}) {
+  const utils = api.useContext();
+
+  const datasetEntry = useDatasetEntry(datasetEntryId).data;
+
+  const savedInputMessages = useMemo(
+    () => datasetEntry?.input as unknown as CreateChatCompletionRequestMessage[],
+    [datasetEntry],
+  );
+  const savedOutputMessage = useMemo(
+    () => datasetEntry?.output as unknown as CreateChatCompletionRequestMessage,
+    [datasetEntry],
+  );
+
+  const [inputMessagesToSave, setInputMessagesToSave] = useState<
+    CreateChatCompletionRequestMessage[]
+  >([]);
+  const [outputMessageToSave, setOutputMessageToSave] =
+    useState<CreateChatCompletionRequestMessage | null>(null);
+
+  useEffect(() => {
+    if (savedInputMessages) {
+      setInputMessagesToSave(savedInputMessages);
+      setOutputMessageToSave(savedOutputMessage);
+    }
+  }, [savedInputMessages, savedOutputMessage]);
+
+  const updateMutation = api.datasetEntries.update.useMutation();
+  const [onSave, savingInProgress] = useHandledAsyncCallback(async () => {
+    if (!datasetEntryId || !inputMessagesToSave) return;
+    await updateMutation.mutateAsync({
+      id: datasetEntryId,
+      updates: {
+        input: JSON.stringify(inputMessagesToSave),
+        output: JSON.stringify(outputMessageToSave),
+      },
+    });
+    await utils.datasetEntries.list.invalidate();
+    await utils.datasetEntries.get.invalidate({ id: datasetEntryId });
+  }, [updateMutation, datasetEntryId, inputMessagesToSave, outputMessageToSave, utils]);
+
+  const [onUpdateType] = useHandledAsyncCallback(
+    async (type: DatasetEntryType) => {
+      if (!datasetEntryId) return;
+      await updateMutation.mutateAsync({
+        id: datasetEntryId,
+        updates: {
+          type,
+        },
+      });
+      await utils.datasetEntries.list.invalidate();
+      await utils.datasetEntries.get.invalidate({ id: datasetEntryId });
+    },
+    [updateMutation, datasetEntryId, utils],
+  );
+
+  return (
+    <Drawer isOpen={!!datasetEntryId} onClose={clearDatasetEntryId} placement="right" size="md">
+      <DrawerOverlay />
+      <DrawerContent>
+        <DrawerCloseButton pt={6} />
+        <DrawerHeader bgColor="orange.50">
+          <HStack w="full" justifyContent="space-between" pr={8}>
+            <Heading size="md">Dataset Entry</Heading>
+            {datasetEntry && (
+              <EntryTypeDropdown type={datasetEntry.type} onTypeChange={onUpdateType} />
+            )}
+          </HStack>
+        </DrawerHeader>
+        <DrawerBody h="full" pb={4} bgColor="orange.50">
+          <VStack h="full" justifyContent="space-between">
+            <VStack w="full" spacing={12} py={4}>
+              <VStack w="full" alignItems="flex-start">
+                <Text fontWeight="bold">Input</Text>
+                {inputMessagesToSave.map((message, i) => {
+                  return (
+                    <>
+                      <Divider key={`divider-${i}`} my={4} />
+                      <EditableMessage
+                        key={i}
+                        message={message}
+                        onEdit={(message) => {
+                          const newInputMessages = [...inputMessagesToSave];
+                          newInputMessages[i] = message;
+                          setInputMessagesToSave(newInputMessages);
+                        }}
+                        onDelete={() => {
+                          const newInputMessages = [...inputMessagesToSave];
+                          newInputMessages.splice(i, 1);
+                          setInputMessagesToSave(newInputMessages);
+                        }}
+                      />
+                    </>
+                  );
+                })}
+                <Divider my={4} />
+                <Button
+                  w="full"
+                  onClick={() =>
+                    setInputMessagesToSave([...inputMessagesToSave, { role: "user", content: "" }])
+                  }
+                  variant="outline"
+                  color="gray.500"
+                  _hover={{ bgColor: "orange.100" }}
+                >
+                  <HStack spacing={0}>
+                    <Text>Add Message</Text>
+                    <Icon as={BsPlus} boxSize={6} />
+                  </HStack>
+                </Button>
+              </VStack>
+              <VStack w="full" alignItems="flex-start">
+                <Text fontWeight="bold">Output</Text>
+                <Divider my={4} />
+                <EditableMessage
+                  message={outputMessageToSave}
+                  onEdit={(message) => setOutputMessageToSave(message)}
+                  isOutput
+                />
+              </VStack>
+            </VStack>
+          </VStack>
+        </DrawerBody>
+        <DrawerFooter bgColor="orange.50">
+          <HStack>
+            <Button
+              onClick={() => {
+                setInputMessagesToSave(savedInputMessages);
+                setOutputMessageToSave(savedOutputMessage);
+              }}
+            >
+              Reset
+            </Button>
+            <Button isLoading={savingInProgress} onClick={onSave} colorScheme="orange">
+              Save
+            </Button>
+          </HStack>
+        </DrawerFooter>
+      </DrawerContent>
+    </Drawer>
+  );
+}

app/src/components/datasets/DatasetEntriesTable/EditableMessage.tsx

@@ -0,0 +1,105 @@
+import { VStack, HStack, Tooltip, IconButton, Icon } from "@chakra-ui/react";
+import { type CreateChatCompletionRequestMessage } from "openai/resources/chat";
+import { BsX } from "react-icons/bs";
+
+import AutoResizeTextArea from "~/components/AutoResizeTextArea";
+import InputDropdown from "~/components/InputDropdown";
+import { parseableToFunctionCall } from "~/utils/utils";
+import FunctionCallEditor from "./FunctionCallEditor";
+
+const MESSAGE_ROLE_OPTIONS = ["system", "user", "assistant", "function"] as const;
+const OUTPUT_OPTIONS = ["plaintext", "func_call"] as const;
+
+const EditableMessage = ({
+  message,
+  onEdit,
+  onDelete,
+  isOutput,
+}: {
+  message: CreateChatCompletionRequestMessage | null;
+  onEdit: (message: CreateChatCompletionRequestMessage) => void;
+  onDelete?: () => void;
+  isOutput?: boolean;
+}) => {
+  const { role = "assistant", content = "", function_call } = message || {};
+
+  const currentOutputOption: (typeof OUTPUT_OPTIONS)[number] = function_call
+    ? "func_call"
+    : "plaintext";
+
+  return (
+    <VStack w="full">
+      <HStack w="full" justifyContent="space-between">
+        <HStack>
+          {!isOutput && (
+            <InputDropdown
+              options={MESSAGE_ROLE_OPTIONS}
+              selectedOption={role}
+              onSelect={(option) => {
+                const updatedMessage = { role: option, content };
+                if (role === "assistant" && currentOutputOption === "func_call") {
+                  updatedMessage.content = JSON.stringify(function_call, null, 2);
+                }
+                onEdit(updatedMessage);
+              }}
+              inputGroupProps={{ w: "32", bgColor: "white" }}
+            />
+          )}
+          {role === "assistant" && (
+            <InputDropdown
+              options={OUTPUT_OPTIONS}
+              selectedOption={currentOutputOption}
+              onSelect={(option) => {
+                const updatedMessage: CreateChatCompletionRequestMessage = {
+                  role,
+                  content: null,
+                  function_call: undefined,
+                };
+                if (option === "plaintext") {
+                  updatedMessage.content = JSON.stringify(function_call, null, 2);
+                } else if (option === "func_call") {
+                  updatedMessage.function_call =
+                    content && parseableToFunctionCall(content)
+                      ? JSON.parse(content)
+                      : { name: "", arguments: "{}" };
+                }
+                onEdit(updatedMessage);
+              }}
+              inputGroupProps={{ w: "32", bgColor: "white" }}
+            />
+          )}
+        </HStack>
+        {!isOutput && (
+          <HStack>
+            <Tooltip label="Delete" hasArrow>
+              <IconButton
+                aria-label="Delete"
+                icon={<Icon as={BsX} boxSize={6} />}
+                onClick={onDelete}
+                size="xs"
+                display="flex"
+                colorScheme="gray"
+                color="gray.500"
+                variant="ghost"
+              />
+            </Tooltip>
+          </HStack>
+        )}
+      </HStack>
+      {function_call ? (
+        <FunctionCallEditor
+          function_call={function_call}
+          onEdit={(function_call) => onEdit({ role, function_call, content: null })}
+        />
+      ) : (
+        <AutoResizeTextArea
+          value={content || JSON.stringify(function_call, null, 2)}
+          onChange={(e) => onEdit({ role, content: e.target.value })}
+          bgColor="white"
+        />
+      )}
+    </VStack>
+  );
+};
+
+export default EditableMessage;

app/src/components/datasets/DatasetEntriesTable/EntryTypeDropdown.tsx

@@ -0,0 +1,24 @@
+import { type DatasetEntryType } from "@prisma/client";
+
+import InputDropdown from "~/components/InputDropdown";
+
+const ENTRY_TYPE_OPTIONS: DatasetEntryType[] = ["TRAIN", "TEST"];
+
+const EntryTypeDropdown = ({
+  type,
+  onTypeChange,
+}: {
+  type: DatasetEntryType;
+  onTypeChange: (type: DatasetEntryType) => void;
+}) => {
+  return (
+    <InputDropdown
+      options={ENTRY_TYPE_OPTIONS}
+      selectedOption={type}
+      onSelect={onTypeChange}
+      inputGroupProps={{ w: "32", bgColor: "white" }}
+    />
+  );
+};
+
+export default EntryTypeDropdown;

app/src/components/datasets/DatasetEntriesTable/FunctionCallEditor.tsx

@@ -0,0 +1,125 @@
+import { useRef, useMemo, useEffect } from "react";
+import { VStack, HStack, Text, Input, Box } from "@chakra-ui/react";
+import { type CreateChatCompletionRequestMessage } from "openai/resources/chat";
+
+import { useAppStore } from "~/state/store";
+import { type CreatedEditor } from "~/state/sharedVariantEditor.slice";
+
+const FunctionCallEditor = ({
+  function_call,
+  onEdit,
+}: {
+  function_call: CreateChatCompletionRequestMessage.FunctionCall;
+  onEdit: (function_call: CreateChatCompletionRequestMessage.FunctionCall) => void;
+}) => {
+  const monaco = useAppStore.use.sharedArgumentsEditor.monaco();
+  const editorRef = useRef<CreatedEditor | null>(null);
+  const editorId = useMemo(() => `editor_${Math.random().toString(36).substring(7)}`, []);
+
+  useEffect(() => {
+    if (monaco) {
+      const container = document.getElementById(editorId) as HTMLElement;
+
+      const editor = monaco.editor.create(container, {
+        value: function_call.arguments,
+        language: "json",
+        theme: "customTheme",
+        lineNumbers: "off",
+        minimap: { enabled: false },
+        wrappingIndent: "indent",
+        wrappingStrategy: "advanced",
+        wordWrap: "on",
+        folding: false,
+        scrollbar: {
+          alwaysConsumeMouseWheel: false,
+          verticalScrollbarSize: 0,
+        },
+        wordWrapBreakAfterCharacters: "",
+        wordWrapBreakBeforeCharacters: "",
+        quickSuggestions: true,
+        renderLineHighlight: "none",
+        fontSize: 14,
+        scrollBeyondLastLine: false,
+      });
+
+      editorRef.current = editor;
+
+      const updateHeight = () => {
+        const contentHeight = editor.getContentHeight();
+        container.style.height = `${contentHeight}px`;
+        editor.layout();
+      };
+
+      const attemptDocumentFormat = () => {
+        const action = editor.getAction("editor.action.formatDocument");
+        if (action) {
+          action
+            .run()
+            .then(updateHeight)
+            .catch((error) => {
+              console.error("Error running formatDocument:", error);
+            });
+          return true;
+        }
+        return false;
+      };
+
+      editor.onDidBlurEditorText(() => {
+        attemptDocumentFormat();
+        onEdit({ name: function_call.name, arguments: editor.getValue() });
+      });
+
+      // Interval function to check for action availability
+      const checkForActionInterval = setInterval(() => {
+        const formatted = attemptDocumentFormat();
+        if (formatted) {
+          clearInterval(checkForActionInterval); // Clear the interval once the action is found and run
+        }
+      }, 100); // Check every 100ms
+
+      // Add content change listener
+      const contentChangeListener = editor.onDidChangeModelContent(updateHeight);
+
+      const resizeObserver = new ResizeObserver(() => {
+        editor.layout();
+      });
+      resizeObserver.observe(container);
+
+      return () => {
+        contentChangeListener.dispose();
+        resizeObserver.disconnect();
+        editor?.dispose();
+      };
+    }
+  }, [monaco, editorId, function_call.name, function_call.arguments, onEdit]);
+
+  return (
+    <VStack w="full" alignItems="flex-start">
+      <HStack w="full">
+        <Text fontWeight="bold" w={192}>
+          Name:
+        </Text>
+        <Input
+          value={function_call.name}
+          onChange={(e) => onEdit({ name: e.target.value, arguments: function_call.arguments })}
+          bgColor="white"
+        />
+      </HStack>
+      <Text fontWeight="bold" w={32}>
+        Arguments
+      </Text>
+      <VStack
+        borderRadius={4}
+        border="1px solid"
+        borderColor="gray.200"
+        w="full"
+        py={1}
+        bgColor="white"
+      >
+        <Box id={editorId} w="full" />
+      </VStack>
+    </VStack>
+  );
+};
+
+export default FunctionCallEditor;

app/src/components/datasets/DatasetEntriesTable/TableRow.tsx

@@ -0,0 +1,128 @@
+import { Box, Td, Tr, Thead, Th, Tooltip, HStack, Text, Checkbox } from "@chakra-ui/react";
+import Link from "next/link";
+
+import dayjs from "~/utils/dayjs";
+import { type RouterOutputs } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import { useIsClientRehydrated, useDatasetEntries } from "~/utils/hooks";
+import { useMemo } from "react";
+
+type DatasetEntry = RouterOutputs["datasetEntries"]["list"]["entries"][0];
+
+export const TableHeader = () => {
+  const matchingDatasetEntryIds = useDatasetEntries().data?.matchingEntryIds;
+  const selectedDatasetEntryIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+  const addSelectedIds = useAppStore((s) => s.selectedDatasetEntries.addSelectedIds);
+  const clearSelectedIds = useAppStore((s) => s.selectedDatasetEntries.clearSelectedIds);
+  const allSelected = useMemo(() => {
+    if (!matchingDatasetEntryIds || !matchingDatasetEntryIds.length) return false;
+    return matchingDatasetEntryIds.every((id) => selectedDatasetEntryIds.has(id));
+  }, [matchingDatasetEntryIds, selectedDatasetEntryIds]);
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
+  return (
+    <Thead>
+      <Tr>
+        <Th pr={0}>
+          <HStack minW={16}>
+            <Checkbox
+              isChecked={allSelected}
+              onChange={() => {
+                allSelected ? clearSelectedIds() : addSelectedIds(matchingDatasetEntryIds || []);
+              }}
+            />
+            <Text>
+              ({selectedDatasetEntryIds.size ? `${selectedDatasetEntryIds.size}/` : ""}
+              {matchingDatasetEntryIds?.length || 0})
+            </Text>
+          </HStack>
+        </Th>
+        <Th>Created At</Th>
+        <Th isNumeric>Input tokens</Th>
+        <Th isNumeric>Output tokens</Th>
+        <Th isNumeric>Type</Th>
+      </Tr>
+    </Thead>
+  );
+};
+
+export const TableRow = ({
+  datasetEntry,
+  onToggle,
+  showOptions,
+}: {
+  datasetEntry: DatasetEntry;
+  onToggle: () => void;
+  showOptions?: boolean;
+}) => {
+  const createdAt = dayjs(datasetEntry.createdAt).format("MMMM D h:mm A");
+  const fullTime = dayjs(datasetEntry.createdAt).toString();
+
+  const isChecked = useAppStore((s) => s.selectedDatasetEntries.selectedIds.has(datasetEntry.id));
+  const toggleChecked = useAppStore((s) => s.selectedDatasetEntries.toggleSelectedId);
+
+  const isClientRehydrated = useIsClientRehydrated();
+  if (!isClientRehydrated) return null;
+
+  return (
+    <Tr
+      onClick={onToggle}
+      key={datasetEntry.id}
+      _hover={{ bgColor: "gray.50", cursor: "pointer" }}
+      fontSize="sm"
+    >
+      {showOptions && (
+        <Td>
+          <Checkbox isChecked={isChecked} onChange={() => toggleChecked(datasetEntry.id)} />
+        </Td>
+      )}
+      <Td>
+        <Tooltip label={fullTime} placement="top">
+          <Box whiteSpace="nowrap" minW="120px">
+            {createdAt}
+          </Box>
+        </Tooltip>
+      </Td>
+      <Td isNumeric>{datasetEntry.inputTokens}</Td>
+      <Td isNumeric>{datasetEntry.outputTokens}</Td>
+      <Td isNumeric>{datasetEntry.type}</Td>
+    </Tr>
+  );
+};
+
+export const EmptyTableRow = ({ filtersApplied = true }: { filtersApplied?: boolean }) => {
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const filters = useAppStore((state) => state.logFilters.filters);
+  const { isLoading } = useDatasetEntries();
+
+  if (isLoading) return null;
+
+  if (filters.length && filtersApplied) {
+    return (
+      <Tr>
+        <Td w="full" colSpan={visibleColumns.size + 1}>
+          <Text color="gray.500" textAlign="center" w="full" p={4}>
+            No matching entries found. Try removing some filters.
+          </Text>
+        </Td>
+      </Tr>
+    );
+  }
+
+  return (
+    <Tr>
+      <Td w="full" colSpan={visibleColumns.size + 1}>
+        <Text color="gray.500" textAlign="center" w="full" p={4}>
+          This dataset has no entries. Add some logs in the{" "}
+          <Link href="/request-logs">
+            <Text as="span" color="blue.600">
+              Request Logs
+            </Text>
+          </Link>{" "}
+          tab.
+        </Text>
+      </Td>
+    </Tr>
+  );
+};

app/src/components/datasets/DatasetEntryPaginator.tsx

@@ -0,0 +1,16 @@
+import { type StackProps } from "@chakra-ui/react";
+
+import { useDatasetEntries } from "~/utils/hooks";
+import Paginator from "../Paginator";
+
+const DatasetEntryPaginator = (props: StackProps) => {
+  const { data } = useDatasetEntries();
+
+  if (!data) return null;
+
+  const { matchingEntryIds } = data;
+
+  return <Paginator count={matchingEntryIds.length} {...props} />;
+};
+
+export default DatasetEntryPaginator;

app/src/components/datasets/DatasetHeaderButtons.tsx

@@ -0,0 +1,20 @@
+import { Button, HStack, Icon, Text } from "@chakra-ui/react";
+import { useDataset } from "~/utils/hooks";
+import { BsGearFill } from "react-icons/bs";
+
+export const DatasetHeaderButtons = ({ openDrawer }: { openDrawer: () => void }) => {
+  const dataset = useDataset();
+
+  if (dataset.isLoading) return null;
+
+  return (
+    <HStack spacing={0} mt={{ base: 2, md: 0 }}>
+      <Button variant={{ base: "solid", md: "ghost" }} onClick={openDrawer}>
+        <HStack>
+          <Icon as={BsGearFill} />
+          <Text>Configure</Text>
+        </HStack>
+      </Button>
+    </HStack>
+  );
+};

app/src/components/datasets/DatasetsTable.tsx

@@ -0,0 +1,52 @@
+import { Card, Table, Thead, Tr, Th, Tbody, Td, VStack, Icon, Text } from "@chakra-ui/react";
+import { FaTable } from "react-icons/fa";
+import Link from "next/link";
+
+import dayjs from "~/utils/dayjs";
+import { useDatasets } from "~/utils/hooks";
+
+const DatasetsTable = ({}) => {
+  const { data } = useDatasets();
+
+  const datasets = data || [];
+
+  return (
+    <Card width="100%" overflowX="auto">
+      {datasets.length ? (
+        <Table>
+          <Thead>
+            <Tr>
+              <Th>Name</Th>
+              <Th>Created At</Th>
+              <Th>Size</Th>
+            </Tr>
+          </Thead>
+          <Tbody>
+            {datasets.map((dataset) => {
+              return (
+                <Tr key={dataset.id}>
+                  <Td>
+                    <Link href={{ pathname: "/datasets/[id]", query: { id: dataset.id } }}>
+                      <Text color="blue.600">{dataset.name}</Text>
+                    </Link>
+                  </Td>
+                  <Td>{dayjs(dataset.createdAt).format("MMMM D h:mm A")}</Td>
+                  <Td>{dataset._count.datasetEntries}</Td>
+                </Tr>
+              );
+            })}
+          </Tbody>
+        </Table>
+      ) : (
+        <VStack py={8}>
+          <Icon as={FaTable} boxSize={16} color="gray.300" />
+          <Text color="gray.400" fontSize="lg" fontWeight="bold">
+            No Datasets Found. Create your first dataset.
+          </Text>
+        </VStack>
+      )}
+    </Card>
+  );
+};
+
+export default DatasetsTable;

app/src/components/datasets/DeleteButton.tsx

@@ -0,0 +1,107 @@
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  useDisclosure,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import { BsTrash } from "react-icons/bs";
+
+import { useHandledAsyncCallback, useDataset } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import pluralize from "pluralize";
+
+const DeleteButton = () => {
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Delete"
+        icon={BsTrash}
+        isDisabled={selectedIds.size === 0}
+        requireBeta
+      />
+      <DeleteDatasetEntriesModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default DeleteButton;
+
+const DeleteDatasetEntriesModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const dataset = useDataset().data;
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+  const clearSelectedIds = useAppStore((s) => s.selectedDatasetEntries.clearSelectedIds);
+
+  const deleteRowsMutation = api.datasetEntries.delete.useMutation();
+
+  const utils = api.useContext();
+
+  const [deleteRows, deletionInProgress] = useHandledAsyncCallback(async () => {
+    if (!dataset?.id || !selectedIds.size) return;
+
+    // divide selectedIds into chunks of 15000 to reduce request size
+    const chunkSize = 15000;
+    const idsArray = Array.from(selectedIds);
+    for (let i = 0; i < idsArray.length; i += chunkSize) {
+      const response = await deleteRowsMutation.mutateAsync({
+        ids: idsArray.slice(i, i + chunkSize),
+      });
+
+      if (maybeReportError(response)) return;
+    }
+
+    await utils.datasetEntries.list.invalidate();
+    disclosure.onClose();
+    clearSelectedIds();
+  }, [deleteRowsMutation, dataset, selectedIds, utils]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={BsTrash} />
+            <Text>Delete Logs</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              Are you sure you want to delete the <b>{selectedIds.size}</b>{" "}
+              {pluralize("row", selectedIds.size)} rows you've selected?
+            </Text>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button colorScheme="red" onClick={deleteRows} isLoading={deletionInProgress} minW={24}>
+              Delete
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/datasets/DownloadButton.tsx

@@ -0,0 +1,182 @@
+import { useState, useEffect } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  Checkbox,
+  NumberInput,
+  NumberInputField,
+  NumberInputStepper,
+  NumberIncrementStepper,
+  NumberDecrementStepper,
+  Collapse,
+  Flex,
+  useDisclosure,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import { AiOutlineDownload } from "react-icons/ai";
+
+import { useHandledAsyncCallback, useDataset } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+import { FiChevronUp, FiChevronDown } from "react-icons/fi";
+import InfoCircle from "../InfoCircle";
+
+const ExportButton = () => {
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Download"
+        icon={AiOutlineDownload}
+        isDisabled={selectedIds.size === 0}
+        requireBeta
+      />
+      <ExportDatasetEntriesModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default ExportButton;
+
+const ExportDatasetEntriesModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const dataset = useDataset().data;
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+  const clearSelectedIds = useAppStore((s) => s.selectedDatasetEntries.clearSelectedIds);
+
+  const [testingSplit, setTestingSplit] = useState(10);
+  const [removeDuplicates, setRemoveDuplicates] = useState(false);
+  const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      setTestingSplit(10);
+      setRemoveDuplicates(false);
+    }
+  }, [disclosure.isOpen]);
+
+  const exportDataMutation = api.datasetEntries.export.useMutation();
+
+  const [exportData, exportInProgress] = useHandledAsyncCallback(async () => {
+    if (!dataset?.id || !selectedIds.size || !testingSplit) return;
+    const response = await exportDataMutation.mutateAsync({
+      datasetId: dataset.id,
+      datasetEntryIds: Array.from(selectedIds),
+      testingSplit,
+      removeDuplicates,
+    });
+
+    const dataUrl = `data:application/pdf;base64,${response}`;
+    const blob = await fetch(dataUrl).then((res) => res.blob());
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement("a");
+
+    a.href = url;
+    a.download = `data.zip`;
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+
+    disclosure.onClose();
+    clearSelectedIds();
+  }, [exportDataMutation, dataset, selectedIds, testingSplit, removeDuplicates]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={AiOutlineDownload} />
+            <Text>Export Logs</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              We'll export the <b>{selectedIds.size}</b> rows you have selected in the OpenAI
+              training format.
+            </Text>
+            <VStack alignItems="flex-start" spacing={4}>
+              <Flex
+                flexDir={{ base: "column", md: "row" }}
+                alignItems={{ base: "flex-start", md: "center" }}
+              >
+                <HStack w={48} alignItems="center" spacing={1}>
+                  <Text fontWeight="bold">Testing Split:</Text>
+                  <InfoCircle tooltipText="The percent of your logs that will be reserved for testing and saved in another file. Logs are split randomly." />
+                </HStack>
+                <HStack>
+                  <NumberInput
+                    defaultValue={10}
+                    onChange={(_, num) => setTestingSplit(num)}
+                    min={1}
+                    max={100}
+                    w={48}
+                  >
+                    <NumberInputField />
+                    <NumberInputStepper>
+                      <NumberIncrementStepper />
+                      <NumberDecrementStepper />
+                    </NumberInputStepper>
+                  </NumberInput>
+                </HStack>
+              </Flex>
+            </VStack>
+            <VStack alignItems="flex-start" spacing={0}>
+              <Button
+                variant="unstyled"
+                color="blue.600"
+                onClick={() => setShowAdvancedOptions(!showAdvancedOptions)}
+              >
+                <HStack>
+                  <Text>Advanced Options</Text>
+                  <Icon as={showAdvancedOptions ? FiChevronUp : FiChevronDown} />
+                </HStack>
+              </Button>
+              <Collapse in={showAdvancedOptions} unmountOnExit={true}>
+                <VStack align="stretch" pt={4}>
+                  <HStack>
+                    <Checkbox
+                      colorScheme="blue"
+                      isChecked={removeDuplicates}
+                      onChange={(e) => setRemoveDuplicates(e.target.checked)}
+                    >
+                      <Text>Remove duplicates</Text>
+                    </Checkbox>
+                    <InfoCircle tooltipText="To avoid overfitting and speed up training, automatically deduplicate logs with matching input and output." />
+                  </HStack>
+                </VStack>
+              </Collapse>
+            </VStack>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button colorScheme="blue" onClick={exportData} isLoading={exportInProgress} minW={24}>
+              Download
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/datasets/ExperimentButton.tsx

@@ -0,0 +1,21 @@
+import { RiFlaskLine } from "react-icons/ri";
+
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+
+const ExperimentButton = () => {
+  const selectedIds = useAppStore((s) => s.selectedDatasetEntries.selectedIds);
+  return (
+    <ActionButton
+      onClick={() => {
+        console.log("experimenting with these ids", selectedIds);
+      }}
+      label="Experiment"
+      icon={RiFlaskLine}
+      isDisabled={selectedIds.size === 0}
+      requireBeta
+    />
+  );
+};
+
+export default ExperimentButton;

app/src/components/datasets/FileUploadsCard.tsx

@@ -0,0 +1,139 @@
+import { useState, useEffect } from "react";
+import { VStack, HStack, Button, Text, Progress, IconButton, Portal } from "@chakra-ui/react";
+import { BsX } from "react-icons/bs";
+
+import { type RouterOutputs, api } from "~/utils/api";
+import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
+import { formatFileSize } from "~/utils/utils";
+
+type FileUpload = RouterOutputs["datasets"]["listFileUploads"][0];
+
+const FileUploadsCard = () => {
+  const dataset = useDataset();
+  const [fileUploadsRefetchInterval, setFileUploadsRefetchInterval] = useState<number>(500);
+  const fileUploads = api.datasets.listFileUploads.useQuery(
+    { datasetId: dataset.data?.id as string },
+    { enabled: !!dataset.data?.id, refetchInterval: fileUploadsRefetchInterval },
+  );
+  useEffect(() => {
+    if (fileUploads?.data?.some((fu) => fu.status !== "COMPLETE" && fu.status !== "ERROR")) {
+      setFileUploadsRefetchInterval(500);
+    } else {
+      setFileUploadsRefetchInterval(15000);
+    }
+  }, [fileUploads]);
+
+  const utils = api.useContext();
+
+  const hideFileUploadsMutation = api.datasets.hideFileUploads.useMutation();
+  const [hideAllFileUploads] = useHandledAsyncCallback(async () => {
+    if (!fileUploads.data?.length) return;
+    await hideFileUploadsMutation.mutateAsync({
+      fileUploadIds: fileUploads.data.map((upload) => upload.id),
+    });
+    await utils.datasets.listFileUploads.invalidate();
+  }, [hideFileUploadsMutation, fileUploads.data, utils]);
+
+  if (!fileUploads.data?.length) return null;
+
+  return (
+    <Portal>
+      <VStack
+        w={72}
+        borderRadius={8}
+        position="fixed"
+        bottom={8}
+        right={8}
+        overflow="hidden"
+        borderWidth={1}
+        boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
+        minW={0}
+        bgColor="white"
+      >
+        <HStack p={4} w="full" bgColor="gray.200" justifyContent="space-between">
+          <Text fontWeight="bold">Uploads</Text>
+          <IconButton
+            aria-label="Close uploads"
+            as={BsX}
+            boxSize={6}
+            minW={0}
+            variant="ghost"
+            onClick={hideAllFileUploads}
+            cursor="pointer"
+          />
+        </HStack>
+        {fileUploads?.data?.map((upload) => <FileUploadRow key={upload.id} fileUpload={upload} />)}
+      </VStack>
+    </Portal>
+  );
+};
+
+export default FileUploadsCard;
+
+const FileUploadRow = ({ fileUpload }: { fileUpload: FileUpload }) => {
+  const { id, fileName, fileSize, progress, status, errorMessage } = fileUpload;
+
+  const utils = api.useContext();
+
+  const hideFileUploadsMutation = api.datasets.hideFileUploads.useMutation();
+  const [hideFileUpload, hidingInProgress] = useHandledAsyncCallback(async () => {
+    await hideFileUploadsMutation.mutateAsync({ fileUploadIds: [id] });
+    await utils.datasets.listFileUploads.invalidate();
+  }, [id, hideFileUploadsMutation, utils]);
+
+  useEffect(() => {
+    // Invalidate dataset entries list when upload is processed
+    if (status === "COMPLETE") void utils.datasetEntries.list.invalidate();
+  }, [status, utils]);
+
+  return (
+    <VStack w="full" alignItems="flex-start" p={4} borderBottomWidth={1}>
+      <HStack w="full" justifyContent="space-between" alignItems="flex-start">
+        <VStack alignItems="flex-start" spacing={0}>
+          <Text fontWeight="bold">{fileName}</Text>
+          <Text fontSize="xs">({formatFileSize(fileSize, 2)})</Text>
+        </VStack>
+        <Button
+          aria-label="Hide file upload"
+          minW={0}
+          variant="ghost"
+          isLoading={hidingInProgress}
+          onClick={hideFileUpload}
+          size="xs"
+        >
+          HIDE
+        </Button>
+      </HStack>
+
+      {errorMessage ? (
+        <Text alignSelf="center" pt={2}>
+          {errorMessage}
+        </Text>
+      ) : (
+        <>
+          <Text alignSelf="center" fontSize="xs">
+            {getStatusText(status)}
+          </Text>
+          <Progress w="full" value={progress} borderRadius={2} />
+        </>
+      )}
+    </VStack>
+  );
+};
+
+const getStatusText = (status: FileUpload["status"]) => {
+  switch (status) {
+    case "PENDING":
+      return "Pending";
+    case "DOWNLOADING":
+      return "Loading Data";
+    case "PROCESSING":
+      return "Processing";
+    case "SAVING":
+      return "Saving";
+    case "COMPLETE":
+      return "Complete";
+    case "ERROR":
+      return "Error";
+  }
+};

app/src/components/datasets/FineTuneButton.tsx

@@ -20,17 +20,18 @@ import { AiTwotoneThunderbolt } from "react-icons/ai";
 import humanId from "human-id";
 import { useRouter } from "next/router";
 
-import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useDataset, useDatasetEntries, useHandledAsyncCallback } from "~/utils/hooks";
 import { api } from "~/utils/api";
-import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
-import ActionButton from "./ActionButton";
 import InputDropdown from "../InputDropdown";
-import { FiChevronDown } from "react-icons/fi";
+// import { FiChevronDown } from "react-icons/fi";
 
 const SUPPORTED_BASE_MODELS = ["llama2-7b", "llama2-13b", "llama2-70b", "gpt-3.5-turbo"];
 
 const FineTuneButton = () => {
-  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const datasetEntries = useDatasetEntries().data;
+
+  const numEntries = datasetEntries?.matchingEntryIds.length || 0;
 
   const disclosure = useDisclosure();
 
@@ -40,7 +41,8 @@ const FineTuneButton = () => {
         onClick={disclosure.onOpen}
         label="Fine Tune"
         icon={AiTwotoneThunderbolt}
-        isDisabled={selectedLogIds.size === 0}
+        isDisabled={numEntries === 0}
+        requireBeta
       />
       <FineTuneModal disclosure={disclosure} />
     </>
@@ -50,9 +52,8 @@ const FineTuneButton = () => {
 export default FineTuneButton;
 
 const FineTuneModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
-  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
+  const dataset = useDataset().data;
-  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const datasetEntries = useDatasetEntries().data;
-  const clearSelectedLogIds = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
 
   const [selectedBaseModel, setSelectedBaseModel] = useState(SUPPORTED_BASE_MODELS[0]);
   const [modelSlug, setModelSlug] = useState(humanId({ separator: "-", capitalize: false }));
@@ -70,19 +71,17 @@ const FineTuneModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
   const createFineTuneMutation = api.fineTunes.create.useMutation();
 
   const [createFineTune, creationInProgress] = useHandledAsyncCallback(async () => {
-    if (!selectedProjectId || !modelSlug || !selectedBaseModel || !selectedLogIds.size) return;
+    if (!modelSlug || !selectedBaseModel || !dataset) return;
     await createFineTuneMutation.mutateAsync({
-      projectId: selectedProjectId,
       slug: modelSlug,
       baseModel: selectedBaseModel,
-      selectedLogIds: Array.from(selectedLogIds),
+      datasetId: dataset.id,
     });
 
     await utils.fineTunes.list.invalidate();
     await router.push({ pathname: "/fine-tunes" });
-    clearSelectedLogIds();
     disclosure.onClose();
-  }, [createFineTuneMutation, selectedProjectId, selectedLogIds, modelSlug, selectedBaseModel]);
+  }, [createFineTuneMutation, modelSlug, selectedBaseModel]);
 
   return (
     <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
@@ -98,7 +97,8 @@ const FineTuneModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
         <ModalBody maxW="unset">
           <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
             <Text>
-              We'll train on the <b>{selectedLogIds.size}</b> logs you've selected.
+              We'll train on <b>{datasetEntries?.trainingCount}</b> and test on{" "}
+              <b>{datasetEntries?.testingCount}</b> entries in this dataset.
             </Text>
             <VStack>
               <HStack spacing={2} w="full">
@@ -131,12 +131,12 @@ const FineTuneModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
                 />
               </HStack>
             </VStack>
-            <Button variant="unstyled" color="blue.600">
+            {/* <Button variant="unstyled" color="blue.600">
               <HStack>
                 <Text>Advanced Options</Text>
                 <Icon as={FiChevronDown} />
               </HStack>
-            </Button>
+            </Button> */}
           </VStack>
         </ModalBody>
         <ModalFooter>

app/src/components/datasets/UploadDataButton.tsx

@@ -0,0 +1,288 @@
+import { useState, useEffect, useRef, useCallback } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  Box,
+  useDisclosure,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import pluralize from "pluralize";
+import { AiOutlineCloudUpload, AiOutlineFile } from "react-icons/ai";
+
+import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import ActionButton from "../ActionButton";
+import { validateTrainingRows, type TrainingRow, parseJSONL } from "./validateTrainingRows";
+import { uploadDatasetEntryFile } from "~/utils/azure/website";
+import { formatFileSize } from "~/utils/utils";
+
+const UploadDataButton = () => {
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Upload Data"
+        icon={AiOutlineCloudUpload}
+        iconBoxSize={4}
+        requireBeta
+      />
+      <UploadDataModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default UploadDataButton;
+
+const UploadDataModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const dataset = useDataset().data;
+
+  const [validationError, setValidationError] = useState<string | null>(null);
+  const [trainingRows, setTrainingRows] = useState<TrainingRow[] | null>(null);
+  const [file, setFile] = useState<File | null>(null);
+
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileDrop = (e: React.DragEvent<HTMLDivElement>) => {
+    e.preventDefault();
+    const files = e.dataTransfer.files;
+    if (files.length > 0) {
+      processFile(files[0] as File);
+    }
+  };
+
+  const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const files = e.target.files;
+    if (files && files.length > 0) {
+      processFile(files[0] as File);
+    }
+  };
+
+  const processFile = (file: File) => {
+    setFile(file);
+
+    // skip reading if file is larger than 10MB
+    if (file.size > 10000000) {
+      setTrainingRows(null);
+      return;
+    }
+
+    const reader = new FileReader();
+    reader.onload = (e: ProgressEvent<FileReader>) => {
+      const content = e.target?.result as string;
+      // Process the content, e.g., set to state
+      let parsedJSONL;
+      try {
+        parsedJSONL = parseJSONL(content) as TrainingRow[];
+        const validationError = validateTrainingRows(parsedJSONL);
+        if (validationError) {
+          setValidationError(validationError);
+          setTrainingRows(null);
+          return;
+        }
+        setTrainingRows(parsedJSONL);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      } catch (e: any) {
+        setValidationError("Unable to parse JSONL file: " + (e.message as string));
+        setTrainingRows(null);
+        return;
+      }
+    };
+    reader.readAsText(file);
+  };
+
+  const resetState = useCallback(() => {
+    setValidationError(null);
+    setTrainingRows(null);
+    setFile(null);
+  }, [setValidationError, setTrainingRows, setFile]);
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      resetState();
+    }
+  }, [disclosure.isOpen, resetState]);
+
+  const triggerFileDownloadMutation = api.datasets.triggerFileDownload.useMutation();
+
+  const utils = api.useContext();
+
+  const [sendJSONL, sendingInProgress] = useHandledAsyncCallback(async () => {
+    if (!dataset || !file) return;
+
+    const blobName = await uploadDatasetEntryFile(file);
+
+    await triggerFileDownloadMutation.mutateAsync({
+      datasetId: dataset.id,
+      blobName,
+      fileName: file.name,
+      fileSize: file.size,
+    });
+
+    await utils.datasets.listFileUploads.invalidate();
+
+    disclosure.onClose();
+  }, [dataset, trainingRows, triggerFileDownloadMutation, file, utils]);
+
+  return (
+    <Modal
+      size={{ base: "xl", md: "2xl" }}
+      closeOnOverlayClick={false}
+      closeOnEsc={false}
+      {...disclosure}
+    >
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Text>Upload Training Logs</Text>
+          </HStack>
+        </ModalHeader>
+        {!sendingInProgress && <ModalCloseButton />}
+        <ModalBody maxW="unset" p={8}>
+          <Box w="full" aspectRatio={1.5}>
+            {validationError && (
+              <VStack w="full" h="full" justifyContent="center" spacing={8}>
+                <Icon as={AiOutlineFile} boxSize={24} color="gray.300" />
+                <VStack w="full">
+                  <Text fontSize={32} color="gray.500" fontWeight="bold">
+                    Error
+                  </Text>
+                  <Text color="gray.500">{validationError}</Text>
+                </VStack>
+                <Text
+                  as="span"
+                  textDecor="underline"
+                  color="gray.500"
+                  _hover={{ color: "orange.400" }}
+                  cursor="pointer"
+                  onClick={resetState}
+                >
+                  Try again
+                </Text>
+              </VStack>
+            )}
+            {!validationError && !file && (
+              <VStack
+                w="full"
+                h="full"
+                stroke="gray.300"
+                justifyContent="center"
+                borderRadius={8}
+                sx={{
+                  "background-image": `url("data:image/svg+xml,%3csvg width='100%25' height='100%25' xmlns='http://www.w3.org/2000/svg'%3e%3crect x='2%25' y='2%25' width='96%25' height='96%25' fill='none' stroke='%23eee' stroke-width='4' stroke-dasharray='6%2c 14' stroke-dashoffset='0' stroke-linecap='square' rx='8' ry='8'/%3e%3c/svg%3e")`,
+                }}
+                onDragOver={(e) => e.preventDefault()}
+                onDrop={handleFileDrop}
+              >
+                <JsonFileIcon />
+                <Icon as={AiOutlineCloudUpload} boxSize={24} color="gray.300" />
+
+                <Text fontSize={32} color="gray.500" fontWeight="bold">
+                  Drag & Drop
+                </Text>
+                <Text color="gray.500">
+                  your .jsonl file here, or{" "}
+                  <input
+                    type="file"
+                    ref={fileInputRef}
+                    onChange={handleFileChange}
+                    style={{ display: "none" }}
+                    accept=".jsonl"
+                  />
+                  <Text
+                    as="span"
+                    textDecor="underline"
+                    _hover={{ color: "orange.400" }}
+                    cursor="pointer"
+                    onClick={() => fileInputRef.current?.click()}
+                  >
+                    browse
+                  </Text>
+                </Text>
+              </VStack>
+            )}
+            {!validationError && file && (
+              <VStack w="full" h="full" justifyContent="center" spacing={8}>
+                <JsonFileIcon />
+                <VStack w="full">
+                  {trainingRows ? (
+                    <>
+                      <Text fontSize={32} color="gray.500" fontWeight="bold">
+                        Success
+                      </Text>
+                      <Text color="gray.500">
+                        We'll upload <b>{trainingRows.length}</b>{" "}
+                        {pluralize("row", trainingRows.length)} into <b>{dataset?.name}</b>.{" "}
+                      </Text>
+                    </>
+                  ) : (
+                    <>
+                      <Text fontSize={32} color="gray.500" fontWeight="bold">
+                        {file.name}
+                      </Text>
+                      <Text color="gray.500">{formatFileSize(file.size)}</Text>
+                    </>
+                  )}
+                </VStack>
+                {!sendingInProgress && (
+                  <Text
+                    as="span"
+                    textDecor="underline"
+                    color="gray.500"
+                    _hover={{ color: "orange.400" }}
+                    cursor="pointer"
+                    onClick={resetState}
+                  >
+                    Change file
+                  </Text>
+                )}
+              </VStack>
+            )}
+          </Box>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button
+              colorScheme="gray"
+              isDisabled={sendingInProgress}
+              onClick={disclosure.onClose}
+              minW={24}
+            >
+              Cancel
+            </Button>
+            <Button
+              colorScheme="orange"
+              onClick={sendJSONL}
+              isLoading={sendingInProgress}
+              minW={24}
+              isDisabled={!file || !!validationError}
+            >
+              Upload
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};
+
+const JsonFileIcon = () => (
+  <Box position="relative" display="flex" alignItems="center" justifyContent="center">
+    <Icon as={AiOutlineFile} boxSize={24} color="gray.300" />
+    <Text position="absolute" color="orange.400" fontWeight="bold" fontSize={12} pt={4}>
+      JSONL
+    </Text>
+  </Box>
+);

app/src/components/datasets/validateTrainingRows.ts

@@ -0,0 +1,71 @@
+import { type CreateChatCompletionRequestMessage } from "openai/resources/chat";
+
+export type TrainingRow = {
+  input: CreateChatCompletionRequestMessage[];
+  output?: CreateChatCompletionRequestMessage;
+};
+
+export const parseJSONL = (jsonlString: string): unknown[] => {
+  const lines = jsonlString.trim().split("\n");
+
+  let lineNumber = 0;
+  const parsedLines = [];
+
+  try {
+    for (const line of lines) {
+      lineNumber++;
+      parsedLines.push(JSON.parse(line));
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  } catch (e: any) {
+    throw new Error(`Error parsing line ${lineNumber}: ${e.message as string}`);
+  }
+  return parsedLines;
+};
+
+export const validateTrainingRows = (rows: unknown): string | null => {
+  if (!Array.isArray(rows)) return "training data is not an array";
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i] as TrainingRow;
+    let errorMessage: string | null = null;
+    try {
+      errorMessage = validateTrainingRow(row);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (error: any) {
+      errorMessage = error.message;
+    }
+    if (errorMessage) return `row ${i + 1}: ${errorMessage}`;
+  }
+
+  return null;
+};
+
+const validateTrainingRow = (row: TrainingRow): string | null => {
+  if (!row) return "empty row";
+  if (!row.input) return "missing input";
+
+  //   Validate input
+  if (!Array.isArray(row.input)) return "input is not an array";
+  if ((row.input as unknown[]).some((x) => typeof x !== "object"))
+    return "input contains invalid item";
+  if (row.input.some((x) => !x)) return "input contains empty item";
+  if (row.input.some((x) => !x.content && !x.function_call))
+    return "input contains item with no content or function_call";
+  if (row.input.some((x) => x.function_call && !x.function_call.arguments))
+    return "input contains item with function_call but no arguments";
+  if (row.input.some((x) => x.function_call && !x.function_call.name))
+    return "input contains item with function_call but no name";
+
+  //   Validate output
+  if (row.output) {
+    if (typeof row.output !== "object") return "output is not an object";
+    if (!row.output.content && !row.output.function_call)
+      return "output contains no content or function_call";
+    if (row.output.function_call && !row.output.function_call.arguments)
+      return "output contains function_call but no arguments";
+    if (row.output.function_call && !row.output.function_call.name)
+      return "output contains function_call but no name";
+  }
+
+  return null;
+};

app/src/components/experiments/DeleteExperimentDialog.tsx

@@ -1,36 +1,43 @@
+import { useRef } from "react";
 import {
-  Button,
+  type UseDisclosureReturn,
   AlertDialog,
+  AlertDialogOverlay,
+  AlertDialogContent,
+  AlertDialogHeader,
   AlertDialogBody,
   AlertDialogFooter,
-  AlertDialogHeader,
+  Button,
-  AlertDialogContent,
-  AlertDialogOverlay,
 } from "@chakra-ui/react";
-
-import { useRouter } from "next/router";
-import { useRef } from "react";
 import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 
-export const DeleteDialog = ({ onClose }: { onClose: () => void }) => {
+import { useHandledAsyncCallback } from "~/utils/hooks";
-  const experiment = useExperiment();
-  const deleteMutation = api.experiments.delete.useMutation();
-  const utils = api.useContext();
-  const router = useRouter();
 
+const DeleteExperimentDialog = ({
+  experimentId,
+  onDelete,
+  disclosure,
+}: {
+  experimentId?: string;
+  onDelete?: () => void;
+  disclosure: UseDisclosureReturn;
+}) => {
   const cancelRef = useRef<HTMLButtonElement>(null);
 
-  const [onDeleteConfirm] = useHandledAsyncCallback(async () => {
+  const mutation = api.experiments.delete.useMutation();
-    if (!experiment.data?.id) return;
+  const utils = api.useContext();
-    await deleteMutation.mutateAsync({ id: experiment.data.id });
+
+  const [onDeleteConfirm, deletionInProgress] = useHandledAsyncCallback(async () => {
+    if (!experimentId) return;
+    await mutation.mutateAsync({ id: experimentId });
     await utils.experiments.list.invalidate();
-    await router.push({ pathname: "/experiments" });
+    onDelete?.();
-    onClose();
+
-  }, [deleteMutation, experiment.data?.id, router]);
+    disclosure.onClose();
+  }, [mutation, experimentId, disclosure.onClose]);
 
   return (
-    <AlertDialog isOpen leastDestructiveRef={cancelRef} onClose={onClose}>
+    <AlertDialog leastDestructiveRef={cancelRef} {...disclosure}>
       <AlertDialogOverlay>
         <AlertDialogContent>
           <AlertDialogHeader fontSize="lg" fontWeight="bold">
@@ -43,10 +50,15 @@ export const DeleteDialog = ({ onClose }: { onClose: () => void }) => {
           </AlertDialogBody>
 
           <AlertDialogFooter>
-            <Button ref={cancelRef} onClick={onClose}>
+            <Button ref={cancelRef} onClick={disclosure.onClose}>
               Cancel
             </Button>
-            <Button colorScheme="red" onClick={onDeleteConfirm} ml={3}>
+            <Button
+              colorScheme="red"
+              isLoading={deletionInProgress}
+              onClick={onDeleteConfirm}
+              ml={3}
+            >
               Delete
             </Button>
           </AlertDialogFooter>
@@ -55,3 +67,5 @@ export const DeleteDialog = ({ onClose }: { onClose: () => void }) => {
     </AlertDialog>
   );
 };
+
+export default DeleteExperimentDialog;

app/src/components/experiments/ExperimentCard.tsx

@@ -1,3 +1,4 @@
+import { type MouseEvent, useState } from "react";
 import {
   HStack,
   Icon,
@@ -8,17 +9,29 @@ import {
   AspectRatio,
   SkeletonText,
   Card,
+  useDisclosure,
+  Box,
+  Menu,
+  MenuButton,
+  MenuList,
+  MenuItem,
+  IconButton,
+  useToast,
 } from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
-import { formatTimePast } from "~/utils/dayjs";
 import Link from "next/link";
 import { useRouter } from "next/router";
-import { BsPlusSquare } from "react-icons/bs";
+import { BsPlusSquare, BsThreeDotsVertical, BsLink45Deg, BsTrash } from "react-icons/bs";
-import { RouterOutputs, api } from "~/utils/api";
+
+import { formatTimePast } from "~/utils/dayjs";
+import { type RouterOutputs, api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
 import { useAppStore } from "~/state/store";
+import DeleteExperimentDialog from "./DeleteExperimentDialog";
 
 export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["list"][0] }) => {
+  const [isMenuHovered, setIsMenuHovered] = useState(false);
+
   return (
     <Card
       w="full"
@@ -27,7 +40,7 @@ export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["lis
       p={4}
       bg="white"
       borderRadius={4}
-      _hover={{ bg: "gray.100" }}
+      _hover={{ bg: isMenuHovered ? undefined : "gray.100" }}
       transition="background 0.2s"
       aspectRatio={1.2}
     >
@@ -38,9 +51,17 @@ export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["lis
         href={{ pathname: "/experiments/[experimentSlug]", query: { experimentSlug: exp.slug } }}
         justify="space-between"
       >
-        <HStack w="full" color="gray.700" justify="center">
+        <HStack w="full" justify="space-between" spacing={0}>
-          <Icon as={RiFlaskLine} boxSize={4} />
+          <Box w={6} />
-          <Text fontWeight="bold">{exp.label}</Text>
+          <HStack color="gray.700" justify="center">
+            <Icon as={RiFlaskLine} boxSize={4} />
+            <Text fontWeight="bold">{exp.label}</Text>
+          </HStack>
+          <CardMenu
+            experimentId={exp.id}
+            experimentSlug={exp.slug}
+            setIsMenuHovered={setIsMenuHovered}
+          />
         </HStack>
         <HStack h="full" spacing={4} flex={1} align="center">
           <CountLabel label="Variants" count={exp.promptVariantCount} />
@@ -57,6 +78,75 @@ export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["lis
   );
 };
 
+const CardMenu = ({
+  experimentId,
+  experimentSlug,
+  setIsMenuHovered,
+}: {
+  experimentId: string;
+  experimentSlug: string;
+  setIsMenuHovered: (isHovered: boolean) => void;
+}) => {
+  const deleteDisclosure = useDisclosure();
+  const menuDisclosure = useDisclosure();
+  const toast = useToast();
+  const [copyShareLink] = useHandledAsyncCallback(
+    async (e: MouseEvent<HTMLButtonElement>) => {
+      if (typeof window === "undefined") return;
+      e.preventDefault();
+      e.stopPropagation();
+      const shareLink = `${window.location.origin}/experiments/${experimentSlug}`;
+      await navigator.clipboard.writeText(shareLink);
+      toast({
+        title: "Share link copied to clipboard",
+        status: "success",
+        duration: 2000,
+        isClosable: true,
+      });
+      menuDisclosure.onClose();
+    },
+    [toast, menuDisclosure.onClose, experimentSlug],
+  );
+  return (
+    <>
+      <Menu isLazy {...menuDisclosure}>
+        <MenuButton
+          as={IconButton}
+          aria-label="Options"
+          icon={<BsThreeDotsVertical />}
+          variant="ghost"
+          onClick={(e) => {
+            e.preventDefault();
+            e.stopPropagation();
+            menuDisclosure.onOpen();
+          }}
+          onMouseEnter={() => setIsMenuHovered(true)}
+          onMouseLeave={() => setIsMenuHovered(false)}
+          boxSize={6}
+          minW={0}
+        />
+        <MenuList>
+          <MenuItem icon={<Icon as={BsLink45Deg} boxSize={5} />} onClick={copyShareLink}>
+            Copy Link
+          </MenuItem>
+          <MenuItem
+            icon={<Icon as={BsTrash} boxSize={5} />}
+            onClick={(e) => {
+              e.preventDefault();
+              e.stopPropagation();
+              deleteDisclosure.onOpen();
+            }}
+            color="red.500"
+          >
+            Delete
+          </MenuItem>
+        </MenuList>
+      </Menu>
+      <DeleteExperimentDialog experimentId={experimentId} disclosure={deleteDisclosure} />
+    </>
+  );
+};
+
 const CountLabel = ({ label, count }: { label: string; count: number }) => {
   return (
     <VStack alignItems="center" flex={1}>

app/src/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons.tsx

@@ -3,17 +3,14 @@ import { useOnForkButtonPressed } from "./useOnForkButtonPressed";
 import { useExperiment } from "~/utils/hooks";
 import { BsGearFill } from "react-icons/bs";
 import { TbGitFork } from "react-icons/tb";
-import { useAppStore } from "~/state/store";
 
-export const ExperimentHeaderButtons = () => {
+export const ExperimentHeaderButtons = ({ openDrawer }: { openDrawer: () => void }) => {
   const experiment = useExperiment();
 
   const canModify = experiment.data?.access.canModify ?? false;
 
   const { onForkButtonPressed, isForking } = useOnForkButtonPressed();
 
-  const openDrawer = useAppStore((s) => s.openDrawer);
-
   if (experiment.isLoading) return null;
 
   return (

app/src/components/experiments/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -0,0 +1,39 @@
+import { Button, Icon, useDisclosure, Text } from "@chakra-ui/react";
+import { useRouter } from "next/router";
+import { BsTrash } from "react-icons/bs";
+
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import DeleteExperimentDialog from "../DeleteExperimentDialog";
+
+export const DeleteButton = ({ closeDrawer }: { closeDrawer: () => void }) => {
+  const experiment = useExperiment();
+  const router = useRouter();
+
+  const disclosure = useDisclosure();
+
+  const [onDelete] = useHandledAsyncCallback(async () => {
+    await router.push({ pathname: "/experiments" });
+    closeDrawer();
+  }, [router, closeDrawer]);
+
+  return (
+    <>
+      <Button
+        size="sm"
+        variant="ghost"
+        colorScheme="red"
+        fontWeight="normal"
+        onClick={disclosure.onOpen}
+      >
+        <Icon as={BsTrash} boxSize={4} />
+        <Text ml={2}>Delete Experiment</Text>
+      </Button>
+
+      <DeleteExperimentDialog
+        experimentId={experiment.data?.id}
+        onDelete={onDelete}
+        disclosure={disclosure}
+      />
+    </>
+  );
+};

app/src/components/experiments/ExperimentSettingsDrawer/EditEvaluations.tsx

@@ -19,7 +19,7 @@ import { useCallback, useState } from "react";
 import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-import AutoResizeTextArea from "../AutoResizeTextArea";
+import AutoResizeTextArea from "~/components/AutoResizeTextArea";
 
 type EvalValues = Pick<Evaluation, "label" | "value" | "evalType">;
 

app/src/components/experiments/ExperimentSettingsDrawer/EditScenarioVars.tsx

@@ -5,7 +5,7 @@ import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
 import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
-import { FloatingLabelInput } from "./FloatingLabelInput";
+import { FloatingLabelInput } from "~/components/OutputsTable/FloatingLabelInput";
 
 export const ScenarioVar = ({
   variable,

app/src/components/experiments/ExperimentSettingsDrawer/ExperimentSettingsDrawer.tsx

@@ -7,18 +7,19 @@ import {
   DrawerOverlay,
   Heading,
   VStack,
+  type UseDisclosureReturn,
 } from "@chakra-ui/react";
-import EditScenarioVars from "../OutputsTable/EditScenarioVars";
+import EditScenarioVars from "./EditScenarioVars";
-import EditEvaluations from "../OutputsTable/EditEvaluations";
+import EditEvaluations from "./EditEvaluations";
-import { useAppStore } from "~/state/store";
 import { DeleteButton } from "./DeleteButton";
 
-export default function ExperimentSettingsDrawer() {
+export default function ExperimentSettingsDrawer({
-  const isOpen = useAppStore((state) => state.drawerOpen);
+  disclosure,
-  const closeDrawer = useAppStore((state) => state.closeDrawer);
+}: {
-
+  disclosure: UseDisclosureReturn;
+}) {
   return (
-    <Drawer isOpen={isOpen} placement="right" onClose={closeDrawer} size="md">
+    <Drawer placement="right" size="md" {...disclosure}>
       <DrawerOverlay />
       <DrawerContent>
         <DrawerCloseButton />
@@ -31,7 +32,7 @@ export default function ExperimentSettingsDrawer() {
               <EditScenarioVars />
               <EditEvaluations />
             </VStack>
-            <DeleteButton />
+            <DeleteButton closeDrawer={disclosure.onClose} />
           </VStack>
         </DrawerBody>
       </DrawerContent>

app/src/components/nav/AppShell.tsx

@@ -13,15 +13,18 @@ import {
 } from "@chakra-ui/react";
 import Head from "next/head";
 import Link from "next/link";
+import { useRouter } from "next/router";
 import { BsGearFill, BsGithub, BsPersonCircle } from "react-icons/bs";
 import { IoStatsChartOutline } from "react-icons/io5";
 import { RiHome3Line, RiFlaskLine } from "react-icons/ri";
-import { AiOutlineThunderbolt } from "react-icons/ai";
+import { AiOutlineThunderbolt, AiOutlineDatabase } from "react-icons/ai";
+import { FaReadme } from "react-icons/fa";
 import { signIn, useSession } from "next-auth/react";
+
 import ProjectMenu from "./ProjectMenu";
 import NavSidebarOption from "./NavSidebarOption";
 import IconLink from "./IconLink";
-import { BetaModal } from "./BetaModal";
+import { BetaModal } from "../BetaModal";
 import { useAppStore } from "~/state/store";
 
 const Divider = () => <Box h="1px" bgColor="gray.300" w="full" />;
@@ -73,8 +76,9 @@ const NavSidebar = () => {
             <ProjectMenu />
             <Divider />
 
-            <IconLink icon={RiHome3Line} label="Dashboard" href="/dashboard" beta />
+            <IconLink icon={RiHome3Line} label="Dashboard" href="/dashboard" />
-            <IconLink icon={IoStatsChartOutline} label="Request Logs" href="/request-logs" beta />
+            <IconLink icon={IoStatsChartOutline} label="Request Logs" href="/request-logs" />
+            <IconLink icon={AiOutlineDatabase} label="Datasets" href="/datasets" beta />
             <IconLink icon={AiOutlineThunderbolt} label="Fine Tunes" href="/fine-tunes" beta />
             <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
             <VStack w="full" alignItems="flex-start" spacing={0} pt={8}>
@@ -111,7 +115,22 @@ const NavSidebar = () => {
           </NavSidebarOption>
         )}
       </VStack>
-
+      <HStack
+        w="full"
+        px={{ base: 3, md: 4 }}
+        py={{ base: 0, md: 1 }}
+        as={ChakraLink}
+        justifyContent="start"
+        href="https://docs.openpipe.ai"
+        target="_blank"
+        color="gray.500"
+        spacing={1}
+      >
+        <Icon as={FaReadme} boxSize={4} mr={2} />
+        <Text fontWeight="bold" fontSize="sm" display={{ base: "none", md: "flex" }}>
+          Open Documentation
+        </Text>
+      </HStack>
       <Divider />
       <VStack spacing={0} align="center">
         <ChakraLink
@@ -140,6 +159,7 @@ export default function AppShell({
   requireBeta?: boolean;
 }) {
   const [vh, setVh] = useState("100vh"); // Default height to prevent flicker on initial render
+  const router = useRouter();
 
   useEffect(() => {
     const setHeight = () => {
@@ -181,7 +201,7 @@ export default function AppShell({
           {children}
         </Box>
       </Flex>
-      {requireBeta && flagsLoaded && !flags.betaAccess && <BetaModal />}
+      <BetaModal isOpen={!!requireBeta && flagsLoaded && !flags.betaAccess} onClose={router.back} />
     </>
   );
 }

app/src/components/nav/ProjectMenu.tsx

@@ -57,6 +57,7 @@ export default function ProjectMenu() {
     await utils.projects.list.invalidate();
     setSelectedProjectId(newProj.id);
     await router.push({ pathname: "/project/settings" });
+    popover.onClose();
   }, [createMutation, router]);
 
   const user = useSession().data;

app/src/components/requestLogs/ActionButton.tsx

@@ -1,30 +0,0 @@
-import { Button, HStack, type ButtonProps, Icon, Text } from "@chakra-ui/react";
-import { type IconType } from "react-icons";
-
-const ActionButton = ({
-  icon,
-  label,
-  ...buttonProps
-}: { icon: IconType; label: string } & ButtonProps) => {
-  return (
-    <Button
-      colorScheme="blue"
-      color="black"
-      bgColor="white"
-      borderColor="gray.300"
-      borderRadius={4}
-      variant="outline"
-      size="sm"
-      fontSize="sm"
-      fontWeight="normal"
-      {...buttonProps}
-    >
-      <HStack spacing={1}>
-        {icon && <Icon as={icon} />}
-        <Text display={{ base: "none", md: "flex" }}>{label}</Text>
-      </HStack>
-    </Button>
-  );
-};
-
-export default ActionButton;

app/src/components/requestLogs/AddToDatasetButton.tsx

@@ -0,0 +1,194 @@
+import { useState, useEffect, useMemo } from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  ModalFooter,
+  HStack,
+  VStack,
+  Icon,
+  Text,
+  Button,
+  Flex,
+  Input,
+  useDisclosure,
+  type UseDisclosureReturn,
+  Checkbox,
+} from "@chakra-ui/react";
+import { FiPlusSquare } from "react-icons/fi";
+
+import { useDatasets, useHandledAsyncCallback } from "~/utils/hooks";
+import { api } from "~/utils/api";
+import { useAppStore } from "~/state/store";
+import ActionButton from "../ActionButton";
+import InputDropdown from "../InputDropdown";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { useRouter } from "next/router";
+
+const AddToDatasetButton = () => {
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+
+  const disclosure = useDisclosure();
+
+  return (
+    <>
+      <ActionButton
+        onClick={disclosure.onOpen}
+        label="Add to Dataset"
+        icon={FiPlusSquare}
+        isDisabled={selectedLogIds.size === 0}
+        requireBeta
+      />
+      <AddToDatasetModal disclosure={disclosure} />
+    </>
+  );
+};
+
+export default AddToDatasetButton;
+
+const AddToDatasetModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) => {
+  const selectedProjectId = useAppStore((s) => s.selectedProjectId);
+  const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+  const clearSelectedLogIds = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
+  const router = useRouter();
+
+  const datasets = useDatasets().data;
+
+  const existingDatasetOptions = useMemo(
+    () =>
+      datasets?.length
+        ? datasets.map((d) => ({ label: d.name, id: d.id }))
+        : [{ label: "", id: "" }],
+    [datasets],
+  );
+
+  const [selectedDatasetOption, setSelectedDatasetOption] = useState(existingDatasetOptions?.[0]);
+  const [newDatasetName, setNewDatasetName] = useState("");
+  const [createNewDataset, setCreateNewDataset] = useState(false);
+
+  useEffect(() => {
+    if (disclosure.isOpen) {
+      setSelectedDatasetOption(existingDatasetOptions?.[0]);
+      setCreateNewDataset(!existingDatasetOptions[0]?.id);
+    }
+  }, [disclosure.isOpen, existingDatasetOptions]);
+
+  const createDatasetEntriesMutation = api.datasetEntries.create.useMutation();
+
+  const [addToDataset, addingInProgress] = useHandledAsyncCallback(async () => {
+    if (
+      !selectedProjectId ||
+      !selectedLogIds.size ||
+      !(createNewDataset ? newDatasetName : selectedDatasetOption?.id)
+    )
+      return;
+    const datasetParams = createNewDataset
+      ? { newDatasetParams: { projectId: selectedProjectId, name: newDatasetName } }
+      : { datasetId: selectedDatasetOption?.id };
+    const response = await createDatasetEntriesMutation.mutateAsync({
+      loggedCallIds: Array.from(selectedLogIds),
+      ...datasetParams,
+    });
+
+    if (maybeReportError(response)) return;
+
+    const datasetId = response.payload;
+
+    await router.push({ pathname: "/datasets/[id]", query: { id: datasetId } });
+
+    disclosure.onClose();
+    clearSelectedLogIds();
+  }, [
+    selectedProjectId,
+    selectedLogIds,
+    createNewDataset,
+    selectedDatasetOption?.id,
+    newDatasetName,
+    router,
+  ]);
+
+  return (
+    <Modal size={{ base: "xl", md: "2xl" }} {...disclosure}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Icon as={FiPlusSquare} />
+            <Text>Add to Dataset</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack w="full" spacing={8} pt={4} alignItems="flex-start">
+            <Text>
+              We'll add the <b>{selectedLogIds.size}</b> logs you have selected to the dataset you
+              choose.
+            </Text>
+            <VStack alignItems="flex-start" spacing={4}>
+              {existingDatasetOptions?.length && selectedDatasetOption && (
+                <Flex
+                  flexDir={{ base: "column", md: "row" }}
+                  alignItems={{ base: "flex-start", md: "center" }}
+                >
+                  <Text fontWeight="bold" w={48}>
+                    Dataset:
+                  </Text>
+                  <InputDropdown
+                    options={existingDatasetOptions}
+                    selectedOption={selectedDatasetOption}
+                    getDisplayLabel={(option) => option.label}
+                    onSelect={(option) => setSelectedDatasetOption(option)}
+                    inputGroupProps={{ w: 48 }}
+                    isDisabled={createNewDataset}
+                  />
+                  <Checkbox
+                    isChecked={createNewDataset}
+                    onChange={(e) => setCreateNewDataset(e.target.checked)}
+                    paddingLeft={4}
+                    isDisabled={!existingDatasetOptions[0]?.id}
+                  >
+                    <Text>Create New Dataset</Text>
+                  </Checkbox>
+                </Flex>
+              )}
+
+              {createNewDataset && (
+                <Flex
+                  flexDir={{ base: "column", md: "row" }}
+                  alignItems={{ base: "flex-start", md: "center" }}
+                >
+                  <Text w={48} fontWeight="bold">
+                    Dataset Name:
+                  </Text>
+                  <Input
+                    w={48}
+                    value={newDatasetName}
+                    onChange={(e) => setNewDatasetName(e.target.value)}
+                  />
+                </Flex>
+              )}
+            </VStack>
+          </VStack>
+        </ModalBody>
+        <ModalFooter>
+          <HStack>
+            <Button colorScheme="gray" onClick={disclosure.onClose} minW={24}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="blue"
+              onClick={addToDataset}
+              isLoading={addingInProgress}
+              minW={24}
+            >
+              Add
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/requestLogs/ColumnVisiblityDropdown.tsx

@@ -17,7 +17,7 @@ import { useMemo } from "react";
 import { useIsClientRehydrated, useTagNames } from "~/utils/hooks";
 import { useAppStore } from "~/state/store";
 import { StaticColumnKeys } from "~/state/columnVisiblitySlice";
-import ActionButton from "./ActionButton";
+import ActionButton from "../ActionButton";
 
 const ColumnVisiblityDropdown = () => {
   const tagNames = useTagNames().data;

app/src/components/requestLogs/ExportButton.tsx

@@ -28,7 +28,7 @@ import { BiExport } from "react-icons/bi";
 import { useHandledAsyncCallback } from "~/utils/hooks";
 import { api } from "~/utils/api";
 import { useAppStore } from "~/state/store";
-import ActionButton from "./ActionButton";
+import ActionButton from "../ActionButton";
 import InputDropdown from "../InputDropdown";
 import { FiChevronUp, FiChevronDown } from "react-icons/fi";
 import InfoCircle from "../InfoCircle";
@@ -47,6 +47,7 @@ const ExportButton = () => {
         label="Export"
         icon={BiExport}
         isDisabled={selectedLogIds.size === 0}
+        requireBeta
       />
       <ExportLogsModal disclosure={disclosure} />
     </>
@@ -80,7 +81,7 @@ const ExportLogsModal = ({ disclosure }: { disclosure: UseDisclosureReturn }) =>
       return;
     const response = await exportLogsMutation.mutateAsync({
       projectId: selectedProjectId,
-      selectedLogIds: Array.from(selectedLogIds),
+      loggedCallIds: Array.from(selectedLogIds),
       testingSplit,
       selectedExportFormat,
       removeDuplicates,

app/src/components/requestLogs/LoggedCallsTable.tsx

@@ -1,7 +1,7 @@
 import { Card, Table, Tbody } from "@chakra-ui/react";
 import { useState } from "react";
 import { useLoggedCalls } from "~/utils/hooks";
-import { TableHeader, TableRow } from "./TableRow";
+import { TableHeader, TableRow, EmptyTableRow } from "./TableRow";
 
 export default function LoggedCallsTable() {
   const [expandedRow, setExpandedRow] = useState<string | null>(null);
@@ -12,23 +12,27 @@ export default function LoggedCallsTable() {
       <Table>
         <TableHeader showOptions />
         <Tbody>
-          {loggedCalls?.calls?.map((loggedCall) => {
+          {loggedCalls?.calls.length ? (
-            return (
+            loggedCalls?.calls?.map((loggedCall) => {
-              <TableRow
+              return (
-                key={loggedCall.id}
+                <TableRow
-                loggedCall={loggedCall}
+                  key={loggedCall.id}
-                isExpanded={loggedCall.id === expandedRow}
+                  loggedCall={loggedCall}
-                onToggle={() => {
+                  isExpanded={loggedCall.id === expandedRow}
-                  if (loggedCall.id === expandedRow) {
+                  onToggle={() => {
-                    setExpandedRow(null);
+                    if (loggedCall.id === expandedRow) {
-                  } else {
+                      setExpandedRow(null);
-                    setExpandedRow(loggedCall.id);
+                    } else {
-                  }
+                      setExpandedRow(loggedCall.id);
-                }}
+                    }
-                showOptions
+                  }}
-              />
+                  showOptions
-            );
+                />
-          })}
+              );
+            })
+          ) : (
+            <EmptyTableRow />
+          )}
         </Tbody>
       </Table>
     </Card>

app/src/components/requestLogs/TableRow.tsx

@@ -9,16 +9,14 @@ import {
   Collapse,
   HStack,
   VStack,
-  Button,
-  ButtonGroup,
   Text,
   Checkbox,
+  Link as ChakraLink,
 } from "@chakra-ui/react";
-import Link from "next/link";
 
 import dayjs from "~/utils/dayjs";
 import { type RouterOutputs } from "~/utils/api";
-import { FormattedJson } from "./FormattedJson";
+import { FormattedJson } from "../FormattedJson";
 import { useAppStore } from "~/state/store";
 import { useIsClientRehydrated, useLoggedCalls, useTagNames } from "~/utils/hooks";
 import { useMemo } from "react";
@@ -175,26 +173,57 @@ export const TableRow = ({
       <Tr>
         <Td colSpan={visibleColumns.size + 1} w="full" p={0}>
           <Collapse in={isExpanded} unmountOnExit={true}>
-            <VStack p={4} align="stretch">
+            <HStack align="stretch" p={4}>
-              <HStack align="stretch">
+              <VStack flex={1} align="stretch">
-                <VStack flex={1} align="stretch">
+                <Heading size="sm">Input</Heading>
-                  <Heading size="sm">Input</Heading>
+                <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
-                  <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
+              </VStack>
-                </VStack>
+              <VStack flex={1} align="stretch">
-                <VStack flex={1} align="stretch">
+                <Heading size="sm">Output</Heading>
-                  <Heading size="sm">Output</Heading>
+                <FormattedJson json={loggedCall.modelResponse?.respPayload} />
-                  <FormattedJson json={loggedCall.modelResponse?.respPayload} />
+              </VStack>
-                </VStack>
+            </HStack>
-              </HStack>
-              <ButtonGroup alignSelf="flex-end">
-                <Button as={Link} colorScheme="blue" href={{ pathname: "/experiments" }}>
-                  Experiments
-                </Button>
-              </ButtonGroup>
-            </VStack>
           </Collapse>
         </Td>
       </Tr>
     </>
   );
 };
+
+export const EmptyTableRow = ({ filtersApplied = true }: { filtersApplied?: boolean }) => {
+  const visibleColumns = useAppStore((s) => s.columnVisibility.visibleColumns);
+  const filters = useAppStore((state) => state.logFilters.filters);
+  const { isLoading } = useLoggedCalls();
+
+  if (isLoading) return null;
+
+  if (filters.length && filtersApplied) {
+    return (
+      <Tr>
+        <Td w="full" colSpan={visibleColumns.size + 1}>
+          <Text color="gray.500" textAlign="center" w="full" p={4}>
+            No matching request logs found. Try removing some filters.
+          </Text>
+        </Td>
+      </Tr>
+    );
+  }
+
+  return (
+    <Tr>
+      <Td w="full" colSpan={visibleColumns.size + 1}>
+        <Text color="gray.500" textAlign="center" w="full" p={4}>
+          This project has no request logs. Learn how to add request logs to your project in our{" "}
+          <ChakraLink
+            href="https://docs.openpipe.ai/getting-started/quick-start"
+            target="_blank"
+            color="blue.600"
+          >
+            Quick Start
+          </ChakraLink>{" "}
+          guide.
+        </Text>
+      </Td>
+    </Tr>
+  );
+};

app/src/env.mjs

@@ -26,6 +26,9 @@ export const env = createEnv({
     SMTP_PORT: z.string().default("placeholder"),
     SMTP_LOGIN: z.string().default("placeholder"),
     SMTP_PASSWORD: z.string().default("placeholder"),
+    AZURE_STORAGE_ACCOUNT_NAME: z.string().default("placeholder"),
+    AZURE_STORAGE_ACCOUNT_KEY: z.string().default("placeholder"),
+    AZURE_STORAGE_CONTAINER_NAME: z.string().default("placeholder"),
     WORKER_CONCURRENCY: z
       .string()
       .default("10")
@@ -72,6 +75,9 @@ export const env = createEnv({
     SMTP_PORT: process.env.SMTP_PORT,
     SMTP_LOGIN: process.env.SMTP_LOGIN,
     SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+    AZURE_STORAGE_ACCOUNT_NAME: process.env.AZURE_STORAGE_ACCOUNT_NAME,
+    AZURE_STORAGE_ACCOUNT_KEY: process.env.AZURE_STORAGE_ACCOUNT_KEY,
+    AZURE_STORAGE_CONTAINER_NAME: process.env.AZURE_STORAGE_CONTAINER_NAME,
     WORKER_CONCURRENCY: process.env.WORKER_CONCURRENCY,
     WORKER_MAX_POOL_SIZE: process.env.WORKER_MAX_POOL_SIZE,
   },

app/src/modelProviders/openai-ChatCompletion/getCompletion.ts

@@ -2,7 +2,7 @@
 import { isArray, isString } from "lodash-es";
 import { APIError } from "openai";
 import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
-import mergeChunks from "openpipe/src/openai/mergeChunks";
+import mergeChunks from "openpipe/openai/mergeChunks";
 import { openai } from "~/server/utils/openai";
 import { type CompletionResponse } from "../types";
 
@@ -14,7 +14,7 @@ export async function getCompletion(
   let finalCompletion: ChatCompletion | null = null;
 
   try {
-    if (onStream) {
+    if (onStream && !input.function_call) {
       const resp = await openai.chat.completions.create(
         {
           ...input,

app/src/modelProviders/openai-ChatCompletion/index.ts

@@ -42,24 +42,21 @@ const modelProvider: OpenaiChatModelProvider = {
   canStream: true,
   getCompletion,
   getUsage: (input, output) => {
-    if (output.choices.length === 0) return null;
-
     const model = modelProvider.getModel(input);
     if (!model) return null;
 
     let inputTokens: number;
     let outputTokens: number;
 
-    if (output.usage) {
+    if (output?.usage) {
       inputTokens = output.usage.prompt_tokens;
       outputTokens = output.usage.completion_tokens;
     } else {
       try {
         inputTokens = countOpenAIChatTokens(model, input.messages);
-        outputTokens = countOpenAIChatTokens(
+        outputTokens = output
-          model,
+          ? countOpenAIChatTokens(model, output.choices.map((c) => c.message).filter(truthyFilter))
-          output.choices.map((c) => c.message).filter(truthyFilter),
+          : 0;
-        );
       } catch (err) {
         inputTokens = 0;
         outputTokens = 0;

app/src/modelProviders/replicate-llama2/getCompletion.ts

@@ -8,9 +8,9 @@ const replicate = new Replicate({
 });
 
 const modelIds: Record<ReplicateLlama2Input["model"], string> = {
-  "7b-chat": "7b0bfc9aff140d5b75bacbed23e91fd3c34b01a1e958d32132de6e0a19796e2c",
+  "7b-chat": "658b64a1e83d7caaba4ef10d5ee9c12c40770003f45852f05c2564962f921d8e",
-  "13b-chat": "2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
+  "13b-chat": "7457c09004773f9f9710f7eb3b270287ffcebcfb23a13c8ec30cfb98f6bff9b2",
-  "70b-chat": "2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1",
+  "70b-chat": "4dfd64cc207097970659087cf5670e3c1fbe02f83aa0f751e079cfba72ca790a",
 };
 
 export async function getCompletion(

app/src/modelProviders/types.ts

@@ -59,7 +59,7 @@ export type ModelProvider<SupportedModels extends string, InputSchema, OutputSch
   ) => Promise<CompletionResponse<OutputSchema>>;
   getUsage: (
     input: InputSchema,
-    output: OutputSchema,
+    output?: OutputSchema,
   ) => { gpuRuntime?: number; inputTokens?: number; outputTokens?: number; cost?: number } | null;
 
   // This is just a convenience for type inference, don't use it at runtime

app/src/pages/dashboard/index.tsx

@@ -33,7 +33,7 @@ export default function Dashboard() {
   );
 
   return (
-    <AppShell title="Dashboard" requireAuth requireBeta>
+    <AppShell title="Dashboard" requireAuth>
       <VStack px={8} py={8} alignItems="flex-start" spacing={4}>
         <Text fontSize="2xl" fontWeight="bold">
           Dashboard

app/src/pages/datasets/[id].tsx

@@ -0,0 +1,121 @@
+import {
+  Breadcrumb,
+  BreadcrumbItem,
+  Center,
+  Flex,
+  Icon,
+  Input,
+  VStack,
+  HStack,
+  useDisclosure,
+} from "@chakra-ui/react";
+import Link from "next/link";
+import { useState, useEffect } from "react";
+import { AiOutlineDatabase } from "react-icons/ai";
+
+import AppShell from "~/components/nav/AppShell";
+import { api } from "~/utils/api";
+import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import DatasetConfigurationDrawer from "~/components/datasets/DatasetConfigurationDrawer/DatasetConfigurationDrawer";
+import { DatasetHeaderButtons } from "~/components/datasets/DatasetHeaderButtons";
+import DatasetEntriesTable from "~/components/datasets/DatasetEntriesTable/DatasetEntriesTable";
+import DatasetEntryPaginator from "~/components/datasets/DatasetEntryPaginator";
+import { useAppStore } from "~/state/store";
+import FineTuneButton from "~/components/datasets/FineTuneButton";
+// import ExperimentButton from "~/components/datasets/ExperimentButton";
+import UploadDataButton from "~/components/datasets/UploadDataButton";
+// import DownloadButton from "~/components/datasets/DownloadButton";
+import DeleteButton from "~/components/datasets/DeleteButton";
+import FileUploadsCard from "~/components/datasets/FileUploadsCard";
+
+export default function Dataset() {
+  const utils = api.useContext();
+
+  const dataset = useDataset();
+
+  const drawerDisclosure = useDisclosure();
+  const [name, setName] = useState(dataset.data?.name || "");
+  useEffect(() => {
+    setName(dataset.data?.name || "");
+  }, [dataset.data?.name]);
+
+  useEffect(() => {
+    useAppStore.getState().sharedArgumentsEditor.loadMonaco().catch(console.error);
+  }, []);
+
+  const updateMutation = api.datasets.update.useMutation();
+  const [onSaveName] = useHandledAsyncCallback(async () => {
+    if (name && name !== dataset.data?.name && dataset.data?.id) {
+      await updateMutation.mutateAsync({
+        id: dataset.data.id,
+        name,
+      });
+      await Promise.all([utils.datasets.list.invalidate(), utils.datasets.get.invalidate()]);
+    }
+  }, [updateMutation, dataset.data?.id, dataset.data?.name, name]);
+
+  if (!dataset.isLoading && !dataset.data) {
+    return (
+      <AppShell title="Dataset not found">
+        <Center h="100%">
+          <div>Dataset not found 😕</div>
+        </Center>
+      </AppShell>
+    );
+  }
+
+  return (
+    <>
+      <AppShell title={dataset.data?.name}>
+        <VStack h="full" overflowY="scroll">
+          <PageHeaderContainer>
+            <Breadcrumb>
+              <BreadcrumbItem>
+                <ProjectBreadcrumbContents projectName={dataset.data?.project?.name} />
+              </BreadcrumbItem>
+              <BreadcrumbItem>
+                <Link href="/datasets">
+                  <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
+                    <Icon as={AiOutlineDatabase} boxSize={4} mr={2} /> Datasets
+                  </Flex>
+                </Link>
+              </BreadcrumbItem>
+              <BreadcrumbItem isCurrentPage>
+                <Input
+                  size="sm"
+                  value={name}
+                  onChange={(e) => setName(e.target.value)}
+                  onBlur={onSaveName}
+                  borderWidth={1}
+                  borderColor="transparent"
+                  fontSize={16}
+                  px={0}
+                  minW={{ base: 100, lg: 300 }}
+                  flex={1}
+                  _hover={{ borderColor: "gray.300" }}
+                  _focus={{ borderColor: "blue.500", outline: "none" }}
+                />
+              </BreadcrumbItem>
+            </Breadcrumb>
+            <DatasetHeaderButtons openDrawer={drawerDisclosure.onOpen} />
+          </PageHeaderContainer>
+          <VStack px={8} py={8} alignItems="flex-start" spacing={4} w="full">
+            <HStack w="full" justifyContent="flex-end">
+              <FineTuneButton />
+              <UploadDataButton />
+              {/* <ExperimentButton /> */}
+              {/* <DownloadButton /> */}
+              <DeleteButton />
+            </HStack>
+            <DatasetEntriesTable />
+            <DatasetEntryPaginator />
+          </VStack>
+        </VStack>
+        <FileUploadsCard />
+      </AppShell>
+      <DatasetConfigurationDrawer disclosure={drawerDisclosure} />
+    </>
+  );
+}

app/src/pages/datasets/index.tsx

@@ -0,0 +1,17 @@
+import { VStack, Text, Divider } from "@chakra-ui/react";
+import AppShell from "~/components/nav/AppShell";
+import DatasetsTable from "~/components/datasets/DatasetsTable";
+
+export default function DatasetsPage() {
+  return (
+    <AppShell title="Datasets" requireAuth>
+      <VStack w="full" py={8} px={8} spacing={4} alignItems="flex-start">
+        <Text fontSize="2xl" fontWeight="bold">
+          Datasets
+        </Text>
+        <Divider />
+        <DatasetsTable />
+      </VStack>
+    </AppShell>
+  );
+}

app/src/pages/experiments/[experimentSlug].tsx

@@ -8,26 +8,25 @@ import {
   Input,
   Text,
   VStack,
+  useDisclosure,
 } from "@chakra-ui/react";
 import Link from "next/link";
 
-import { useRouter } from "next/router";
 import { useState, useEffect } from "react";
 import { RiFlaskLine } from "react-icons/ri";
 import OutputsTable from "~/components/OutputsTable";
-import ExperimentSettingsDrawer from "~/components/ExperimentSettingsDrawer/ExperimentSettingsDrawer";
+import ExperimentSettingsDrawer from "~/components/experiments/ExperimentSettingsDrawer/ExperimentSettingsDrawer";
+import { ExperimentHeaderButtons } from "~/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons";
 import AppShell from "~/components/nav/AppShell";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import { useAppStore } from "~/state/store";
 import { useSyncVariantEditor } from "~/state/sync";
-import { ExperimentHeaderButtons } from "~/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons";
 import Head from "next/head";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
 import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
 export default function Experiment() {
-  const router = useRouter();
   const utils = api.useContext();
   useSyncVariantEditor();
 
@@ -44,6 +43,7 @@ export default function Experiment() {
     useAppStore.getState().sharedVariantEditor.loadMonaco().catch(console.error);
   }, []);
 
+  const drawerDisclosure = useDisclosure();
   const [label, setLabel] = useState(experiment.data?.label || "");
   useEffect(() => {
     setLabel(experiment.data?.label || "");
@@ -121,11 +121,11 @@ export default function Experiment() {
                 )}
               </BreadcrumbItem>
             </Breadcrumb>
-            <ExperimentHeaderButtons />
+            <ExperimentHeaderButtons openDrawer={drawerDisclosure.onOpen} />
           </PageHeaderContainer>
-          <ExperimentSettingsDrawer />
+          <ExperimentSettingsDrawer disclosure={drawerDisclosure} />
           <Box w="100%" overflowX="auto" flex={1} id="output-container">
-            <OutputsTable experimentId={experiment.data?.id} />
+            <OutputsTable experimentId={experiment.data?.id} openDrawer={drawerDisclosure.onOpen} />
           </Box>
         </VStack>
       </AppShell>

app/src/pages/request-logs/index.tsx

@@ -4,14 +4,13 @@ import { Text, VStack, Divider, HStack, Box } from "@chakra-ui/react";
 import AppShell from "~/components/nav/AppShell";
 import LoggedCallTable from "~/components/requestLogs/LoggedCallsTable";
 import LoggedCallsPaginator from "~/components/requestLogs/LoggedCallsPaginator";
-import ActionButton from "~/components/requestLogs/ActionButton";
+import ActionButton from "~/components/ActionButton";
 import { useAppStore } from "~/state/store";
-import { RiFlaskLine } from "react-icons/ri";
 import { FiFilter } from "react-icons/fi";
 import LogFilters from "~/components/requestLogs/LogFilters/LogFilters";
 import ColumnVisiblityDropdown from "~/components/requestLogs/ColumnVisiblityDropdown";
-import FineTuneButton from "~/components/requestLogs/FineTuneButton";
 import ExportButton from "~/components/requestLogs/ExportButton";
+import AddToDatasetButton from "~/components/requestLogs/AddToDatasetButton";
 
 export default function LoggedCalls() {
   const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
@@ -19,7 +18,7 @@ export default function LoggedCalls() {
   const [filtersShown, setFiltersShown] = useState(true);
 
   return (
-    <AppShell title="Request Logs" requireAuth requireBeta>
+    <AppShell title="Request Logs" requireAuth>
       <Box h="100vh" overflowY="scroll">
         <VStack px={8} py={8} alignItems="flex-start" spacing={4} w="full">
           <Text fontSize="2xl" fontWeight="bold">
@@ -27,15 +26,7 @@ export default function LoggedCalls() {
           </Text>
           <Divider />
           <HStack w="full" justifyContent="flex-end">
-            <FineTuneButton />
+            <AddToDatasetButton />
-            <ActionButton
-              onClick={() => {
-                console.log("experimenting with these ids", selectedLogIds);
-              }}
-              label="Experiment"
-              icon={RiFlaskLine}
-              isDisabled={selectedLogIds.size === 0}
-            />
             <ExportButton />
             <ColumnVisiblityDropdown />
             <ActionButton

app/src/server/api/external/v1Api.router.ts

@@ -119,10 +119,10 @@ export const v1ApiRouter = createOpenApiRouter({
 
       let usage;
       let model;
-      if (reqPayload.success && respPayload.success) {
+      if (reqPayload.success) {
         usage = modelProvider.getUsage(
           input.reqPayload as CompletionCreateParams,
-          input.respPayload as ChatCompletion,
+          respPayload.success ? (input.respPayload as ChatCompletion) : undefined,
         );
         model = reqPayload.data.model;
       }

app/src/server/api/root.router.ts

@@ -9,6 +9,8 @@ import { worldChampsRouter } from "./routers/worldChamps.router";
 import { projectsRouter } from "./routers/projects.router";
 import { dashboardRouter } from "./routers/dashboard.router";
 import { loggedCallsRouter } from "./routers/loggedCalls.router";
+import { datasetsRouter } from "./routers/datasets.router";
+import { datasetEntriesRouter } from "./routers/datasetEntries.router";
 import { fineTunesRouter } from "./routers/fineTunes.router";
 import { usersRouter } from "./routers/users.router";
 import { adminJobsRouter } from "./routers/adminJobs.router";
@@ -29,6 +31,8 @@ export const appRouter = createTRPCRouter({
   projects: projectsRouter,
   dashboard: dashboardRouter,
   loggedCalls: loggedCallsRouter,
+  datasets: datasetsRouter,
+  datasetEntries: datasetEntriesRouter,
   fineTunes: fineTunesRouter,
   users: usersRouter,
   adminJobs: adminJobsRouter,

app/src/server/api/routers/datasetEntries.router.ts

@@ -0,0 +1,337 @@
+import { z } from "zod";
+import { v4 as uuidv4 } from "uuid";
+import {
+  type ChatCompletion,
+  type CompletionCreateParams,
+  type CreateChatCompletionRequestMessage,
+} from "openai/resources/chat";
+import { TRPCError } from "@trpc/server";
+import archiver from "archiver";
+import { WritableStreamBuffer } from "stream-buffers";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { requireCanModifyProject, requireCanViewProject } from "~/utils/accessControl";
+import { error, success } from "~/utils/errorHandling/standardResponses";
+import { countOpenAIChatTokens } from "~/utils/countTokens";
+import { type TrainingRow } from "~/components/datasets/validateTrainingRows";
+import hashObject from "~/server/utils/hashObject";
+import { type JsonValue } from "type-fest";
+import { formatEntriesFromTrainingRows } from "~/server/utils/createEntriesFromTrainingRows";
+
+export const datasetEntriesRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(z.object({ datasetId: z.string(), page: z.number(), pageSize: z.number() }))
+    .query(async ({ input, ctx }) => {
+      const { datasetId, page, pageSize } = input;
+
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: datasetId },
+      });
+      await requireCanViewProject(projectId, ctx);
+
+      const [entries, matchingEntries, trainingCount, testingCount] = await prisma.$transaction([
+        prisma.datasetEntry.findMany({
+          where: {
+            datasetId: datasetId,
+          },
+          orderBy: [{ createdAt: "desc" }, { id: "desc" }],
+          skip: (page - 1) * pageSize,
+          take: pageSize,
+        }),
+        prisma.datasetEntry.findMany({
+          where: {
+            datasetId: datasetId,
+          },
+          select: {
+            id: true,
+          },
+        }),
+        prisma.datasetEntry.count({
+          where: {
+            datasetId: datasetId,
+            type: "TRAIN",
+          },
+        }),
+        prisma.datasetEntry.count({
+          where: {
+            datasetId: datasetId,
+            type: "TEST",
+          },
+        }),
+      ]);
+
+      return {
+        entries,
+        matchingEntryIds: matchingEntries.map((entry) => entry.id),
+        trainingCount,
+        testingCount,
+      };
+    }),
+  get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
+    const entry = await prisma.datasetEntry.findUniqueOrThrow({
+      where: { id: input.id },
+      include: {
+        dataset: true,
+      },
+    });
+
+    if (!entry.dataset) {
+      throw new TRPCError({ message: "Dataset not found for dataset entry", code: "NOT_FOUND" });
+    }
+
+    await requireCanViewProject(entry.dataset.projectId, ctx);
+
+    if (!entry) {
+      throw new TRPCError({ message: "Dataset entry not found", code: "NOT_FOUND" });
+    }
+
+    return entry;
+  }),
+  create: protectedProcedure
+    .input(
+      z.object({
+        datasetId: z.string().optional(),
+        newDatasetParams: z
+          .object({
+            projectId: z.string(),
+            name: z.string(),
+          })
+          .optional(),
+        loggedCallIds: z.string().array().optional(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      let datasetId: string;
+      let trainingRatio = 0.8;
+      if (input.datasetId) {
+        datasetId = input.datasetId;
+        const { projectId, trainingRatio: datasetTrainingRatio } =
+          await prisma.dataset.findUniqueOrThrow({
+            where: { id: input.datasetId },
+          });
+        trainingRatio = datasetTrainingRatio;
+        await requireCanModifyProject(projectId, ctx);
+      } else if (input.newDatasetParams) {
+        await requireCanModifyProject(input.newDatasetParams.projectId, ctx);
+        datasetId = uuidv4();
+      } else {
+        return error("No datasetId or newDatasetParams provided");
+      }
+
+      if (!input.loggedCallIds) {
+        return error("No loggedCallIds provided");
+      }
+
+      const loggedCalls = await prisma.loggedCall.findMany({
+        where: {
+          id: {
+            in: input.loggedCallIds,
+          },
+          modelResponse: {
+            isNot: null,
+          },
+        },
+        include: {
+          modelResponse: {
+            select: {
+              reqPayload: true,
+              respPayload: true,
+              inputTokens: true,
+              outputTokens: true,
+            },
+          },
+        },
+        orderBy: { createdAt: "desc" },
+      });
+
+      const trainingRows = loggedCalls.map((loggedCall) => {
+        const inputMessages = (
+          loggedCall.modelResponse?.reqPayload as unknown as CompletionCreateParams
+        ).messages;
+        let output: ChatCompletion.Choice.Message | undefined = undefined;
+        const resp = loggedCall.modelResponse?.respPayload as unknown as ChatCompletion | undefined;
+        if (resp && resp.choices?.[0]) {
+          output = resp.choices[0].message;
+        }
+        return {
+          input: inputMessages as unknown as CreateChatCompletionRequestMessage[],
+          output: output as unknown as CreateChatCompletionRequestMessage,
+        };
+      });
+
+      const datasetEntriesToCreate = await formatEntriesFromTrainingRows(datasetId, trainingRows);
+
+      // Ensure dataset and dataset entries are created atomically
+      await prisma.$transaction([
+        prisma.dataset.upsert({
+          where: { id: datasetId },
+          update: {},
+          create: {
+            id: datasetId,
+            projectId: input.newDatasetParams?.projectId ?? "",
+            name: input.newDatasetParams?.name ?? "",
+            trainingRatio,
+          },
+        }),
+        prisma.datasetEntry.createMany({
+          data: datasetEntriesToCreate,
+        }),
+      ]);
+
+      return success(datasetId);
+    }),
+  update: protectedProcedure
+    .input(
+      z.object({
+        id: z.string(),
+        updates: z.object({
+          type: z.enum(["TRAIN", "TEST"]).optional(),
+          input: z.string().optional(),
+          output: z.string().optional(),
+        }),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const { dataset } = await prisma.datasetEntry.findUniqueOrThrow({
+        where: { id: input.id },
+        include: {
+          dataset: true,
+        },
+      });
+
+      if (!dataset) {
+        return error("Dataset not found for dataset entry");
+      }
+
+      await requireCanModifyProject(dataset.projectId, ctx);
+
+      let parsedInput = undefined;
+      let inputTokens = undefined;
+      if (input.updates.input) {
+        parsedInput = JSON.parse(input.updates.input);
+        inputTokens = countOpenAIChatTokens(
+          "gpt-4-0613",
+          parsedInput as unknown as CreateChatCompletionRequestMessage[],
+        );
+      }
+
+      let parsedOutput = undefined;
+      let outputTokens = undefined;
+      // The client might send "null" as a string, so we need to check for that
+      if (input.updates.output && input.updates.output !== "null") {
+        parsedOutput = JSON.parse(input.updates.output);
+        outputTokens = countOpenAIChatTokens("gpt-4-0613", [
+          parsedOutput as unknown as ChatCompletion.Choice.Message,
+        ]);
+      }
+
+      await prisma.datasetEntry.update({
+        where: { id: input.id },
+        data: {
+          type: input.updates.type,
+          input: parsedInput,
+          output: parsedOutput,
+          inputTokens,
+          outputTokens,
+        },
+      });
+
+      return success("Dataset entry updated");
+    }),
+
+  delete: protectedProcedure
+    .input(z.object({ ids: z.string().array() }))
+    .mutation(async ({ input, ctx }) => {
+      if (input.ids.length === 0) {
+        return error("No ids provided");
+      }
+      const { dataset } = await prisma.datasetEntry.findUniqueOrThrow({
+        where: { id: input.ids[0] },
+        include: {
+          dataset: true,
+        },
+      });
+
+      if (!dataset) {
+        return error("Dataset not found for dataset entry");
+      }
+
+      await requireCanModifyProject(dataset.projectId, ctx);
+
+      await prisma.datasetEntry.deleteMany({
+        where: {
+          id: {
+            in: input.ids,
+          },
+          datasetId: dataset?.id,
+        },
+      });
+
+      return success("Dataset entries deleted");
+    }),
+
+  export: protectedProcedure
+    .input(
+      z.object({
+        datasetId: z.string(),
+        datasetEntryIds: z.string().array(),
+        testingSplit: z.number(),
+        removeDuplicates: z.boolean(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.datasetId },
+      });
+      await requireCanViewProject(projectId, ctx);
+
+      const datasetEntries = await ctx.prisma.datasetEntry.findMany({
+        where: {
+          id: {
+            in: input.datasetEntryIds,
+          },
+        },
+      });
+
+      let rows: TrainingRow[] = datasetEntries.map((entry) => ({
+        input: entry.input as unknown as CreateChatCompletionRequestMessage[],
+        output: entry.output as unknown as CreateChatCompletionRequestMessage,
+      }));
+
+      if (input.removeDuplicates) {
+        const deduplicatedRows = [];
+        const rowHashSet = new Set<string>();
+        for (const row of rows) {
+          const rowHash = hashObject(row as unknown as JsonValue);
+          if (!rowHashSet.has(rowHash)) {
+            rowHashSet.add(rowHash);
+            deduplicatedRows.push(row);
+          }
+        }
+        rows = deduplicatedRows;
+      }
+
+      const splitIndex = Math.floor((rows.length * input.testingSplit) / 100);
+
+      const testingData = rows.slice(0, splitIndex);
+      const trainingData = rows.slice(splitIndex);
+
+      // Convert arrays to JSONL format
+      const trainingDataJSONL = trainingData.map((item) => JSON.stringify(item)).join("\n");
+      const testingDataJSONL = testingData.map((item) => JSON.stringify(item)).join("\n");
+
+      const output = new WritableStreamBuffer();
+      const archive = archiver("zip");
+
+      archive.pipe(output);
+      archive.append(trainingDataJSONL, { name: "train.jsonl" });
+      archive.append(testingDataJSONL, { name: "test.jsonl" });
+      await archive.finalize();
+
+      // Convert buffer to base64
+      const base64 = output.getContents().toString("base64");
+
+      return base64;
+    }),
+});

app/src/server/api/routers/datasets.router.ts

@@ -0,0 +1,183 @@
+import { z } from "zod";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { requireCanModifyProject, requireCanViewProject } from "~/utils/accessControl";
+import { error, success } from "~/utils/errorHandling/standardResponses";
+import { generateServiceClientUrl } from "~/utils/azure/server";
+import { queueImportDatasetEntries } from "~/server/tasks/importDatasetEntries.task";
+
+export const datasetsRouter = createTRPCRouter({
+  get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
+    const dataset = await prisma.dataset.findUniqueOrThrow({
+      where: { id: input.id },
+      include: {
+        project: true,
+      },
+    });
+
+    await requireCanViewProject(dataset.projectId, ctx);
+
+    return dataset;
+  }),
+  list: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewProject(input.projectId, ctx);
+
+      return await prisma.dataset.findMany({
+        where: {
+          projectId: input.projectId,
+        },
+        include: {
+          _count: {
+            select: {
+              datasetEntries: true,
+            },
+          },
+        },
+        orderBy: { createdAt: "desc" },
+      });
+    }),
+
+  create: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        name: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyProject(input.projectId, ctx);
+
+      const dataset = await prisma.dataset.create({
+        data: {
+          projectId: input.projectId,
+          name: input.name,
+        },
+      });
+
+      return success(dataset);
+    }),
+
+  update: protectedProcedure
+    .input(
+      z.object({
+        id: z.string(),
+        name: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+      await requireCanModifyProject(projectId, ctx);
+
+      await prisma.dataset.update({
+        where: { id: input.id },
+        data: {
+          name: input.name,
+        },
+      });
+
+      return success("Dataset updated");
+    }),
+
+  delete: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+      await requireCanModifyProject(projectId, ctx);
+
+      await prisma.dataset.delete({
+        where: { id: input.id },
+      });
+
+      return success("Dataset deleted");
+    }),
+  getServiceClientUrl: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      // The user must at least be authenticated to get a SAS token
+      await requireCanModifyProject(input.projectId, ctx);
+      return generateServiceClientUrl();
+    }),
+  triggerFileDownload: protectedProcedure
+    .input(
+      z.object({
+        datasetId: z.string(),
+        blobName: z.string(),
+        fileName: z.string(),
+        fileSize: z.number(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.datasetId },
+      });
+      await requireCanViewProject(projectId, ctx);
+
+      const { id } = await prisma.datasetFileUpload.create({
+        data: {
+          datasetId: input.datasetId,
+          blobName: input.blobName,
+          status: "PENDING",
+          fileName: input.fileName,
+          fileSize: input.fileSize,
+          uploadedAt: new Date(),
+        },
+      });
+
+      await queueImportDatasetEntries(id);
+    }),
+  listFileUploads: protectedProcedure
+    .input(z.object({ datasetId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: { id: input.datasetId },
+      });
+      await requireCanViewProject(projectId, ctx);
+
+      return await prisma.datasetFileUpload.findMany({
+        where: {
+          datasetId: input.datasetId,
+          visible: true,
+        },
+        orderBy: { createdAt: "desc" },
+      });
+    }),
+  hideFileUploads: protectedProcedure
+    .input(z.object({ fileUploadIds: z.string().array() }))
+    .mutation(async ({ input, ctx }) => {
+      if (!input.fileUploadIds.length) return error("No file upload ids provided");
+
+      const {
+        dataset: { projectId, id: datasetId },
+      } = await prisma.datasetFileUpload.findUniqueOrThrow({
+        where: { id: input.fileUploadIds[0] },
+        select: {
+          dataset: {
+            select: {
+              id: true,
+              projectId: true,
+            },
+          },
+        },
+      });
+      await requireCanModifyProject(projectId, ctx);
+
+      await prisma.datasetFileUpload.updateMany({
+        where: {
+          id: {
+            in: input.fileUploadIds,
+          },
+          datasetId,
+        },
+        data: {
+          visible: false,
+        },
+      });
+    }),
+});

app/src/server/api/routers/fineTunes.router.ts

@@ -1,6 +1,4 @@
 import { z } from "zod";
-import { v4 as uuidv4 } from "uuid";
-import { type Prisma } from "@prisma/client";
 
 import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
@@ -55,14 +53,18 @@ export const fineTunesRouter = createTRPCRouter({
   create: protectedProcedure
     .input(
       z.object({
-        projectId: z.string(),
+        datasetId: z.string(),
-        selectedLogIds: z.array(z.string()),
         slug: z.string(),
         baseModel: z.string(),
       }),
     )
     .mutation(async ({ input, ctx }) => {
-      await requireCanModifyProject(input.projectId, ctx);
+      const { projectId } = await prisma.dataset.findUniqueOrThrow({
+        where: {
+          id: input.datasetId,
+        },
+      });
+      await requireCanModifyProject(projectId, ctx);
 
       const existingFineTune = await prisma.fineTune.findFirst({
         where: {
@@ -74,39 +76,14 @@ export const fineTunesRouter = createTRPCRouter({
         return error("A fine tune with that slug already exists");
       }
 
-      const newDatasetId = uuidv4();
+      await prisma.fineTune.create({
-
+        data: {
-      const datasetEntriesToCreate: Prisma.DatasetEntryCreateManyDatasetInput[] =
+          projectId,
-        input.selectedLogIds.map((loggedCallId) => ({
+          slug: input.slug,
-          loggedCallId,
+          baseModel: input.baseModel,
-        }));
+          datasetId: input.datasetId,
-
+        },
-      await prisma.$transaction([
+      });
-        prisma.dataset.create({
-          data: {
-            id: newDatasetId,
-            name: input.slug,
-            project: {
-              connect: {
-                id: input.projectId,
-              },
-            },
-            datasetEntries: {
-              createMany: {
-                data: datasetEntriesToCreate,
-              },
-            },
-          },
-        }),
-        prisma.fineTune.create({
-          data: {
-            projectId: input.projectId,
-            slug: input.slug,
-            baseModel: input.baseModel,
-            datasetId: newDatasetId,
-          },
-        }),
-      ]);
 
       return success();
     }),

app/src/server/api/routers/loggedCalls.router.ts

@@ -189,7 +189,7 @@ export const loggedCallsRouter = createTRPCRouter({
     .input(
       z.object({
         projectId: z.string(),
-        selectedLogIds: z.string().array(),
+        loggedCallIds: z.string().array(),
         testingSplit: z.number(),
         selectedExportFormat: z.string(),
         removeDuplicates: z.boolean(),
@@ -203,7 +203,7 @@ export const loggedCallsRouter = createTRPCRouter({
         where: {
           originalLoggedCall: {
             projectId: input.projectId,
-            id: { in: input.selectedLogIds },
+            id: { in: input.loggedCallIds },
           },
           statusCode: 200,
         },

app/src/server/api/routers/promptVariants.router.ts

@@ -93,17 +93,12 @@ export const promptVariantsRouter = createTRPCRouter({
           visible: true,
         },
       });
-      const outputCount = await prisma.scenarioVariantCell.count({
+      const finishedCount = await prisma.scenarioVariantCell.count({
         where: {
           promptVariantId: input.variantId,
           testScenario: { visible: true },
-          modelResponses: {
+          retrievalStatus: {
-            some: {
+            in: ["COMPLETE", "ERROR"],
-              outdated: false,
-              respPayload: {
-                not: Prisma.AnyNull,
-              },
-            },
           },
         },
       });
@@ -131,7 +126,7 @@ export const promptVariantsRouter = createTRPCRouter({
       const inputTokens = overallTokens._sum?.inputTokens ?? 0;
       const outputTokens = overallTokens._sum?.outputTokens ?? 0;
 
-      const awaitingCompletions = outputCount < scenarioCount;
+      const awaitingCompletions = finishedCount < scenarioCount;
 
       const awaitingEvals = !!evalResults.find(
         (result) => result.totalCount < scenarioCount * evals.length,
@@ -143,7 +138,7 @@ export const promptVariantsRouter = createTRPCRouter({
         outputTokens,
         overallCost: overallTokens._sum?.cost ?? 0,
         scenarioCount,
-        outputCount,
+        finishedCount,
         awaitingCompletions,
         awaitingEvals,
       };
@@ -196,7 +191,10 @@ export const promptVariantsRouter = createTRPCRouter({
           ? `${originalVariant?.label} Copy`
           : `Prompt Variant ${largestSortIndex + 2}`;
 
-      const newConstructFn = await deriveNewConstructFn(originalVariant);
+      const newConstructFn = await deriveNewConstructFn(
+        originalVariant,
+        originalVariant?.promptConstructor,
+      );
 
       const createNewVariantAction = prisma.promptVariant.create({
         data: {

app/src/server/tasks/importDatasetEntries.task.ts

@@ -0,0 +1,152 @@
+import { type DatasetFileUpload } from "@prisma/client";
+import { prisma } from "~/server/db";
+import defineTask from "./defineTask";
+import { downloadBlobToString } from "~/utils/azure/server";
+import {
+  type TrainingRow,
+  validateTrainingRows,
+  parseJSONL,
+} from "~/components/datasets/validateTrainingRows";
+import { formatEntriesFromTrainingRows } from "~/server/utils/createEntriesFromTrainingRows";
+
+export type ImportDatasetEntriesJob = {
+  datasetFileUploadId: string;
+};
+
+export const importDatasetEntries = defineTask<ImportDatasetEntriesJob>(
+  "importDatasetEntries",
+  async (task) => {
+    const { datasetFileUploadId } = task;
+    const datasetFileUpload = await prisma.datasetFileUpload.findUnique({
+      where: { id: datasetFileUploadId },
+    });
+    if (!datasetFileUpload) {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          errorMessage: "Dataset File Upload not found",
+          status: "ERROR",
+        },
+      });
+      return;
+    }
+    await prisma.datasetFileUpload.update({
+      where: { id: datasetFileUploadId },
+      data: {
+        status: "DOWNLOADING",
+        progress: 5,
+      },
+    });
+
+    const onBlobDownloadProgress = async (progress: number) => {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          progress: 5 + Math.floor((progress / datasetFileUpload.fileSize) * 25),
+        },
+      });
+    };
+
+    const jsonlStr = await downloadBlobToString(datasetFileUpload.blobName, onBlobDownloadProgress);
+
+    let trainingRows: TrainingRow[] = [];
+    let validationError: string | null = null;
+    try {
+      trainingRows = parseJSONL(jsonlStr) as TrainingRow[];
+      validationError = validateTrainingRows(trainingRows);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (e: any) {
+      validationError = e.message;
+    }
+
+    if (validationError) {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          errorMessage: `Invalid JSONL: ${validationError}`,
+          status: "ERROR",
+        },
+      });
+      return;
+    }
+
+    await prisma.datasetFileUpload.update({
+      where: { id: datasetFileUploadId },
+      data: {
+        status: "PROCESSING",
+        progress: 30,
+      },
+    });
+
+    const updatePromises: Promise<DatasetFileUpload>[] = [];
+
+    const updateCallback = async (progress: number) => {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          progress: 30 + Math.floor((progress / trainingRows.length) * 69),
+        },
+      });
+    };
+
+    let datasetEntriesToCreate;
+    try {
+      datasetEntriesToCreate = await formatEntriesFromTrainingRows(
+        datasetFileUpload.datasetId,
+        trainingRows,
+        updateCallback,
+        500,
+      );
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (e: any) {
+      await prisma.datasetFileUpload.update({
+        where: { id: datasetFileUploadId },
+        data: {
+          errorMessage: `Error formatting rows: ${e.message as string}`,
+          status: "ERROR",
+          visible: true,
+        },
+      });
+      return;
+    }
+
+    await Promise.all(updatePromises);
+
+    await prisma.datasetFileUpload.update({
+      where: { id: datasetFileUploadId },
+      data: {
+        status: "SAVING",
+        progress: 99,
+      },
+    });
+
+    await prisma.datasetEntry.createMany({
+      data: datasetEntriesToCreate,
+    });
+
+    await prisma.datasetFileUpload.update({
+      where: { id: datasetFileUploadId },
+      data: {
+        status: "COMPLETE",
+        progress: 100,
+        visible: true,
+      },
+    });
+  },
+);
+
+export const queueImportDatasetEntries = async (datasetFileUploadId: string) => {
+  await Promise.all([
+    prisma.datasetFileUpload.update({
+      where: {
+        id: datasetFileUploadId,
+      },
+      data: {
+        errorMessage: null,
+        status: "PENDING",
+      },
+    }),
+
+    importDatasetEntries.enqueue({ datasetFileUploadId }),
+  ]);
+};

app/src/server/tasks/worker.ts

@@ -5,10 +5,11 @@ import "../../../sentry.server.config";
 import { env } from "~/env.mjs";
 import { queryModel } from "./queryModel.task";
 import { runNewEval } from "./runNewEval.task";
+import { importDatasetEntries } from "./importDatasetEntries.task";
 
 console.log("Starting worker");
 
-const registeredTasks = [queryModel, runNewEval];
+const registeredTasks = [queryModel, runNewEval, importDatasetEntries];
 
 const taskList = registeredTasks.reduce((acc, task) => {
   acc[task.task.identifier] = task.task.handler;

app/src/server/utils/createEntriesFromTrainingRows.ts

@@ -0,0 +1,70 @@
+import { type Prisma } from "@prisma/client";
+import { shuffle } from "lodash-es";
+import {
+  type CreateChatCompletionRequestMessage,
+  type ChatCompletion,
+} from "openai/resources/chat";
+
+import { prisma } from "~/server/db";
+import { type TrainingRow } from "~/components/datasets/validateTrainingRows";
+import { countLlamaChatTokens } from "~/utils/countTokens";
+
+export const formatEntriesFromTrainingRows = async (
+  datasetId: string,
+  trainingRows: TrainingRow[],
+  updateCallback?: (progress: number) => Promise<void>,
+  updateFrequency = 1000,
+) => {
+  const [dataset, existingTrainingCount, existingTestingCount] = await prisma.$transaction([
+    prisma.dataset.findUnique({ where: { id: datasetId } }),
+    prisma.datasetEntry.count({
+      where: {
+        datasetId,
+        type: "TRAIN",
+      },
+    }),
+    prisma.datasetEntry.count({
+      where: {
+        datasetId,
+        type: "TEST",
+      },
+    }),
+  ]);
+
+  const trainingRatio = dataset?.trainingRatio ?? 0.8;
+
+  const newTotalEntries = existingTrainingCount + existingTestingCount + trainingRows.length;
+  const numTrainingToAdd = Math.floor(trainingRatio * newTotalEntries) - existingTrainingCount;
+  const numTestingToAdd = trainingRows.length - numTrainingToAdd;
+  const typesToAssign = shuffle([
+    ...Array(numTrainingToAdd).fill("TRAIN"),
+    ...Array(numTestingToAdd).fill("TEST"),
+  ]);
+  const datasetEntriesToCreate: Prisma.DatasetEntryCreateManyInput[] = [];
+  let i = 0;
+  for (const row of trainingRows) {
+    // console.log(row);
+    if (updateCallback && i % updateFrequency === 0) await updateCallback(i);
+    let outputTokens = 0;
+    if (row.output) {
+      outputTokens = countLlamaChatTokens([row.output as unknown as ChatCompletion.Choice.Message]);
+    }
+    // console.log("outputTokens", outputTokens);
+    datasetEntriesToCreate.push({
+      datasetId: datasetId,
+      input: row.input as unknown as Prisma.InputJsonValue,
+      output: (row.output as unknown as Prisma.InputJsonValue) ?? {
+        role: "assistant",
+        content: "",
+      },
+      inputTokens: countLlamaChatTokens(
+        row.input as unknown as CreateChatCompletionRequestMessage[],
+      ),
+      outputTokens,
+      type: typesToAssign.pop() as "TRAIN" | "TEST",
+    });
+    i++;
+  }
+
+  return datasetEntriesToCreate;
+};

app/src/server/utils/deriveNewContructFn.ts

@@ -28,15 +28,15 @@ export async function deriveNewConstructFn(
     );
   }
   return dedent`
-    prompt = {
+  definePrompt("openai/ChatCompletion", {
-      model: "gpt-3.5-turbo",
+    model: "gpt-3.5-turbo-0613",
-      messages: [
+    messages: [
-        {
+      {
-          role: "system",
+        role: "system",
-          content: "Return 'Hello, world!'",
+        content: \`Hello, world!\`,
-        }
+      },
-      ]
+    ],
-    }`;
+  });`;
 }
 
 const NUM_RETRIES = 5;

app/src/server/utils/openai.ts

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import OpenAI, { type ClientOptions } from "openpipe/src/openai";
+import OpenAI, { type ClientOptions } from "openpipe/openai";
 
 import { env } from "~/env.mjs";
 

app/src/state/selectedDatasetEntriesSlice.ts

@@ -0,0 +1,33 @@
+import { type SliceCreator } from "./store";
+
+export type SelectedDatasetEntriesSlice = {
+  selectedIds: Set<string>;
+  toggleSelectedId: (id: string) => void;
+  addSelectedIds: (ids: string[]) => void;
+  clearSelectedIds: () => void;
+};
+
+export const createSelectedDatasetEntriesSlice: SliceCreator<SelectedDatasetEntriesSlice> = (
+  set,
+) => ({
+  selectedIds: new Set(),
+  toggleSelectedId: (id: string) =>
+    set((state) => {
+      if (state.selectedDatasetEntries.selectedIds.has(id)) {
+        state.selectedDatasetEntries.selectedIds.delete(id);
+      } else {
+        state.selectedDatasetEntries.selectedIds.add(id);
+      }
+    }),
+  addSelectedIds: (ids: string[]) =>
+    set((state) => {
+      state.selectedDatasetEntries.selectedIds = new Set([
+        ...state.selectedDatasetEntries.selectedIds,
+        ...ids,
+      ]);
+    }),
+  clearSelectedIds: () =>
+    set((state) => {
+      state.selectedDatasetEntries.selectedIds = new Set();
+    }),
+});

app/src/state/selectedLogsSlice.ts

@@ -7,7 +7,7 @@ export type SelectedLogsSlice = {
   clearSelectedLogIds: () => void;
 };
 
-export const createSelectedLogsSlice: SliceCreator<SelectedLogsSlice> = (set, get) => ({
+export const createSelectedLogsSlice: SliceCreator<SelectedLogsSlice> = (set) => ({
   selectedLogIds: new Set(),
   toggleSelectedLogId: (id: string) =>
     set((state) => {

app/src/state/sharedArgumentsEditor.slice.ts

@@ -0,0 +1,33 @@
+import loader, { type Monaco } from "@monaco-editor/loader";
+
+import { type SliceCreator } from "./store";
+
+export const editorBackground = "#fafafa";
+
+export type SharedArgumentsEditorSlice = {
+  monaco: null | Monaco;
+  loadMonaco: () => Promise<void>;
+};
+
+export const createArgumentsEditorSlice: SliceCreator<SharedArgumentsEditorSlice> = (set, get) => ({
+  monaco: loader.__getMonacoInstance(),
+  loadMonaco: async () => {
+    // We only want to run this client-side
+    if (typeof window === "undefined") return;
+
+    const monaco = await loader.init();
+
+    monaco.editor.defineTheme("customTheme", {
+      base: "vs",
+      inherit: true,
+      rules: [],
+      colors: {
+        "editor.background": "#ffffff",
+      },
+    });
+
+    set((state) => {
+      state.sharedArgumentsEditor.monaco = monaco;
+    });
+  },
+});

app/src/state/store.ts

@@ -7,9 +7,17 @@ import {
   type SharedVariantEditorSlice,
   createVariantEditorSlice,
 } from "./sharedVariantEditor.slice";
+import {
+  type SharedArgumentsEditorSlice,
+  createArgumentsEditorSlice,
+} from "./sharedArgumentsEditor.slice";
 import { type APIClient } from "~/utils/api";
 import { type PersistedState, persistOptions } from "./persist";
 import { type SelectedLogsSlice, createSelectedLogsSlice } from "./selectedLogsSlice";
+import {
+  type SelectedDatasetEntriesSlice,
+  createSelectedDatasetEntriesSlice,
+} from "./selectedDatasetEntriesSlice";
 import { type LogFiltersSlice, createLogFiltersSlice } from "./logFiltersSlice";
 import { type ColumnVisibilitySlice, createColumnVisibilitySlice } from "./columnVisiblitySlice";
 import { type FeatureFlagsSlice, createFeatureFlagsSlice } from "./featureFlags";
@@ -18,15 +26,14 @@ enableMapSet();
 
 export type State = {
   isRehydrated: boolean;
-  drawerOpen: boolean;
-  openDrawer: () => void;
-  closeDrawer: () => void;
   api: APIClient | null;
   setApi: (api: APIClient) => void;
   sharedVariantEditor: SharedVariantEditorSlice;
+  sharedArgumentsEditor: SharedArgumentsEditorSlice;
   selectedProjectId: string | null;
   setSelectedProjectId: (id: string) => void;
   selectedLogs: SelectedLogsSlice;
+  selectedDatasetEntries: SelectedDatasetEntriesSlice;
   logFilters: LogFiltersSlice;
   columnVisibility: ColumnVisibilitySlice;
   featureFlags: FeatureFlagsSlice;
@@ -46,22 +53,15 @@ const useBaseStore = create<State, [["zustand/persist", PersistedState], ["zusta
         set((state) => {
           state.api = api;
         }),
-      drawerOpen: false,
-      openDrawer: () =>
-        set((state) => {
-          state.drawerOpen = true;
-        }),
-      closeDrawer: () =>
-        set((state) => {
-          state.drawerOpen = false;
-        }),
       sharedVariantEditor: createVariantEditorSlice(set, get, ...rest),
+      sharedArgumentsEditor: createArgumentsEditorSlice(set, get, ...rest),
       selectedProjectId: null,
       setSelectedProjectId: (id: string) =>
         set((state) => {
           state.selectedProjectId = id;
         }),
       selectedLogs: createSelectedLogsSlice(set, get, ...rest),
+      selectedDatasetEntries: createSelectedDatasetEntriesSlice(set, get, ...rest),
       logFilters: createLogFiltersSlice(set, get, ...rest),
       columnVisibility: createColumnVisibilitySlice(set, get, ...rest),
       featureFlags: createFeatureFlagsSlice(set, get, ...rest),

app/src/utils/azure/server.ts

@@ -0,0 +1,93 @@
+import {
+  BlobServiceClient,
+  generateAccountSASQueryParameters,
+  AccountSASPermissions,
+  AccountSASServices,
+  AccountSASResourceTypes,
+  StorageSharedKeyCredential,
+  SASProtocol,
+} from "@azure/storage-blob";
+
+const accountName = process.env.AZURE_STORAGE_ACCOUNT_NAME;
+if (!accountName) throw Error("Azure Storage accountName not found");
+const accountKey = process.env.AZURE_STORAGE_ACCOUNT_KEY;
+if (!accountKey) throw Error("Azure Storage accountKey not found");
+const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
+if (!containerName) throw Error("Azure Storage containerName not found");
+
+const sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey);
+
+const blobServiceClient = new BlobServiceClient(
+  `https://${accountName}.blob.core.windows.net`,
+  sharedKeyCredential,
+);
+
+const containerClient = blobServiceClient.getContainerClient(containerName);
+
+export const generateServiceClientUrl = () => {
+  const sasOptions = {
+    services: AccountSASServices.parse("b").toString(), // blobs
+    resourceTypes: AccountSASResourceTypes.parse("sco").toString(), // service, container, object
+    permissions: AccountSASPermissions.parse("w"), // write permissions
+    protocol: SASProtocol.Https,
+    startsOn: new Date(),
+    expiresOn: new Date(new Date().valueOf() + 10 * 60 * 1000), // 10 minutes
+  };
+  let sasToken = generateAccountSASQueryParameters(sasOptions, sharedKeyCredential).toString();
+
+  // remove leading "?"
+  sasToken = sasToken[0] === "?" ? sasToken.substring(1) : sasToken;
+  return {
+    serviceClientUrl: `https://${accountName}.blob.core.windows.net?${sasToken}`,
+    containerName,
+  };
+};
+
+export async function downloadBlobToString(
+  blobName: string,
+  onProgress?: (progress: number) => Promise<void>,
+  chunkInterval?: number,
+) {
+  const blobClient = containerClient.getBlobClient(blobName);
+
+  const downloadResponse = await blobClient.download();
+
+  if (!downloadResponse) throw Error("error downloading blob");
+  if (!downloadResponse.readableStreamBody)
+    throw Error("downloadResponse.readableStreamBody not found");
+
+  const downloaded = await streamToBuffer(
+    downloadResponse.readableStreamBody,
+    onProgress,
+    chunkInterval,
+  );
+  return downloaded.toString();
+}
+
+async function streamToBuffer(
+  readableStream: NodeJS.ReadableStream,
+  onProgress?: (progress: number) => Promise<void>,
+  chunkInterval = 1048576, // send progress every 1MB
+): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const chunks: Uint8Array[] = [];
+    let bytesDownloaded = 0;
+    let lastReportedByteCount = 0;
+
+    readableStream.on("data", (data: ArrayBuffer) => {
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
+      bytesDownloaded += data.byteLength;
+
+      if (onProgress && bytesDownloaded - lastReportedByteCount >= chunkInterval) {
+        void onProgress(bytesDownloaded); // progress in Bytes
+        lastReportedByteCount = bytesDownloaded;
+      }
+    });
+
+    readableStream.on("end", () => {
+      resolve(Buffer.concat(chunks));
+    });
+
+    readableStream.on("error", reject);
+  });
+}

app/src/utils/azure/website.ts

@@ -0,0 +1,30 @@
+import { BlobServiceClient } from "@azure/storage-blob";
+import { v4 as uuidv4 } from "uuid";
+
+import { useAppStore } from "~/state/store";
+
+export const uploadDatasetEntryFile = async (file: File) => {
+  const { selectedProjectId: projectId, api } = useAppStore.getState();
+  if (!projectId) throw Error("projectId not found");
+  if (!api) throw Error("api not initialized");
+  const { serviceClientUrl, containerName } = await api.client.datasets.getServiceClientUrl.query({
+    projectId,
+  });
+
+  const blobServiceClient = new BlobServiceClient(serviceClientUrl);
+  // create container client
+  const containerClient = blobServiceClient.getContainerClient(containerName);
+
+  // base name without extension
+  const basename = file.name.split("/").pop()?.split(".").shift();
+  if (!basename) throw Error("basename not found");
+
+  const blobName = `${basename}-${uuidv4()}.jsonl`;
+  // create blob client
+  const blobClient = containerClient.getBlockBlobClient(blobName);
+
+  // upload file
+  await blobClient.uploadData(file);
+
+  return blobName;
+};

app/src/utils/countTokens.ts

@@ -1,5 +1,7 @@
 import { type ChatCompletion } from "openai/resources/chat";
 import { GPTTokens } from "gpt-tokens";
+import llamaTokenizer from "llama-tokenizer-js";
+
 import { type SupportedModel } from "~/modelProviders/openai-ChatCompletion";
 
 interface GPTTokensMessageItem {
@@ -12,6 +14,21 @@ export const countOpenAIChatTokens = (
   model: SupportedModel,
   messages: ChatCompletion.Choice.Message[],
 ) => {
-  return new GPTTokens({ model, messages: messages as unknown as GPTTokensMessageItem[] })
+  const reformattedMessages = messages.map((message) => ({
-    .usedTokens;
+    role: message.role,
+    // Not completely accurate, but gives a rough idea of the token count
+    content: message.content ?? JSON.stringify(message.function_call),
+  }));
+  return new GPTTokens({
+    model,
+    messages: reformattedMessages as unknown as GPTTokensMessageItem[],
+  }).usedTokens;
+};
+
+export const countLlamaChatTokens = (messages: ChatCompletion.Choice.Message[]) => {
+  const stringToTokenize = messages
+    .map((message) => message.content || JSON.stringify(message.function_call))
+    .join("\n");
+  const tokens = llamaTokenizer.encode(stringToTokenize);
+  return tokens.length;
 };

app/src/utils/hooks.ts

@@ -148,13 +148,56 @@ export const useScenarioVars = () => {
   );
 };
 
-export const useLoggedCalls = () => {
+export const useDatasets = () => {
+  const selectedProjectId = useAppStore((state) => state.selectedProjectId);
+  return api.datasets.list.useQuery(
+    { projectId: selectedProjectId ?? "" },
+    { enabled: !!selectedProjectId },
+  );
+};
+
+export const useDataset = () => {
+  const router = useRouter();
+  const dataset = api.datasets.get.useQuery(
+    { id: router.query.id as string },
+    { enabled: !!router.query.id },
+  );
+
+  return dataset;
+};
+
+export const useDatasetEntries = () => {
+  const dataset = useDataset().data;
+  const { page, pageSize } = usePageParams();
+
+  const { data, isLoading, ...rest } = api.datasetEntries.list.useQuery(
+    { datasetId: dataset?.id ?? "", page, pageSize },
+    { enabled: !!dataset?.id },
+  );
+
+  const [stableData, setStableData] = useState(data);
+
+  useEffect(() => {
+    // Prevent annoying flashes while logs are loading from the server
+    if (!isLoading) {
+      setStableData(data);
+    }
+  }, [data, isLoading]);
+
+  return { data: stableData, isLoading, ...rest };
+};
+
+export const useDatasetEntry = (entryId: string | null) => {
+  return api.datasetEntries.get.useQuery({ id: entryId as string }, { enabled: !!entryId });
+};
+
+export const useLoggedCalls = (applyFilters = true) => {
   const selectedProjectId = useAppStore((state) => state.selectedProjectId);
   const { page, pageSize } = usePageParams();
   const filters = useAppStore((state) => state.logFilters.filters);
 
   const { data, isLoading, ...rest } = api.loggedCalls.list.useQuery(
-    { projectId: selectedProjectId ?? "", page, pageSize, filters },
+    { projectId: selectedProjectId ?? "", page, pageSize, filters: applyFilters ? filters : [] },
     { enabled: !!selectedProjectId },
   );
 

app/src/utils/utils.ts

@@ -10,3 +10,60 @@ export const lookupModel = (provider: string, model: string) => {
 
 export const modelLabel = (provider: string, model: string) =>
   `${provider}/${lookupModel(provider, model)?.name ?? model}`;
+
+// Check if the str could be parsed to a message function call
+export const parseableToFunctionCall = (str: string) => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let parsedJSON: any;
+  try {
+    parsedJSON = JSON.parse(str);
+  } catch {
+    return false;
+  }
+
+  // Check if the parsedJSON is an object and not null
+  if (typeof parsedJSON !== "object" || parsedJSON === null) {
+    return false;
+  }
+
+  // Check if only the keys "name" and "arguments" exist
+  const keys = Object.keys(parsedJSON as Record<string, unknown>);
+  if (keys.length !== 2 || !keys.includes("name") || !keys.includes("arguments")) {
+    return false;
+  }
+
+  // Check if both "name" and "arguments" are of type string
+  if (typeof parsedJSON.name !== "string" || typeof parsedJSON.arguments !== "string") {
+    return false;
+  }
+
+  // Check if the "arguments" value is parseable to an object
+  let parsedArguments: unknown;
+  try {
+    parsedArguments = JSON.parse(parsedJSON["arguments"]);
+  } catch {
+    return false;
+  }
+
+  // Check if parsedArguments is an object and not null
+  if (typeof parsedArguments !== "object" || parsedArguments === null) {
+    return false;
+  }
+
+  return true;
+};
+
+export const formatFileSize = (bytes: number, decimals = 2) => {
+  if (bytes === 0) return "0 Bytes";
+
+  const k = 1024;
+  const dm = decimals < 0 ? 0 : decimals;
+  const sizes = ["Bytes", "KB", "MB", "GB", "TB"];
+
+  for (const size of sizes) {
+    if (bytes < k) return `${parseFloat(bytes.toFixed(dm))} ${size}`;
+    bytes /= k;
+  }
+
+  return "> 1024 TB";
+};

app/tsconfig.json

@@ -19,7 +19,9 @@
     "baseUrl": ".",
     "paths": {
       "~/*": ["./src/*"]
-    }
+    },
+    "typeRoots": ["./types", "./node_modules/@types"],
+    "types": ["llama-tokenizer-js", "node"]
   },
   "include": [
     ".eslintrc.cjs",

app/types/llama-tokenizer-js/index.d.ts

@@ -0,0 +1,4 @@
+declare module "llama-tokenizer-js" {
+  export function encode(input: string): number[];
+  export function decode(input: number[]): string;
+}

client-libs/python/README.md

@@ -3,6 +3,7 @@
 This client allows you automatically report your OpenAI calls to [OpenPipe](https://openpipe.ai/). OpenPipe
 
 ## Installation
+
 `pip install openpipe`
 
 ## Usage
@@ -15,7 +16,7 @@ This client allows you automatically report your OpenAI calls to [OpenPipe](http
 from openpipe import openai, configure_openpipe
 import os
 
-# Set the OpenPipe API key you got in step (3) above.
+# Set the OpenPipe API key you got in step (2) above.
 # If you have the `OPENPIPE_API_KEY` environment variable set we'll read from it by default.
 configure_openpipe(api_key=os.getenv("OPENPIPE_API_KEY"))
 
@@ -23,7 +24,7 @@ configure_openpipe(api_key=os.getenv("OPENPIPE_API_KEY"))
 openai.api_key = os.getenv("OPENAI_API_KEY")
 ```
 
-You can use the OpenPipe client for normal
+You can now use your new OpenAI client, which functions identically to the generic OpenAI client while also reporting calls to your OpenPipe instance.
 
 ## Special Features
 
@@ -37,4 +38,4 @@ completion = openai.ChatCompletion.create(
     messages=[{"role": "system", "content": "count to 10"}],
     openpipe={"tags": {"prompt_id": "counting"}},
 )
-```
+```

client-libs/python/openpipe/shared.py

@@ -6,11 +6,9 @@ from openpipe.api_client.client import AuthenticatedClient
 from openpipe.api_client.models.report_json_body_tags import (
     ReportJsonBodyTags,
 )
-import toml
 import time
 import os
-
+import pkg_resources
-version = toml.load("pyproject.toml")["tool"]["poetry"]["version"]
 
 configured_client = AuthenticatedClient(
     base_url="https://app.openpipe.ai/api/v1", token=""
@@ -23,7 +21,7 @@ if os.environ.get("OPENPIPE_API_KEY"):
 def _get_tags(openpipe_options):
     tags = openpipe_options.get("tags") or {}
     tags["$sdk"] = "python"
-    tags["$sdk.version"] = version
+    tags["$sdk.version"] = pkg_resources.get_distribution('openpipe').version
 
     return ReportJsonBodyTags.from_dict(tags)
 

client-libs/python/poetry.lock

@@ -1056,6 +1056,7 @@ files = [
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
+    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
     {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
     {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
     {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
@@ -1063,8 +1064,15 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
+    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
     {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
+    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
+    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
+    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
+    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
+    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
     {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
     {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
     {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
@@ -1081,6 +1089,7 @@ files = [
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
+    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
     {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
     {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
     {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
@@ -1088,6 +1097,7 @@ files = [
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
+    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
     {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
     {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
     {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
@@ -1147,17 +1157,6 @@ files = [
     {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"},
 ]
 
-[[package]]
-name = "toml"
-version = "0.10.2"
-description = "Python Library for Tom's Obvious, Minimal Language"
-optional = false
-python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
-files = [
-    {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
-    {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
-]
-
 [[package]]
 name = "tomli"
 version = "2.0.1"
@@ -1367,4 +1366,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "e93c2ecac1b81a4fc1f9ad3dcedf03b1126cc6815e084ae233da7d3ece313ade"
+content-hash = "f50c3ee43ebb9510bf42b9a16d8d6a92d561bec40e8f3c11fb2614e92a5b756f"

client-libs/python/publish.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+# Check if PYPI_OPENPIPE_TOKEN is set
+if [[ -z "${PYPI_OPENPIPE_TOKEN}" ]]; then
+    echo "Error: PYPI_OPENPIPE_TOKEN is not set."
+    exit 1
+fi
+
+# If the token is set, proceed with publishing
+poetry publish --build --username=__token__ --password=$PYPI_OPENPIPE_TOKEN

client-libs/python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "openpipe"
-version = "3.0.1"
+version = "3.1.2"
 description = "Python client library for the OpenPipe service"
 authors = ["Kyle Corbitt <kyle@openpipe.ai>"]
 license = "Apache-2.0"
@@ -14,7 +14,6 @@ openai = "^0.27.8"
 httpx = "^0.24.1"
 attrs = "^23.1.0"
 python-dateutil = "^2.8.2"
-toml = "^0.10.2"
 
 [tool.poetry.dev-dependencies]
 

client-libs/typescript/README.md

@@ -0,0 +1,70 @@
+# OpenPipe Node API Library
+
+[![NPM version](https://img.shields.io/npm/v/openpipe.svg)](https://npmjs.org/package/openpipe)
+
+This library wraps TypeScript or Javascript OpenAI API calls and logs additional data to the configured `OPENPIPE_BASE_URL` for further processing.
+
+It is fully compatible with OpenAI's sdk and logs both streaming and non-streaming requests and responses.
+
+<!-- To learn more about using OpenPipe, check out our [Documentation](https://docs.openpipe.ai/docs/api). -->
+
+## Installation
+
+```sh
+npm install --save openpipe
+# or
+yarn add openpipe
+```
+
+## Usage
+
+1. Create a project at https://app.openpipe.ai
+2. Find your project's API key at https://app.openpipe.ai/project/settings
+3. Configure the OpenPipe client as shown below.
+
+```js
+// import OpenAI from 'openai'
+import OpenAI from "openpipe/openai";
+
+// Fully compatible with original OpenAI initialization
+const openai = new OpenAI({
+  apiKey: "my api key", // defaults to process.env["OPENAI_API_KEY"]
+  // openpipe key is optional
+  openpipe: {
+    apiKey: "my api key", // defaults to process.env["OPENPIPE_API_KEY"]
+    baseUrl: "my url", // defaults to process.env["OPENPIPE_BASE_URL"] or https://app.openpipe.ai/api/v1 if not set
+  },
+});
+
+async function main() {
+  // Allows optional openpipe object
+  const completion = await openai.chat.completions.create({
+    messages: [{ role: "user", content: "Say this is a test" }],
+    model: "gpt-3.5-turbo",
+    // optional
+    openpipe: {
+      // Add custom searchable tags
+      tags: {
+        prompt_id: "getCompletion",
+        any_key: "any_value",
+      },
+    },
+  });
+
+  console.log(completion.choices);
+}
+
+main();
+```
+
+## FAQ
+
+<b><i>How do I report calls to my self-hosted instance?</i></b>
+
+Start an instance by following the instructions on [Running Locally](https://github.com/OpenPipe/OpenPipe#running-locally). Once it's running, point your `OPENPIPE_BASE_URL` to your self-hosted instance.
+
+<b><i>What if my `OPENPIPE_BASE_URL` is misconfigured or my instance goes down? Will my OpenAI calls stop working?</i></b>
+
+Your OpenAI calls will continue to function as expected no matter what. The sdk handles logging errors gracefully without affecting OpenAI inference.
+
+See the [GitHub repo](https://github.com/OpenPipe/OpenPipe) for more details.

client-libs/typescript/build.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+rm -rf dist
+
+npx tsup
+
+# copy the package.json file to /dist
+cp package.json dist
+
+# copy the README.md file to /dist
+cp README.md dist
+
+python3 -c "
+import json
+
+# Load the package.json file
+with open('dist/package.json', 'r') as file:
+    data = json.load(file)
+
+# Change the names
+data['name'] = 'openpipe'
+
+# Write the changes back to the package.json file
+with open('dist/package.json', 'w') as file:
+    json.dump(data, file, indent=2)
+"