alihan/OpenPipe-llm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #123 from OpenPipe/add-openapi

Browse Source 
Add Logged Calls and projects
This commit is contained in:
Kyle Corbitt
2023-08-09 14:37:17 -07:00
committed by GitHub
parent deabbb094b f6f04e537e
commit 868d7084f0
84 changed files with 6070 additions and 879 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/.env.example
View File

@@ -31,3 +31,6 @@ NEXT_PUBLIC_HOST="http://localhost:3000"
# Next Auth Github Provider
GITHUB_CLIENT_ID="your_client_id"
GITHUB_CLIENT_SECRET="your_secret"
OPENPIPE_BASE_URL="http://localhost:3000/api"
OPENPIPE_API_KEY="your_key"

4
app/@types/nextjs-routes.d.ts vendored
View File

@@ -12,8 +12,10 @@ declare module "nextjs-routes" {
export type Route =
| StaticRoute<"/account/signin">
| DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
| DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
| StaticRoute<"/api/experiments/og-image">
| StaticRoute<"/api/openapi">
| StaticRoute<"/api/sentry-example-api">
| DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
| DynamicRoute<"/data/[id]", { "id": string }>
@@ -21,6 +23,8 @@ declare module "nextjs-routes" {
| DynamicRoute<"/experiments/[id]", { "id": string }>
| StaticRoute<"/experiments">
| StaticRoute<"/">
| StaticRoute<"/logged-calls">
| StaticRoute<"/project/settings">
| StaticRoute<"/sentry-example-page">
| StaticRoute<"/world-champs">
| StaticRoute<"/world-champs/signup">;

1
app/Dockerfile
View File

@@ -23,6 +23,7 @@ ARG NEXT_PUBLIC_SOCKET_URL
ARG NEXT_PUBLIC_HOST
ARG NEXT_PUBLIC_SENTRY_DSN
ARG SENTRY_AUTH_TOKEN
ARG NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS
WORKDIR /app
COPY --from=deps /app/node_modules ./node_modules

7
app/openapitools.json Normal file
View File

@@ -0,0 +1,7 @@
{
"$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json",
"spaces": 2,
"generator-cli": {
"version": "6.6.0"
}
}

10
app/package.json
View File

@@ -16,7 +16,7 @@
"postinstall": "prisma generate",
"lint": "next lint",
"start": "next start",
"codegen": "tsx src/codegen/export-openai-types.ts",
"codegen": "tsx src/server/scripts/client-codegen.ts",
"seed": "tsx prisma/seed.ts",
"check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
"test": "pnpm vitest --no-threads"
@@ -50,6 +50,7 @@
"chroma-js": "^2.4.2",
"concurrently": "^8.2.0",
"cors": "^2.8.5",
"crypto-random-string": "^5.0.0",
"dayjs": "^1.11.8",
"dedent": "^1.0.1",
"dotenv": "^16.3.1",
@@ -62,12 +63,16 @@
"json-schema-to-typescript": "^13.0.2",
"json-stringify-pretty-compact": "^4.0.0",
"jsonschema": "^1.4.1",
"kysely": "^0.26.1",
"lodash-es": "^4.17.21",
"lucide-react": "^0.265.0",
"next": "^13.4.2",
"next-auth": "^4.22.1",
"next-query-params": "^4.2.3",
"nextjs-cors": "^2.1.2",
"nextjs-routes": "^2.0.1",
"openai": "4.0.0-beta.7",
"pg": "^8.11.2",
"pluralize": "^8.0.0",
"posthog-js": "^1.75.3",
"posthog-node": "^3.1.1",
@@ -83,10 +88,12 @@
"react-syntax-highlighter": "^15.5.0",
"react-textarea-autosize": "^8.5.0",
"recast": "^0.23.3",
"recharts": "^2.7.2",
"replicate": "^0.12.3",
"socket.io": "^4.7.1",
"socket.io-client": "^4.7.1",
"superjson": "1.12.2",
"trpc-openapi": "^1.2.0",
"tsx": "^3.12.7",
"type-fest": "^4.0.0",
"use-query-params": "^2.2.1",
@@ -106,6 +113,7 @@
"@types/json-schema": "^7.0.12",
"@types/lodash-es": "^4.17.8",
"@types/node": "^18.16.0",
"@types/pg": "^8.10.2",
"@types/pluralize": "^0.0.30",
"@types/prismjs": "^1.26.0",
"@types/react": "^18.2.6",

982
app/pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

90
app/prisma/migrations/20230806024615_add_logged_calls_and_api_keys/migration.sql Normal file
View File

@@ -0,0 +1,90 @@
-- CreateTable
CREATE TABLE "LoggedCall" (
"id" UUID NOT NULL,
"startTime" TIMESTAMP(3) NOT NULL,
"cacheHit" BOOLEAN NOT NULL,
"modelResponseId" UUID NOT NULL,
"organizationId" UUID NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "LoggedCall_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "LoggedCallModelResponse" (
"id" UUID NOT NULL,
"reqPayload" JSONB NOT NULL,
"respStatus" INTEGER,
"respPayload" JSONB,
"error" TEXT,
"startTime" TIMESTAMP(3) NOT NULL,
"endTime" TIMESTAMP(3) NOT NULL,
"cacheKey" TEXT,
"durationMs" INTEGER,
"inputTokens" INTEGER,
"outputTokens" INTEGER,
"finishReason" TEXT,
"completionId" TEXT,
"totalCost" DECIMAL(18,12),
"originalLoggedCallId" UUID NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "LoggedCallModelResponse_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "LoggedCallTag" (
"id" UUID NOT NULL,
"name" TEXT NOT NULL,
"value" TEXT,
"loggedCallId" UUID NOT NULL,
CONSTRAINT "LoggedCallTag_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "ApiKey" (
"id" UUID NOT NULL,
"name" TEXT NOT NULL,
"apiKey" TEXT NOT NULL,
"organizationId" UUID NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "ApiKey_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "LoggedCall_startTime_idx" ON "LoggedCall"("startTime");
-- CreateIndex
CREATE UNIQUE INDEX "LoggedCallModelResponse_originalLoggedCallId_key" ON "LoggedCallModelResponse"("originalLoggedCallId");
-- CreateIndex
CREATE INDEX "LoggedCallModelResponse_cacheKey_idx" ON "LoggedCallModelResponse"("cacheKey");
-- CreateIndex
CREATE INDEX "LoggedCallTag_name_idx" ON "LoggedCallTag"("name");
-- CreateIndex
CREATE INDEX "LoggedCallTag_name_value_idx" ON "LoggedCallTag"("name", "value");
-- CreateIndex
CREATE UNIQUE INDEX "ApiKey_apiKey_key" ON "ApiKey"("apiKey");
-- AddForeignKey
ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_modelResponseId_fkey" FOREIGN KEY ("modelResponseId") REFERENCES "LoggedCallModelResponse"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "LoggedCallModelResponse" ADD CONSTRAINT "LoggedCallModelResponse_originalLoggedCallId_fkey" FOREIGN KEY ("originalLoggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "LoggedCallTag" ADD CONSTRAINT "LoggedCallTag_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;

2
app/prisma/migrations/20230807044936_add_name_to_organization/migration.sql Normal file
View File

@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "Organization" ADD COLUMN "name" TEXT NOT NULL DEFAULT 'Project 1';

2
app/prisma/migrations/20230808034313_make_model_response_optional/migration.sql Normal file
View File

@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LoggedCall" ALTER COLUMN "modelResponseId" DROP NOT NULL;

100
app/prisma/schema.prisma
View File

@@ -200,16 +200,21 @@ model DatasetEntry {
updatedAt DateTime @updatedAt
}
// TODO rename Organization to Project
model Organization {
id String @id @default(uuid()) @db.Uuid
name String @default("Project 1")
personalOrgUserId String? @unique @db.Uuid
PersonalOrgUser User? @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
personalOrgUser User? @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
organizationUsers OrganizationUser[]
experiments Experiment[]
datasets Dataset[]
loggedCalls LoggedCall[]
apiKeys ApiKey[]
}
enum OrganizationUserRole {
@@ -249,6 +254,99 @@ model WorldChampEntrant {
@@unique([userId])
}
model LoggedCall {
id String @id @default(uuid()) @db.Uuid
startTime DateTime
// True if this call was served from the cache, false otherwise
cacheHit Boolean
// A LoggedCall is always associated with a LoggedCallModelResponse. If this
// is a cache miss, we create a new LoggedCallModelResponse.
// If it's a cache hit, it's a pre-existing LoggedCallModelResponse.
modelResponseId String? @db.Uuid
modelResponse LoggedCallModelResponse? @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
// The responses created by this LoggedCall. Will be empty if this LoggedCall was a cache hit.
createdResponses LoggedCallModelResponse[] @relation(name: "ModelResponseOriginalCall")
organizationId String @db.Uuid
organization Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
tags LoggedCallTag[]
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@index([startTime])
}
model LoggedCallModelResponse {
id String @id @default(uuid()) @db.Uuid
reqPayload Json
// The HTTP status returned by the model provider
respStatus Int?
respPayload Json?
// Should be null if the request was successful, and some string if the request failed.
error String?
startTime DateTime
endTime DateTime
// Note: the function to calculate the cacheKey should include the project
// ID so we don't share cached responses between projects, which could be an
// attack vector. Also, we should only set the cacheKey on the model if the
// request was successful.
cacheKey String?
// Derived fields
durationMs Int?
inputTokens Int?
outputTokens Int?
finishReason String?
completionId String?
totalCost Decimal? @db.Decimal(18, 12)
// The LoggedCall that created this LoggedCallModelResponse
originalLoggedCallId String @unique @db.Uuid
originalLoggedCall LoggedCall @relation(name: "ModelResponseOriginalCall", fields: [originalLoggedCallId], references: [id], onDelete: Cascade)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
loggedCalls LoggedCall[]
@@index([cacheKey])
}
model LoggedCallTag {
id String @id @default(uuid()) @db.Uuid
name String
value String?
loggedCallId String @db.Uuid
loggedCall LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
@@index([name])
@@index([name, value])
}
model ApiKey {
id String @id @default(uuid()) @db.Uuid
name String
apiKey String @unique
organizationId String @db.Uuid
organization Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}
model Account {
id String @id @default(uuid()) @db.Uuid
userId String @db.Uuid

410
app/prisma/seedDashboard.ts Normal file
View File

File diff suppressed because one or more lines are too long

40
app/src/components/CopiableCode.tsx Normal file
View File

@@ -0,0 +1,40 @@
import { HStack, Icon, IconButton, Tooltip, Text } from "@chakra-ui/react";
import { useState } from "react";
import { MdContentCopy } from "react-icons/md";
import { useHandledAsyncCallback } from "~/utils/hooks";
const CopiableCode = ({ code }: { code: string }) => {
const [copied, setCopied] = useState(false);
const [copyToClipboard] = useHandledAsyncCallback(async () => {
await navigator.clipboard.writeText(code);
setCopied(true);
}, [code]);
return (
<HStack
backgroundColor="blackAlpha.800"
color="white"
borderRadius={4}
padding={3}
w="full"
justifyContent="space-between"
>
<Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5}>
{code}
</Text>
<Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
<IconButton
aria-label="Copy"
icon={<Icon as={MdContentCopy} boxSize={5} />}
size="xs"
colorScheme="white"
variant="ghost"
onClick={copyToClipboard}
onMouseLeave={() => setCopied(false)}
/>
</Tooltip>
</HStack>
);
};
export default CopiableCode;

5
app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx
View File

@@ -14,6 +14,7 @@ import {
import { useRouter } from "next/router";
import { useRef } from "react";
import { BsTrash } from "react-icons/bs";
import { useAppStore } from "~/state/store";
import { api } from "~/utils/api";
import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
@@ -23,6 +24,8 @@ export const DeleteButton = () => {
const utils = api.useContext();
const router = useRouter();
const closeDrawer = useAppStore((s) => s.closeDrawer);
const { isOpen, onOpen, onClose } = useDisclosure();
const cancelRef = useRef<HTMLButtonElement>(null);
@@ -31,6 +34,8 @@ export const DeleteButton = () => {
await mutation.mutateAsync({ id: experiment.data.id });
await utils.experiments.list.invalidate();
await router.push({ pathname: "/experiments" });
closeDrawer();
onClose();
}, [mutation, experiment.data?.id, router]);

26
app/src/components/StatsCard.tsx Normal file
View File

@@ -0,0 +1,26 @@
import { VStack, HStack, type StackProps, Text, Divider } from "@chakra-ui/react";
import Link, { type LinkProps } from "next/link";
const StatsCard = ({
title,
href,
children,
...rest
}: { title: string; href: string } & StackProps & LinkProps) => {
return (
<VStack flex={1} borderWidth={1} padding={4} borderRadius={4} borderColor="gray.300" {...rest}>
<HStack w="full" justifyContent="space-between">
<Text fontSize="md" fontWeight="bold">
{title}
</Text>
<Link href={href}>
<Text color="blue">View all</Text>
</Link>
</HStack>
<Divider />
{children}
</VStack>
);
};
export default StatsCard;

201
app/src/components/dashboard/LoggedCallTable.tsx Normal file
View File

@@ -0,0 +1,201 @@
import {
Box,
Card,
CardHeader,
Heading,
Table,
Tbody,
Td,
Th,
Thead,
Tr,
Tooltip,
Collapse,
HStack,
VStack,
IconButton,
useToast,
Icon,
Button,
ButtonGroup,
} from "@chakra-ui/react";
import dayjs from "dayjs";
import relativeTime from "dayjs/plugin/relativeTime";
import { ChevronUpIcon, ChevronDownIcon, CopyIcon } from "lucide-react";
import { useMemo, useState } from "react";
import { type RouterOutputs, api } from "~/utils/api";
import SyntaxHighlighter from "react-syntax-highlighter";
import { atelierCaveLight } from "react-syntax-highlighter/dist/cjs/styles/hljs";
import stringify from "json-stringify-pretty-compact";
import Link from "next/link";
dayjs.extend(relativeTime);
type LoggedCall = RouterOutputs["dashboard"]["loggedCalls"][0];
const FormattedJson = ({ json }: { json: any }) => {
const jsonString = stringify(json, { maxLength: 40 });
const toast = useToast();
const copyToClipboard = async (text: string) => {
try {
await navigator.clipboard.writeText(text);
toast({
title: "Copied to clipboard",
status: "success",
duration: 2000,
});
} catch (err) {
toast({
title: "Failed to copy to clipboard",
status: "error",
duration: 2000,
});
}
};
return (
<Box position="relative" fontSize="sm" borderRadius="md" overflow="hidden">
<SyntaxHighlighter
customStyle={{ overflowX: "unset" }}
language="json"
style={atelierCaveLight}
lineProps={{
style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
}}
wrapLines
>
{jsonString}
</SyntaxHighlighter>
<IconButton
aria-label="Copy"
icon={<CopyIcon />}
position="absolute"
top={1}
right={1}
size="xs"
variant="ghost"
onClick={() => void copyToClipboard(jsonString)}
/>
</Box>
);
};
function TableRow({
loggedCall,
isExpanded,
onToggle,
}: {
loggedCall: LoggedCall;
isExpanded: boolean;
onToggle: () => void;
}) {
const isError = loggedCall.modelResponse?.respStatus !== 200;
const timeAgo = dayjs(loggedCall.startTime).fromNow();
const fullTime = dayjs(loggedCall.startTime).toString();
const model = useMemo(
() => loggedCall.tags.find((tag) => tag.name.startsWith("$model"))?.value,
[loggedCall.tags],
);
return (
<>
<Tr
onClick={onToggle}
key={loggedCall.id}
_hover={{ bgColor: "gray.100", cursor: "pointer" }}
sx={{
"> td": { borderBottom: "none" },
}}
>
<Td>
<Icon boxSize={6} as={isExpanded ? ChevronUpIcon : ChevronDownIcon} />
</Td>
<Td>
<Tooltip label={fullTime} placement="top">
<Box whiteSpace="nowrap" minW="120px">
{timeAgo}
</Box>
</Tooltip>
</Td>
<Td width="100%">{model}</Td>
<Td isNumeric>{((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2)}s</Td>
<Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
<Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
<Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
{loggedCall.modelResponse?.respStatus ?? "No response"}
</Td>
</Tr>
<Tr>
<Td colSpan={8} p={0}>
<Collapse in={isExpanded} unmountOnExit={true}>
<VStack p={4} align="stretch">
<HStack align="stretch">
<VStack flex={1} align="stretch">
<Heading size="sm">Input</Heading>
<FormattedJson json={loggedCall.modelResponse?.reqPayload} />
</VStack>
<VStack flex={1} align="stretch">
<Heading size="sm">Output</Heading>
<FormattedJson json={loggedCall.modelResponse?.respPayload} />
</VStack>
</HStack>
<ButtonGroup alignSelf="flex-end">
<Button as={Link} colorScheme="blue" href={{ pathname: "/experiments" }}>
Experiments
</Button>
</ButtonGroup>
</VStack>
</Collapse>
</Td>
</Tr>
</>
);
}
export default function LoggedCallTable() {
const [expandedRow, setExpandedRow] = useState<string | null>(null);
const loggedCalls = api.dashboard.loggedCalls.useQuery({});
return (
<Card variant="outline" width="100%" overflow="hidden">
<CardHeader>
<Heading as="h3" size="sm">
Logged Calls
</Heading>
</CardHeader>
<Table>
<Thead>
<Tr>
<Th />
<Th>Time</Th>
<Th>Model</Th>
<Th isNumeric>Duration</Th>
<Th isNumeric>Input tokens</Th>
<Th isNumeric>Output tokens</Th>
<Th isNumeric>Status</Th>
</Tr>
</Thead>
<Tbody>
{loggedCalls.data?.map((loggedCall) => {
return (
<TableRow
key={loggedCall.id}
loggedCall={loggedCall}
isExpanded={loggedCall.id === expandedRow}
onToggle={() => {
if (loggedCall.id === expandedRow) {
setExpandedRow(null);
} else {
setExpandedRow(loggedCall.id);
}
}}
/>
);
})}
</Tbody>
</Table>
</Card>
);
}

6
app/src/components/datasets/DatasetCard.tsx
View File

@@ -15,6 +15,7 @@ import { useRouter } from "next/router";
import { BsPlusSquare } from "react-icons/bs";
import { api } from "~/utils/api";
import { useHandledAsyncCallback } from "~/utils/hooks";
import { useAppStore } from "~/state/store";
type DatasetData = {
name: string;
@@ -71,11 +72,12 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
export const NewDatasetCard = () => {
const router = useRouter();
const selectedOrgId = useAppStore((s) => s.selectedOrgId);
const createMutation = api.datasets.create.useMutation();
const [createDataset, isLoading] = useHandledAsyncCallback(async () => {
const newDataset = await createMutation.mutateAsync({ label: "New Dataset" });
const newDataset = await createMutation.mutateAsync({ organizationId: selectedOrgId ?? "" });
await router.push({ pathname: "/data/[id]", query: { id: newDataset.id } });
}, [createMutation, router]);
}, [createMutation, router, selectedOrgId]);
return (
<AspectRatio ratio={1.2} w="full">

13
app/src/components/experiments/ExperimentCard.tsx
View File

@@ -15,6 +15,7 @@ import { useRouter } from "next/router";
import { BsPlusSquare } from "react-icons/bs";
import { api } from "~/utils/api";
import { useHandledAsyncCallback } from "~/utils/hooks";
import { useAppStore } from "~/state/store";
type ExperimentData = {
testScenarioCount: number;
@@ -75,11 +76,17 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
export const NewExperimentCard = () => {
const router = useRouter();
const selectedOrgId = useAppStore((s) => s.selectedOrgId);
const createMutation = api.experiments.create.useMutation();
const [createExperiment, isLoading] = useHandledAsyncCallback(async () => {
const newExperiment = await createMutation.mutateAsync({ label: "New Experiment" });
await router.push({ pathname: "/experiments/[id]", query: { id: newExperiment.id } });
}, [createMutation, router]);
const newExperiment = await createMutation.mutateAsync({
organizationId: selectedOrgId ?? "",
});
await router.push({
pathname: "/experiments/[id]",
query: { id: newExperiment.id },
});
}, [createMutation, router, selectedOrgId]);
return (
<AspectRatio ratio={1.2} w="full">

9
app/src/components/experiments/ExperimentHeaderButtons/useOnForkButtonPressed.tsx
View File

@@ -3,18 +3,23 @@ import { api } from "~/utils/api";
import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
import { signIn, useSession } from "next-auth/react";
import { useRouter } from "next/router";
import { useAppStore } from "~/state/store";
export const useOnForkButtonPressed = () => {
const router = useRouter();
const user = useSession().data;
const experiment = useExperiment();
const selectedOrgId = useAppStore((state) => state.selectedOrgId);
const forkMutation = api.experiments.fork.useMutation();
const [onFork, isForking] = useHandledAsyncCallback(async () => {
if (!experiment.data?.id) return;
const forkedExperimentId = await forkMutation.mutateAsync({ id: experiment.data.id });
if (!experiment.data?.id || !selectedOrgId) return;
const forkedExperimentId = await forkMutation.mutateAsync({
id: experiment.data.id,
organizationId: selectedOrgId,
});
await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
}, [forkMutation, experiment.data?.id, router]);

132
app/src/components/nav/AppShell.tsx
View File

@@ -7,48 +7,22 @@ import {
Image,
Text,
Box,
type BoxProps,
Link as ChakraLink,
Flex,
} from "@chakra-ui/react";
import Head from "next/head";
import Link, { type LinkProps } from "next/link";
import { BsGithub, BsPersonCircle } from "react-icons/bs";
import { useRouter } from "next/router";
import { type IconType } from "react-icons";
import Link from "next/link";
import { BsGearFill, BsGithub, BsPersonCircle } from "react-icons/bs";
import { IoStatsChartOutline } from "react-icons/io5";
import { RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
import { signIn, useSession } from "next-auth/react";
import UserMenu from "./UserMenu";
import { env } from "~/env.mjs";
import ProjectMenu from "./ProjectMenu";
import NavSidebarOption from "./NavSidebarOption";
import IconLink from "./IconLink";
type IconLinkProps = BoxProps & LinkProps & { label?: string; icon: IconType; href: string };
const IconLink = ({ icon, label, href, color, ...props }: IconLinkProps) => {
const router = useRouter();
const isActive = href && router.pathname.startsWith(href);
return (
<Link href={href} style={{ width: "100%" }}>
<HStack
w="full"
p={4}
color={color}
as={ChakraLink}
bgColor={isActive ? "gray.200" : "transparent"}
_hover={{ bgColor: "gray.300", textDecoration: "none" }}
justifyContent="start"
cursor="pointer"
{...props}
>
<Icon as={icon} boxSize={6} mr={2} />
<Text fontWeight="bold" fontSize="sm">
{label}
</Text>
</HStack>
</Link>
);
};
const Divider = () => <Box h="1px" bgColor="gray.200" />;
const Divider = () => <Box h="1px" bgColor="gray.300" w="full" />;
const NavSidebar = () => {
const user = useSession().data;
@@ -56,22 +30,31 @@ const NavSidebar = () => {
return (
<VStack
align="stretch"
bgColor="gray.100"
bgColor="gray.50"
py={2}
px={2}
pb={0}
height="100%"
w={{ base: "56px", md: "200px" }}
w={{ base: "56px", md: "240px" }}
overflow="hidden"
borderRightWidth={1}
borderColor="gray.300"
>
<HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={4} py={2}>
<HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={2} py={2}>
<Image src="/logo.svg" alt="" boxSize={6} mr={4} />
<Heading size="md" fontFamily="inconsolata, monospace">
OpenPipe
</Heading>
</HStack>
<VStack spacing={0} align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
<Divider />
<VStack align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
{user != null && (
<>
<ProjectMenu />
<Divider />
{env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS && (
<IconLink icon={IoStatsChartOutline} label="Logged Calls" href="/logged-calls" beta />
)}
<IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
{env.NEXT_PUBLIC_SHOW_DATA && (
<IconLink icon={RiDatabase2Line} label="Data" href="/data" />
@@ -79,29 +62,39 @@ const NavSidebar = () => {
</>
)}
{user === null && (
<HStack
w="full"
p={4}
as={ChakraLink}
_hover={{ bgColor: "gray.300", textDecoration: "none" }}
justifyContent="start"
cursor="pointer"
onClick={() => {
signIn("github").catch(console.error);
}}
>
<Icon as={BsPersonCircle} boxSize={6} mr={2} />
<Text fontWeight="bold" fontSize="sm">
Sign In
</Text>
</HStack>
<NavSidebarOption>
<HStack
w="full"
p={4}
as={ChakraLink}
justifyContent="start"
onClick={() => {
signIn("github").catch(console.error);
}}
>
<Icon as={BsPersonCircle} boxSize={6} mr={2} />
<Text fontWeight="bold" fontSize="sm">
Sign In
</Text>
</HStack>
</NavSidebarOption>
)}
</VStack>
{user ? (
<UserMenu user={user} borderColor={"gray.200"} borderTopWidth={1} borderBottomWidth={1} />
) : (
<Divider />
)}
<VStack w="full" alignItems="flex-start" spacing={0}>
<Text
pl={2}
pb={2}
fontSize="xs"
fontWeight="bold"
color="gray.500"
display={{ base: "none", md: "flex" }}
>
CONFIGURATION
</Text>
<IconLink icon={BsGearFill} label="Project Settings" href="/project/settings" />
</VStack>
{user && <UserMenu user={user} borderColor={"gray.200"} />}
<Divider />
<VStack spacing={0} align="center">
<ChakraLink
href="https://github.com/openpipe/openpipe"
@@ -117,7 +110,15 @@ const NavSidebar = () => {
);
};
export default function AppShell(props: { children: React.ReactNode; title?: string }) {
export default function AppShell({
children,
title,
requireAuth,
}: {
children: React.ReactNode;
title?: string;
requireAuth?: boolean;
}) {
const [vh, setVh] = useState("100vh"); // Default height to prevent flicker on initial render
useEffect(() => {
@@ -137,14 +138,23 @@ export default function AppShell(props: { children: React.ReactNode; title?: str
};
}, []);
const user = useSession().data;
const authLoading = useSession().status === "loading";
useEffect(() => {
if (requireAuth && user === null && !authLoading) {
signIn("github").catch(console.error);
}
}, [requireAuth, user, authLoading]);
return (
<Flex h={vh} w="100vw">
<Head>
<title>{props.title ? `${props.title} | OpenPipe` : "OpenPipe"}</title>
<title>{title ? `${title} | OpenPipe` : "OpenPipe"}</title>
</Head>
<NavSidebar />
<Box h="100%" flex={1} overflowY="auto">
{props.children}
{children}
</Box>
</Flex>
);

31
app/src/components/nav/IconLink.tsx Normal file
View File

@@ -0,0 +1,31 @@
import { Icon, HStack, Text, type BoxProps } from "@chakra-ui/react";
import Link, { type LinkProps } from "next/link";
import { type IconType } from "react-icons";
import NavSidebarOption from "./NavSidebarOption";
type IconLinkProps = BoxProps &
LinkProps & { label?: string; icon: IconType; href: string; beta?: boolean };
const IconLink = ({ icon, label, href, color, beta, ...props }: IconLinkProps) => {
return (
<Link href={href} style={{ width: "100%" }}>
<NavSidebarOption activeHrefPattern={href}>
<HStack w="full" justifyContent="space-between" p={2} color={color} {...props}>
<HStack w="full" justifyContent="start">
<Icon as={icon} boxSize={6} mr={2} />
<Text fontSize="sm" display={{ base: "none", md: "block" }}>
{label}
</Text>
</HStack>
{beta && (
<Text fontSize="xs" ml={2} fontWeight="bold" color="orange.400">
BETA
</Text>
)}
</HStack>
</NavSidebarOption>
</Link>
);
};
export default IconLink;

27
app/src/components/nav/NavSidebarOption.tsx Normal file
View File

@@ -0,0 +1,27 @@
import { Box, type BoxProps } from "@chakra-ui/react";
import { useRouter } from "next/router";
const NavSidebarOption = ({
activeHrefPattern,
disableHoverEffect,
...props
}: { activeHrefPattern?: string; disableHoverEffect?: boolean } & BoxProps) => {
const router = useRouter();
const isActive = activeHrefPattern && router.pathname.startsWith(activeHrefPattern);
return (
<Box
w="full"
fontWeight={isActive ? "bold" : "500"}
bgColor={isActive ? "gray.200" : "transparent"}
_hover={disableHoverEffect ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
justifyContent="start"
cursor="pointer"
borderRadius={4}
{...props}
>
{props.children}
</Box>
);
};
export default NavSidebarOption;

19
app/src/components/nav/PageHeaderContainer.tsx Normal file
View File

@@ -0,0 +1,19 @@
import { Flex, type FlexProps } from "@chakra-ui/react";
const PageHeaderContainer = (props: FlexProps) => {
return (
<Flex
px={8}
py={2}
minH={16}
w="full"
direction={{ base: "column", sm: "row" }}
alignItems={{ base: "flex-start", sm: "center" }}
justifyContent="space-between"
fontWeight="500"
{...props}
/>
);
};
export default PageHeaderContainer;

28
app/src/components/nav/ProjectBreadcrumbContents.tsx Normal file
View File

@@ -0,0 +1,28 @@
import { HStack, Flex, Text } from "@chakra-ui/react";
import { useSelectedOrg } from "~/utils/hooks";
// Have to export only contents here instead of full BreadcrumbItem because Chakra doesn't
// recognize a BreadcrumbItem exported with this component as a valid child of Breadcrumb.
export default function ProjectBreadcrumbContents({ orgName = "" }: { orgName?: string }) {
const { data: selectedOrg } = useSelectedOrg();
orgName = orgName || selectedOrg?.name || "";
return (
<HStack w="full">
<Flex
p={1}
borderRadius={4}
backgroundColor="orange.100"
boxSize={6}
alignItems="center"
justifyContent="center"
>
<Text>{orgName[0]?.toUpperCase()}</Text>
</Flex>
<Text display={{ base: "none", md: "block" }} py={1}>
{orgName}
</Text>
</HStack>
);
}

178
app/src/components/nav/ProjectMenu.tsx Normal file
View File

@@ -0,0 +1,178 @@
import {
HStack,
VStack,
Text,
Popover,
PopoverTrigger,
PopoverContent,
Flex,
IconButton,
Icon,
Divider,
Button,
useDisclosure,
Spinner,
} from "@chakra-ui/react";
import React, { useEffect, useState } from "react";
import Link from "next/link";
import { AiFillCaretDown } from "react-icons/ai";
import { BsGear, BsPlus } from "react-icons/bs";
import { type Organization } from "@prisma/client";
import { useAppStore } from "~/state/store";
import { api } from "~/utils/api";
import NavSidebarOption from "./NavSidebarOption";
import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
import { useRouter } from "next/router";
export default function ProjectMenu() {
const router = useRouter();
const isActive = router.pathname.startsWith("/home");
const utils = api.useContext();
const selectedOrgId = useAppStore((s) => s.selectedOrgId);
const setSelectedOrgId = useAppStore((s) => s.setSelectedOrgId);
const { data: orgs } = api.organizations.list.useQuery();
useEffect(() => {
if (orgs && orgs[0] && (!selectedOrgId || !orgs.find((org) => org.id === selectedOrgId))) {
setSelectedOrgId(orgs[0].id);
}
}, [selectedOrgId, setSelectedOrgId, orgs]);
const { data: selectedOrg } = useSelectedOrg();
const popover = useDisclosure();
const createMutation = api.organizations.create.useMutation();
const [createProject, isLoading] = useHandledAsyncCallback(async () => {
const newOrg = await createMutation.mutateAsync({ name: "New Project" });
await utils.organizations.list.invalidate();
setSelectedOrgId(newOrg.id);
await router.push({ pathname: "/project/settings" });
}, [createMutation, router]);
return (
<VStack w="full" alignItems="flex-start" spacing={0}>
<Text
pl={2}
pb={2}
fontSize="xs"
fontWeight="bold"
color="gray.500"
display={{ base: "none", md: "flex" }}
>
PROJECT
</Text>
<NavSidebarOption>
<Popover
placement="bottom-start"
isOpen={popover.isOpen}
onClose={popover.onClose}
closeOnBlur
>
<PopoverTrigger>
<HStack w="full" justifyContent="space-between" onClick={popover.onToggle}>
<Flex
p={1}
borderRadius={4}
backgroundColor="orange.100"
minW={{ base: 10, md: 8 }}
minH={{ base: 10, md: 8 }}
m={{ base: 0, md: 1 }}
alignItems="center"
justifyContent="center"
// onClick={sidebarExpanded ? undefined : openMenu}
>
<Text>{selectedOrg?.name[0]?.toUpperCase()}</Text>
</Flex>
<Text fontSize="sm" display={{ base: "none", md: "block" }} py={1}>
{selectedOrg?.name}
</Text>
<Icon as={AiFillCaretDown} boxSize={3} size="xs" color="gray.500" mr={2} />
</HStack>
</PopoverTrigger>
<PopoverContent
_focusVisible={{ boxShadow: "unset" }}
minW={0}
borderColor="blue.400"
w="full"
>
<VStack alignItems="flex-start" spacing={2} py={4} px={2}>
<Text color="gray.500" fontSize="xs" fontWeight="bold" pb={1}>
PROJECTS
</Text>
<Divider />
<VStack spacing={0} w="full">
{orgs?.map((org) => (
<ProjectOption
key={org.id}
org={org}
isActive={org.id === selectedOrgId}
onClose={popover.onClose}
/>
))}
</VStack>
<HStack
as={Button}
variant="ghost"
colorScheme="blue"
color="blue.400"
pr={8}
w="full"
onClick={createProject}
>
<Icon as={isLoading ? Spinner : BsPlus} boxSize={6} />
<Text>New project</Text>
</HStack>
</VStack>
</PopoverContent>
</Popover>
</NavSidebarOption>
</VStack>
);
}
const ProjectOption = ({
org,
isActive,
onClose,
}: {
org: Organization;
isActive: boolean;
onClose: () => void;
}) => {
const setSelectedOrgId = useAppStore((s) => s.setSelectedOrgId);
const [gearHovered, setGearHovered] = useState(false);
return (
<HStack
as={Link}
href="/experiments"
onClick={() => {
setSelectedOrgId(org.id);
onClose();
}}
w="full"
justifyContent="space-between"
bgColor={isActive ? "gray.100" : "transparent"}
_hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
p={2}
>
<Text>{org.name}</Text>
<IconButton
as={Link}
href="/project/settings"
aria-label={`Open ${org.name} settings`}
icon={<Icon as={BsGear} boxSize={5} strokeWidth={0.5} color="gray.500" />}
variant="ghost"
size="xs"
p={0}
onMouseEnter={() => setGearHovered(true)}
onMouseLeave={() => setGearHovered(false)}
_hover={{ bgColor: isActive ? "gray.300" : "gray.100", transitionDelay: 0 }}
borderRadius={4}
/>
</HStack>
);
};

49
app/src/components/nav/UserMenu.tsx
View File

@@ -8,16 +8,15 @@ import {
PopoverTrigger,
PopoverContent,
Link,
useColorMode,
type StackProps,
Box,
} from "@chakra-ui/react";
import { type Session } from "next-auth";
import { signOut } from "next-auth/react";
import { BsBoxArrowRight, BsChevronRight, BsPersonCircle } from "react-icons/bs";
import NavSidebarOption from "./NavSidebarOption";
export default function UserMenu({ user, ...rest }: { user: Session } & StackProps) {
const { colorMode } = useColorMode();
const profileImage = user.user.image ? (
<Image src={user.user.image} alt="profile picture" boxSize={8} borderRadius="50%" />
) : (
@@ -28,28 +27,28 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
<>
<Popover placement="right">
<PopoverTrigger>
<HStack
// Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
px={3}
spacing={3}
py={2}
{...rest}
cursor="pointer"
_hover={{
bgColor: colorMode === "light" ? "gray.200" : "gray.700",
}}
>
{profileImage}
<VStack spacing={0} align="start" flex={1} flexShrink={1}>
<Text fontWeight="bold" fontSize="sm">
{user.user.name}
</Text>
<Text color="gray.500" fontSize="xs">
{user.user.email}
</Text>
</VStack>
<Icon as={BsChevronRight} boxSize={4} color="gray.500" />
</HStack>
<Box>
<NavSidebarOption>
<HStack
// Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
py={2}
px={1}
spacing={3}
{...rest}
>
{profileImage}
<VStack spacing={0} align="start" flex={1} flexShrink={1}>
<Text fontWeight="bold" fontSize="sm">
{user.user.name}
</Text>
<Text color="gray.500" fontSize="xs">
{/* {user.user.email} */}
</Text>
</VStack>
<Icon as={BsChevronRight} boxSize={4} color="gray.500" />
</HStack>
</NavSidebarOption>
</Box>
</PopoverTrigger>
<PopoverContent _focusVisible={{ boxShadow: "unset", outline: "unset" }} maxW="200px">
<VStack align="stretch" spacing={0}>

89
app/src/components/projectSettings/DeleteProjectDialog.tsx Normal file
View File

@@ -0,0 +1,89 @@
import {
Button,
AlertDialog,
AlertDialogBody,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogContent,
AlertDialogOverlay,
Input,
Text,
VStack,
Box,
Spinner,
} from "@chakra-ui/react";
import { useRouter } from "next/router";
import { useRef, useState } from "react";
import { api } from "~/utils/api";
import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
export const DeleteProjectDialog = ({
isOpen,
onClose,
}: {
isOpen: boolean;
onClose: () => void;
}) => {
const selectedOrg = useSelectedOrg();
const deleteMutation = api.organizations.delete.useMutation();
const utils = api.useContext();
const router = useRouter();
const cancelRef = useRef<HTMLButtonElement>(null);
const [onDeleteConfirm, isDeleting] = useHandledAsyncCallback(async () => {
if (!selectedOrg.data?.id) return;
await deleteMutation.mutateAsync({ id: selectedOrg.data.id });
await utils.organizations.list.invalidate();
await router.push({ pathname: "/experiments" });
onClose();
}, [deleteMutation, selectedOrg, router]);
const [nameToDelete, setNameToDelete] = useState("");
return (
<AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
<AlertDialogOverlay>
<AlertDialogContent>
<AlertDialogHeader fontSize="lg" fontWeight="bold">
Delete Project
</AlertDialogHeader>
<AlertDialogBody>
<VStack spacing={4} alignItems="flex-start">
<Text>
If you delete this project all the associated data and experiments will be deleted
as well. If you are sure that you want to delete this project, please type the name
of the project below.
</Text>
<Box bgColor="orange.100" w="full" p={2} borderRadius={4}>
<Text fontFamily="inconsolata">{selectedOrg.data?.name}</Text>
</Box>
<Input
placeholder={selectedOrg.data?.name}
value={nameToDelete}
onChange={(e) => setNameToDelete(e.target.value)}
/>
</VStack>
</AlertDialogBody>
<AlertDialogFooter>
<Button ref={cancelRef} onClick={onClose}>
Cancel
</Button>
<Button
colorScheme="red"
onClick={onDeleteConfirm}
ml={3}
isDisabled={nameToDelete !== selectedOrg.data?.name}
w={20}
>
{isDeleting ? <Spinner /> : "Delete"}
</Button>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialogOverlay>
</AlertDialog>
);
};

4
app/src/env.mjs
View File

@@ -20,6 +20,7 @@ export const env = createEnv({
REPLICATE_API_TOKEN: z.string().default("placeholder"),
ANTHROPIC_API_KEY: z.string().default("placeholder"),
SENTRY_AUTH_TOKEN: z.string().optional(),
OPENPIPE_API_KEY: z.string().optional(),
},
/**
@@ -33,6 +34,7 @@ export const env = createEnv({
NEXT_PUBLIC_HOST: z.string().url().default("http://localhost:3000"),
NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
NEXT_PUBLIC_SHOW_DATA: z.string().optional(),
NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: z.string().optional(),
},
/**
@@ -54,6 +56,8 @@ export const env = createEnv({
ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
NEXT_PUBLIC_SENTRY_DSN: process.env.NEXT_PUBLIC_SENTRY_DSN,
SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
OPENPIPE_API_KEY: process.env.OPENPIPE_API_KEY,
NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: process.env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS,
},
/**
* Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

22
app/src/pages/api/[...trpc].ts Normal file
View File

@@ -0,0 +1,22 @@
import { type NextApiRequest, type NextApiResponse } from "next";
import cors from "nextjs-cors";
import { createOpenApiNextHandler } from "trpc-openapi";
import { createProcedureCache } from "trpc-openapi/dist/adapters/node-http/procedures";
import { appRouter } from "~/server/api/root.router";
import { createTRPCContext } from "~/server/api/trpc";
const openApiHandler = createOpenApiNextHandler({
router: appRouter,
createContext: createTRPCContext,
});
const cache = createProcedureCache(appRouter);
const handler = async (req: NextApiRequest, res: NextApiResponse) => {
// Setup CORS
await cors(req, res);
return openApiHandler(req, res);
};
export default handler;

16
app/src/pages/api/openapi.json.ts Normal file
View File

@@ -0,0 +1,16 @@
import { type NextApiRequest, type NextApiResponse } from "next";
import { generateOpenApiDocument } from "trpc-openapi";
import { appRouter } from "~/server/api/root.router";
export const openApiDocument = generateOpenApiDocument(appRouter, {
title: "OpenPipe API",
description: "The public API for reporting API calls to OpenPipe",
version: "0.1.0",
baseUrl: "https://app.openpipe.ai/api",
});
// Respond with our OpenAPI schema
const hander = (req: NextApiRequest, res: NextApiResponse) => {
res.status(200).send(openApiDocument);
};
export default hander;

20
app/src/pages/data/[id].tsx
View File

@@ -18,6 +18,8 @@ import { api } from "~/utils/api";
import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
import DatasetEntriesTable from "~/components/datasets/DatasetEntriesTable";
import { DatasetHeaderButtons } from "~/components/datasets/DatasetHeaderButtons/DatasetHeaderButtons";
import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
export default function Dataset() {
const router = useRouter();
@@ -55,15 +57,11 @@ export default function Dataset() {
return (
<AppShell title={dataset.data?.name}>
<VStack h="full">
<Flex
pl={4}
pr={8}
py={2}
w="full"
direction={{ base: "column", sm: "row" }}
alignItems={{ base: "flex-start", sm: "center" }}
>
<Breadcrumb flex={1} mt={1}>
<PageHeaderContainer>
<Breadcrumb>
<BreadcrumbItem>
<ProjectBreadcrumbContents orgName={dataset.data?.organization?.name} />
</BreadcrumbItem>
<BreadcrumbItem>
<Link href="/data">
<Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
@@ -89,8 +87,8 @@ export default function Dataset() {
</BreadcrumbItem>
</Breadcrumb>
<DatasetHeaderButtons />
</Flex>
<Box w="full" overflowX="auto" flex={1} pl={4} pr={8} pt={8} pb={16}>
</PageHeaderContainer>
<Box w="full" overflowX="auto" flex={1} px={8} pt={8} pb={16}>
{datasetId && <DatasetEntriesTable />}
</Box>
</VStack>

104
app/src/pages/data/index.tsx
View File

@@ -1,83 +1,49 @@
import {
SimpleGrid,
Icon,
VStack,
Breadcrumb,
BreadcrumbItem,
Flex,
Center,
Text,
Link,
HStack,
} from "@chakra-ui/react";
import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
import AppShell from "~/components/nav/AppShell";
import { api } from "~/utils/api";
import { signIn, useSession } from "next-auth/react";
import { RiDatabase2Line } from "react-icons/ri";
import {
DatasetCard,
DatasetCardSkeleton,
NewDatasetCard,
} from "~/components/datasets/DatasetCard";
import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
import { useDatasets } from "~/utils/hooks";
export default function DatasetsPage() {
const datasets = api.datasets.list.useQuery();
const user = useSession().data;
const authLoading = useSession().status === "loading";
if (user === null || authLoading) {
return (
<AppShell title="Data">
<Center h="100%">
{!authLoading && (
<Text>
<Link
onClick={() => {
signIn("github").catch(console.error);
}}
textDecor="underline"
>
Sign in
</Link>{" "}
to view or create new datasets!
</Text>
)}
</Center>
</AppShell>
);
}
const datasets = useDatasets();
return (
<AppShell title="Data">
<VStack alignItems={"flex-start"} px={4} py={2}>
<HStack minH={8} align="center" pt={2}>
<Breadcrumb flex={1}>
<BreadcrumbItem>
<Flex alignItems="center">
<Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
</Flex>
</BreadcrumbItem>
</Breadcrumb>
</HStack>
<SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} p="4">
<NewDatasetCard />
{datasets.data && !datasets.isLoading ? (
datasets?.data?.map((dataset) => (
<DatasetCard
key={dataset.id}
dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
/>
))
) : (
<>
<DatasetCardSkeleton />
<DatasetCardSkeleton />
<DatasetCardSkeleton />
</>
)}
</SimpleGrid>
</VStack>
<AppShell title="Data" requireAuth>
<PageHeaderContainer>
<Breadcrumb>
<BreadcrumbItem>
<ProjectBreadcrumbContents />
</BreadcrumbItem>
<BreadcrumbItem minH={8}>
<Flex alignItems="center">
<Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
</Flex>
</BreadcrumbItem>
</Breadcrumb>
</PageHeaderContainer>
<SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py={4} px={8}>
<NewDatasetCard />
{datasets.data && !datasets.isLoading ? (
datasets?.data?.map((dataset) => (
<DatasetCard
key={dataset.id}
dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
/>
))
) : (
<>
<DatasetCardSkeleton />
<DatasetCardSkeleton />
<DatasetCardSkeleton />
</>
)}
</SimpleGrid>
</AppShell>
);
}

17
app/src/pages/experiments/[id].tsx
View File

@@ -23,6 +23,8 @@ import { useAppStore } from "~/state/store";
import { useSyncVariantEditor } from "~/state/sync";
import { ExperimentHeaderButtons } from "~/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons";
import Head from "next/head";
import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
// TODO: import less to fix deployment with server side props
// export const getServerSideProps = async (context: GetServerSidePropsContext<{ id: string }>) => {
@@ -104,14 +106,11 @@ export default function Experiment() {
)}
<AppShell title={experiment.data?.label}>
<VStack h="full">
<Flex
px={4}
py={2}
w="full"
direction={{ base: "column", sm: "row" }}
alignItems={{ base: "flex-start", sm: "center" }}
>
<Breadcrumb flex={1}>
<PageHeaderContainer>
<Breadcrumb>
<BreadcrumbItem>
<ProjectBreadcrumbContents orgName={experiment.data?.organization?.name} />
</BreadcrumbItem>
<BreadcrumbItem>
<Link href="/experiments">
<Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
@@ -143,7 +142,7 @@ export default function Experiment() {
</BreadcrumbItem>
</Breadcrumb>
<ExperimentHeaderButtons />
</Flex>
</PageHeaderContainer>
<ExperimentSettingsDrawer />
<Box w="100%" overflowX="auto" flex={1}>
<OutputsTable experimentId={router.query.id as string | undefined} />

94
app/src/pages/experiments/index.tsx
View File

@@ -1,78 +1,44 @@
import {
SimpleGrid,
Icon,
VStack,
Breadcrumb,
BreadcrumbItem,
Flex,
Center,
Text,
Link,
HStack,
} from "@chakra-ui/react";
import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
import { RiFlaskLine } from "react-icons/ri";
import AppShell from "~/components/nav/AppShell";
import { api } from "~/utils/api";
import {
ExperimentCard,
ExperimentCardSkeleton,
NewExperimentCard,
} from "~/components/experiments/ExperimentCard";
import { signIn, useSession } from "next-auth/react";
import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
import { useExperiments } from "~/utils/hooks";
export default function ExperimentsPage() {
const experiments = api.experiments.list.useQuery();
const user = useSession().data;
const authLoading = useSession().status === "loading";
if (user === null || authLoading) {
return (
<AppShell title="Experiments">
<Center h="100%">
{!authLoading && (
<Text>
<Link
onClick={() => {
signIn("github").catch(console.error);
}}
textDecor="underline"
>
Sign in
</Link>{" "}
to view or create new experiments!
</Text>
)}
</Center>
</AppShell>
);
}
const experiments = useExperiments();
return (
<AppShell title="Experiments">
<VStack alignItems={"flex-start"} px={4} py={2}>
<HStack minH={8} align="center" pt={2}>
<Breadcrumb flex={1}>
<BreadcrumbItem>
<Flex alignItems="center">
<Icon as={RiFlaskLine} boxSize={4} mr={2} /> Experiments
</Flex>
</BreadcrumbItem>
</Breadcrumb>
</HStack>
<SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} p="4">
<NewExperimentCard />
{experiments.data && !experiments.isLoading ? (
experiments?.data?.map((exp) => <ExperimentCard key={exp.id} exp={exp} />)
) : (
<>
<ExperimentCardSkeleton />
<ExperimentCardSkeleton />
<ExperimentCardSkeleton />
</>
)}
</SimpleGrid>
</VStack>
<AppShell title="Experiments" requireAuth>
<PageHeaderContainer>
<Breadcrumb>
<BreadcrumbItem>
<ProjectBreadcrumbContents />
</BreadcrumbItem>
<BreadcrumbItem minH={8}>
<Flex alignItems="center">
<Icon as={RiFlaskLine} boxSize={4} mr={2} /> Experiments
</Flex>
</BreadcrumbItem>
</Breadcrumb>
</PageHeaderContainer>
<SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py="4" px={8}>
<NewExperimentCard />
{experiments.data && !experiments.isLoading ? (
experiments?.data?.map((exp) => <ExperimentCard key={exp.id} exp={exp} />)
) : (
<>
<ExperimentCardSkeleton />
<ExperimentCardSkeleton />
<ExperimentCardSkeleton />
</>
)}
</SimpleGrid>
</AppShell>
);
}

181
app/src/pages/logged-calls/index.tsx Normal file
View File

@@ -0,0 +1,181 @@
import {
Heading,
Text,
Stat,
StatLabel,
StatNumber,
VStack,
HStack,
Card,
CardBody,
CardHeader,
Icon,
Table,
Tbody,
Tr,
Td,
Divider,
Breadcrumb,
BreadcrumbItem,
} from "@chakra-ui/react";
import {
LineChart,
Line,
XAxis,
YAxis,
CartesianGrid,
Tooltip,
Legend,
ResponsiveContainer,
} from "recharts";
import { Ban, DollarSign, Hash } from "lucide-react";
import { useMemo } from "react";
import AppShell from "~/components/nav/AppShell";
import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
import { useSelectedOrg } from "~/utils/hooks";
import dayjs from "~/utils/dayjs";
import { api } from "~/utils/api";
import LoggedCallTable from "~/components/dashboard/LoggedCallTable";
export default function LoggedCalls() {
const { data: selectedOrg } = useSelectedOrg();
const stats = api.dashboard.stats.useQuery(
{ organizationId: selectedOrg?.id ?? "" },
{ enabled: !!selectedOrg },
);
const data = useMemo(() => {
return (
stats.data?.periods.map(({ period, numQueries, totalCost }) => ({
period,
Requests: numQueries,
"Total Spent (USD)": parseFloat(totalCost.toString()),
})) || []
);
}, [stats.data]);
return (
<AppShell requireAuth>
<PageHeaderContainer>
<Breadcrumb>
<BreadcrumbItem>
<ProjectBreadcrumbContents />
</BreadcrumbItem>
<BreadcrumbItem isCurrentPage>
<Text>Logged Calls</Text>
</BreadcrumbItem>
</Breadcrumb>
</PageHeaderContainer>
<VStack px={8} pt={4} alignItems="flex-start" spacing={4}>
<Text fontSize="2xl" fontWeight="bold">
{selectedOrg?.name}
</Text>
<Divider />
<VStack margin="auto" spacing={4} align="stretch" w="full">
<HStack gap={4} align="start">
<Card variant="outline" flex={1}>
<CardHeader>
<Heading as="h3" size="sm">
Usage Statistics
</Heading>
</CardHeader>
<CardBody>
<ResponsiveContainer width="100%" height={400}>
<LineChart data={data} margin={{ top: 5, right: 20, left: 10, bottom: 5 }}>
<XAxis
dataKey="period"
tickFormatter={(str: string) => dayjs(str).format("MMM D")}
/>
<YAxis yAxisId="left" dataKey="Requests" orientation="left" stroke="#8884d8" />
<YAxis
yAxisId="right"
dataKey="Total Spent (USD)"
orientation="right"
unit="$"
stroke="#82ca9d"
/>
<Tooltip />
<Legend />
<CartesianGrid stroke="#f5f5f5" />
<Line
dataKey="Requests"
stroke="#8884d8"
yAxisId="left"
dot={false}
strokeWidth={2}
/>
<Line
dataKey="Total Spent (USD)"
stroke="#82ca9d"
yAxisId="right"
dot={false}
strokeWidth={2}
/>
</LineChart>
</ResponsiveContainer>
</CardBody>
</Card>
<VStack spacing="4" width="300px" align="stretch">
<Card variant="outline">
<CardBody>
<Stat>
<HStack>
<StatLabel flex={1}>Total Spent</StatLabel>
<Icon as={DollarSign} boxSize={4} color="gray.500" />
</HStack>
<StatNumber>
${parseFloat(stats.data?.totals?.totalCost?.toString() ?? "0").toFixed(2)}
</StatNumber>
</Stat>
</CardBody>
</Card>
<Card variant="outline">
<CardBody>
<Stat>
<HStack>
<StatLabel flex={1}>Total Requests</StatLabel>
<Icon as={Hash} boxSize={4} color="gray.500" />
</HStack>
<StatNumber>
{stats.data?.totals?.numQueries
? parseInt(stats.data?.totals?.numQueries.toString())?.toLocaleString()
: undefined}
</StatNumber>
</Stat>
</CardBody>
</Card>
<Card variant="outline" overflow="hidden">
<Stat>
<CardHeader>
<HStack>
<StatLabel flex={1}>Errors</StatLabel>
<Icon as={Ban} boxSize={4} color="gray.500" />
</HStack>
</CardHeader>
<Table variant="simple">
<Tbody>
{stats.data?.errors?.map((error) => (
<Tr key={error.code}>
<Td>
{error.name} ({error.code})
</Td>
<Td isNumeric color="red.600">
{parseInt(error.count.toString()).toLocaleString()}
</Td>
</Tr>
))}
</Tbody>
</Table>
</Stat>
</Card>
</VStack>
</HStack>
<LoggedCallTable />
</VStack>
</VStack>
</AppShell>
);
}

152
app/src/pages/project/settings/index.tsx Normal file
View File

@@ -0,0 +1,152 @@
import {
Breadcrumb,
BreadcrumbItem,
Text,
type TextProps,
VStack,
HStack,
Input,
Button,
Divider,
Icon,
useDisclosure,
} from "@chakra-ui/react";
import { useEffect, useState } from "react";
import { BsTrash } from "react-icons/bs";
import AppShell from "~/components/nav/AppShell";
import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
import { api } from "~/utils/api";
import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
import CopiableCode from "~/components/CopiableCode";
import { DeleteProjectDialog } from "~/components/projectSettings/DeleteProjectDialog";
export default function Settings() {
const utils = api.useContext();
const { data: selectedOrg } = useSelectedOrg();
const apiKey =
selectedOrg?.apiKeys?.length && selectedOrg?.apiKeys[0] ? selectedOrg?.apiKeys[0].apiKey : "";
const updateMutation = api.organizations.update.useMutation();
const [onSaveName] = useHandledAsyncCallback(async () => {
if (name && name !== selectedOrg?.name && selectedOrg?.id) {
await updateMutation.mutateAsync({
id: selectedOrg.id,
updates: { name },
});
await Promise.all([utils.organizations.get.invalidate({ id: selectedOrg.id })]);
}
}, [updateMutation, selectedOrg]);
const [name, setName] = useState(selectedOrg?.name);
useEffect(() => {
setName(selectedOrg?.name);
}, [selectedOrg?.name]);
const deleteProjectOpen = useDisclosure();
return (
<>
<AppShell>
<PageHeaderContainer>
<Breadcrumb>
<BreadcrumbItem>
<ProjectBreadcrumbContents />
</BreadcrumbItem>
<BreadcrumbItem isCurrentPage>
<Text>Project Settings</Text>
</BreadcrumbItem>
</Breadcrumb>
</PageHeaderContainer>
<VStack px={8} pt={4} alignItems="flex-start" spacing={4}>
<VStack spacing={0} alignItems="flex-start">
<Text fontSize="2xl" fontWeight="bold">
Project Settings
</Text>
<Text fontSize="sm">
Configure your project settings. These settings only apply to {selectedOrg?.name}.
</Text>
</VStack>
<VStack
w="full"
alignItems="flex-start"
borderWidth={1}
borderRadius={4}
borderColor="gray.300"
p={6}
spacing={6}
>
<VStack alignItems="flex-start" w="full">
<Text fontWeight="bold" fontSize="xl">
Display Name
</Text>
<Input
w="full"
maxW={600}
value={name}
onChange={(e) => setName(e.target.value)}
borderColor="gray.300"
/>
<Button
isDisabled={!name || name === selectedOrg?.name}
colorScheme="orange"
borderRadius={4}
mt={2}
_disabled={{
opacity: 0.6,
}}
onClick={onSaveName}
>
Rename Project
</Button>
</VStack>
<Divider backgroundColor="gray.300" />
<VStack alignItems="flex-start">
<Subtitle>Project API Key</Subtitle>
<Text fontSize="sm">
Use your project API key to authenticate your requests when sending data to
OpenPipe. You can set this key in your environment variables, or use it directly in
your code.
</Text>
</VStack>
<CopiableCode code={apiKey} />
<Divider />
{selectedOrg?.personalOrgUserId ? (
<VStack alignItems="flex-start">
<Subtitle>Personal Project</Subtitle>
<Text fontSize="sm">
This project is {selectedOrg?.personalOrgUser?.name}'s personal project. It cannot
be deleted.
</Text>
</VStack>
) : (
<VStack alignItems="flex-start">
<Subtitle color="red.600">Danger Zone</Subtitle>
<Text fontSize="sm">
Permanently delete your project and all of its data. This action cannot be undone.
</Text>
<HStack
as={Button}
isDisabled={selectedOrg?.role !== "ADMIN"}
colorScheme="red"
variant="outline"
borderRadius={4}
mt={2}
onClick={deleteProjectOpen.onOpen}
>
<Icon as={BsTrash} />
<Text>Delete {selectedOrg?.name}</Text>
</HStack>
</VStack>
)}
</VStack>
</VStack>
</AppShell>
<DeleteProjectDialog isOpen={deleteProjectOpen.isOpen} onClose={deleteProjectOpen.onClose} />
</>
);
}
const Subtitle = (props: TextProps) => <Text fontWeight="bold" fontSize="xl" {...props} />;

6
app/src/server/api/root.router.ts
View File

@@ -8,6 +8,9 @@ import { evaluationsRouter } from "./routers/evaluations.router";
import { worldChampsRouter } from "./routers/worldChamps.router";
import { datasetsRouter } from "./routers/datasets.router";
import { datasetEntries } from "./routers/datasetEntries.router";
import { externalApiRouter } from "./routers/externalApi.router";
import { organizationsRouter } from "./routers/organizations.router";
import { dashboardRouter } from "./routers/dashboard.router";
/**
* This is the primary router for your server.
@@ -24,6 +27,9 @@ export const appRouter = createTRPCRouter({
worldChamps: worldChampsRouter,
datasets: datasetsRouter,
datasetEntries: datasetEntries,
organizations: organizationsRouter,
dashboard: dashboardRouter,
externalApi: externalApiRouter,
});
// export type definition of API

118
app/src/server/api/routers/dashboard.router.ts Normal file
View File

@@ -0,0 +1,118 @@
import { sql } from "kysely";
import { z } from "zod";
import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
import { kysely, prisma } from "~/server/db";
import dayjs from "~/utils/dayjs";
export const dashboardRouter = createTRPCRouter({
stats: publicProcedure
.input(
z.object({
// TODO: actually take startDate into account
startDate: z.string().optional(),
organizationId: z.string(),
}),
)
.query(async ({ input }) => {
// Return the stats group by hour
const periods = await kysely
.selectFrom("LoggedCall")
.leftJoin(
"LoggedCallModelResponse",
"LoggedCall.id",
"LoggedCallModelResponse.originalLoggedCallId",
)
.where("organizationId", "=", input.organizationId)
.select(({ fn }) => [
sql<Date>`date_trunc('day', "LoggedCallModelResponse"."startTime")`.as("period"),
sql<number>`count("LoggedCall"."id")::int`.as("numQueries"),
fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
])
.groupBy("period")
.orderBy("period")
.execute();
let originalDataIndex = periods.length - 1;
// *SLAMS DOWN GLASS OF WHISKEY* timezones, amirite?
let dayToMatch = dayjs(input.startDate || new Date());
// Ensure that the initial date we're matching against is never before the first period
if (
periods[originalDataIndex] &&
dayToMatch.isBefore(periods[originalDataIndex]?.period, "day")
) {
dayToMatch = dayjs(periods[originalDataIndex]?.period);
}
const backfilledPeriods: typeof periods = [];
// Backfill from now to 14 days ago or the date of the first logged call, whichever is earlier
while (
backfilledPeriods.length < 14 ||
(periods[0]?.period && !dayToMatch.isBefore(periods[0]?.period, "day"))
) {
const nextOriginalPeriod = periods[originalDataIndex];
if (nextOriginalPeriod && dayjs(nextOriginalPeriod?.period).isSame(dayToMatch, "day")) {
backfilledPeriods.unshift(nextOriginalPeriod);
originalDataIndex--;
} else {
backfilledPeriods.unshift({
period: dayjs(dayToMatch).toDate(),
numQueries: 0,
totalCost: 0,
});
}
dayToMatch = dayToMatch.subtract(1, "day");
}
const totals = await kysely
.selectFrom("LoggedCall")
.leftJoin(
"LoggedCallModelResponse",
"LoggedCall.id",
"LoggedCallModelResponse.originalLoggedCallId",
)
.where("organizationId", "=", input.organizationId)
.select(({ fn }) => [
fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
fn.count("LoggedCall.id").as("numQueries"),
])
.executeTakeFirst();
const errors = await kysely
.selectFrom("LoggedCall")
.where("organizationId", "=", input.organizationId)
.leftJoin(
"LoggedCallModelResponse",
"LoggedCall.id",
"LoggedCallModelResponse.originalLoggedCallId",
)
.select(({ fn }) => [fn.count("LoggedCall.id").as("count"), "respStatus as code"])
.where("respStatus", ">", 200)
.groupBy("code")
.orderBy("count", "desc")
.execute();
const namedErrors = errors.map((e) => {
if (e.code === 429) {
return { ...e, name: "Rate limited" };
} else if (e.code === 500) {
return { ...e, name: "Internal server error" };
} else {
return { ...e, name: "Other" };
}
});
return { periods: backfilledPeriods, totals, errors: namedErrors };
}),
// TODO useInfiniteQuery
// https://discord.com/channels/966627436387266600/1122258443886153758/1122258443886153758
loggedCalls: publicProcedure.input(z.object({})).query(async ({ input }) => {
const loggedCalls = await prisma.loggedCall.findMany({
orderBy: { startTime: "desc" },
include: { tags: true, modelResponse: true },
take: 20,
});
return loggedCalls;
}),
});

77
app/src/server/api/routers/datasets.router.ts
View File

@@ -3,65 +3,62 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
import { prisma } from "~/server/db";
import {
requireCanModifyDataset,
requireCanModifyOrganization,
requireCanViewDataset,
requireNothing,
requireCanViewOrganization,
} from "~/utils/accessControl";
import userOrg from "~/server/utils/userOrg";
export const datasetsRouter = createTRPCRouter({
list: protectedProcedure.query(async ({ ctx }) => {
// Anyone can list experiments
requireNothing(ctx);
list: protectedProcedure
.input(z.object({ organizationId: z.string() }))
.query(async ({ input, ctx }) => {
await requireCanViewOrganization(input.organizationId, ctx);
const datasets = await prisma.dataset.findMany({
where: {
organization: {
organizationUsers: {
some: { userId: ctx.session.user.id },
const datasets = await prisma.dataset.findMany({
where: {
organizationId: input.organizationId,
},
orderBy: {
createdAt: "desc",
},
include: {
_count: {
select: { datasetEntries: true },
},
},
},
orderBy: {
createdAt: "desc",
},
include: {
_count: {
select: { datasetEntries: true },
},
},
});
});
return datasets;
}),
return datasets;
}),
get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
await requireCanViewDataset(input.id, ctx);
return await prisma.dataset.findFirstOrThrow({
where: { id: input.id },
include: {
organization: true,
},
});
}),
create: protectedProcedure.input(z.object({})).mutation(async ({ ctx }) => {
// Anyone can create an experiment
requireNothing(ctx);
create: protectedProcedure
.input(z.object({ organizationId: z.string() }))
.mutation(async ({ input, ctx }) => {
await requireCanModifyOrganization(input.organizationId, ctx);
const numDatasets = await prisma.dataset.count({
where: {
organization: {
organizationUsers: {
some: { userId: ctx.session.user.id },
},
const numDatasets = await prisma.dataset.count({
where: {
organizationId: input.organizationId,
},
},
});
});
return await prisma.dataset.create({
data: {
name: `Dataset ${numDatasets + 1}`,
organizationId: (await userOrg(ctx.session.user.id)).id,
},
});
}),
return await prisma.dataset.create({
data: {
name: `Dataset ${numDatasets + 1}`,
organizationId: input.organizationId,
},
});
}),
update: protectedProcedure
.input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))

548
app/src/server/api/routers/experiments.router.ts
View File

@@ -8,10 +8,10 @@ import { generateNewCell } from "~/server/utils/generateNewCell";
import {
canModifyExperiment,
requireCanModifyExperiment,
requireCanModifyOrganization,
requireCanViewExperiment,
requireNothing,
requireCanViewOrganization,
} from "~/utils/accessControl";
import userOrg from "~/server/utils/userOrg";
import generateTypes from "~/modelProviders/generateTypes";
import { promptConstructorVersion } from "~/promptConstructor/version";
@@ -43,55 +43,55 @@ export const experimentsRouter = createTRPCRouter({
testScenarioCount,
};
}),
list: protectedProcedure.query(async ({ ctx }) => {
// Anyone can list experiments
requireNothing(ctx);
list: protectedProcedure
.input(z.object({ organizationId: z.string() }))
.query(async ({ input, ctx }) => {
await requireCanViewOrganization(input.organizationId, ctx);
const experiments = await prisma.experiment.findMany({
where: {
organization: {
organizationUsers: {
some: { userId: ctx.session.user.id },
},
const experiments = await prisma.experiment.findMany({
where: {
organizationId: input.organizationId,
},
},
orderBy: {
sortIndex: "desc",
},
});
orderBy: {
sortIndex: "desc",
},
});
// TODO: look for cleaner way to do this. Maybe aggregate?
const experimentsWithCounts = await Promise.all(
experiments.map(async (experiment) => {
const visibleTestScenarioCount = await prisma.testScenario.count({
where: {
experimentId: experiment.id,
visible: true,
},
});
// TODO: look for cleaner way to do this. Maybe aggregate?
const experimentsWithCounts = await Promise.all(
experiments.map(async (experiment) => {
const visibleTestScenarioCount = await prisma.testScenario.count({
where: {
experimentId: experiment.id,
visible: true,
},
});
const visiblePromptVariantCount = await prisma.promptVariant.count({
where: {
experimentId: experiment.id,
visible: true,
},
});
const visiblePromptVariantCount = await prisma.promptVariant.count({
where: {
experimentId: experiment.id,
visible: true,
},
});
return {
...experiment,
testScenarioCount: visibleTestScenarioCount,
promptVariantCount: visiblePromptVariantCount,
};
}),
);
return {
...experiment,
testScenarioCount: visibleTestScenarioCount,
promptVariantCount: visiblePromptVariantCount,
};
}),
);
return experimentsWithCounts;
}),
return experimentsWithCounts;
}),
get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
await requireCanViewExperiment(input.id, ctx);
const experiment = await prisma.experiment.findFirstOrThrow({
where: { id: input.id },
include: {
organization: true,
},
});
const canModify = ctx.session?.user.id
@@ -107,222 +107,224 @@ export const experimentsRouter = createTRPCRouter({
};
}),
fork: protectedProcedure.input(z.object({ id: z.string() })).mutation(async ({ input, ctx }) => {
await requireCanViewExperiment(input.id, ctx);
fork: protectedProcedure
.input(z.object({ id: z.string(), organizationId: z.string() }))
.mutation(async ({ input, ctx }) => {
await requireCanViewExperiment(input.id, ctx);
await requireCanModifyOrganization(input.organizationId, ctx);
const [
existingExp,
existingVariants,
existingScenarios,
existingCells,
evaluations,
templateVariables,
] = await prisma.$transaction([
prisma.experiment.findUniqueOrThrow({
where: {
id: input.id,
},
}),
prisma.promptVariant.findMany({
where: {
experimentId: input.id,
visible: true,
},
}),
prisma.testScenario.findMany({
where: {
experimentId: input.id,
visible: true,
},
}),
prisma.scenarioVariantCell.findMany({
where: {
testScenario: {
visible: true,
const [
existingExp,
existingVariants,
existingScenarios,
existingCells,
evaluations,
templateVariables,
] = await prisma.$transaction([
prisma.experiment.findUniqueOrThrow({
where: {
id: input.id,
},
promptVariant: {
}),
prisma.promptVariant.findMany({
where: {
experimentId: input.id,
visible: true,
},
},
include: {
modelResponses: {
include: {
outputEvaluations: true,
}),
prisma.testScenario.findMany({
where: {
experimentId: input.id,
visible: true,
},
}),
prisma.scenarioVariantCell.findMany({
where: {
testScenario: {
visible: true,
},
promptVariant: {
experimentId: input.id,
visible: true,
},
},
},
}),
prisma.evaluation.findMany({
where: {
experimentId: input.id,
},
}),
prisma.templateVariable.findMany({
where: {
experimentId: input.id,
},
}),
]);
include: {
modelResponses: {
include: {
outputEvaluations: true,
},
},
},
}),
prisma.evaluation.findMany({
where: {
experimentId: input.id,
},
}),
prisma.templateVariable.findMany({
where: {
experimentId: input.id,
},
}),
]);
const newExperimentId = uuidv4();
const newExperimentId = uuidv4();
const existingToNewVariantIds = new Map<string, string>();
const variantsToCreate: Prisma.PromptVariantCreateManyInput[] = [];
for (const variant of existingVariants) {
const newVariantId = uuidv4();
existingToNewVariantIds.set(variant.id, newVariantId);
variantsToCreate.push({
...variant,
id: newVariantId,
experimentId: newExperimentId,
});
}
const existingToNewScenarioIds = new Map<string, string>();
const scenariosToCreate: Prisma.TestScenarioCreateManyInput[] = [];
for (const scenario of existingScenarios) {
const newScenarioId = uuidv4();
existingToNewScenarioIds.set(scenario.id, newScenarioId);
scenariosToCreate.push({
...scenario,
id: newScenarioId,
experimentId: newExperimentId,
variableValues: scenario.variableValues as Prisma.InputJsonValue,
});
}
const existingToNewEvaluationIds = new Map<string, string>();
const evaluationsToCreate: Prisma.EvaluationCreateManyInput[] = [];
for (const evaluation of evaluations) {
const newEvaluationId = uuidv4();
existingToNewEvaluationIds.set(evaluation.id, newEvaluationId);
evaluationsToCreate.push({
...evaluation,
id: newEvaluationId,
experimentId: newExperimentId,
});
}
const cellsToCreate: Prisma.ScenarioVariantCellCreateManyInput[] = [];
const modelResponsesToCreate: Prisma.ModelResponseCreateManyInput[] = [];
const outputEvaluationsToCreate: Prisma.OutputEvaluationCreateManyInput[] = [];
for (const cell of existingCells) {
const newCellId = uuidv4();
const { modelResponses, ...cellData } = cell;
cellsToCreate.push({
...cellData,
id: newCellId,
promptVariantId: existingToNewVariantIds.get(cell.promptVariantId) ?? "",
testScenarioId: existingToNewScenarioIds.get(cell.testScenarioId) ?? "",
prompt: (cell.prompt as Prisma.InputJsonValue) ?? undefined,
});
for (const modelResponse of modelResponses) {
const newModelResponseId = uuidv4();
const { outputEvaluations, ...modelResponseData } = modelResponse;
modelResponsesToCreate.push({
...modelResponseData,
id: newModelResponseId,
scenarioVariantCellId: newCellId,
output: (modelResponse.output as Prisma.InputJsonValue) ?? undefined,
const existingToNewVariantIds = new Map<string, string>();
const variantsToCreate: Prisma.PromptVariantCreateManyInput[] = [];
for (const variant of existingVariants) {
const newVariantId = uuidv4();
existingToNewVariantIds.set(variant.id, newVariantId);
variantsToCreate.push({
...variant,
id: newVariantId,
experimentId: newExperimentId,
});
for (const evaluation of outputEvaluations) {
outputEvaluationsToCreate.push({
...evaluation,
id: uuidv4(),
modelResponseId: newModelResponseId,
evaluationId: existingToNewEvaluationIds.get(evaluation.evaluationId) ?? "",
}
const existingToNewScenarioIds = new Map<string, string>();
const scenariosToCreate: Prisma.TestScenarioCreateManyInput[] = [];
for (const scenario of existingScenarios) {
const newScenarioId = uuidv4();
existingToNewScenarioIds.set(scenario.id, newScenarioId);
scenariosToCreate.push({
...scenario,
id: newScenarioId,
experimentId: newExperimentId,
variableValues: scenario.variableValues as Prisma.InputJsonValue,
});
}
const existingToNewEvaluationIds = new Map<string, string>();
const evaluationsToCreate: Prisma.EvaluationCreateManyInput[] = [];
for (const evaluation of evaluations) {
const newEvaluationId = uuidv4();
existingToNewEvaluationIds.set(evaluation.id, newEvaluationId);
evaluationsToCreate.push({
...evaluation,
id: newEvaluationId,
experimentId: newExperimentId,
});
}
const cellsToCreate: Prisma.ScenarioVariantCellCreateManyInput[] = [];
const modelResponsesToCreate: Prisma.ModelResponseCreateManyInput[] = [];
const outputEvaluationsToCreate: Prisma.OutputEvaluationCreateManyInput[] = [];
for (const cell of existingCells) {
const newCellId = uuidv4();
const { modelResponses, ...cellData } = cell;
cellsToCreate.push({
...cellData,
id: newCellId,
promptVariantId: existingToNewVariantIds.get(cell.promptVariantId) ?? "",
testScenarioId: existingToNewScenarioIds.get(cell.testScenarioId) ?? "",
prompt: (cell.prompt as Prisma.InputJsonValue) ?? undefined,
});
for (const modelResponse of modelResponses) {
const newModelResponseId = uuidv4();
const { outputEvaluations, ...modelResponseData } = modelResponse;
modelResponsesToCreate.push({
...modelResponseData,
id: newModelResponseId,
scenarioVariantCellId: newCellId,
output: (modelResponse.output as Prisma.InputJsonValue) ?? undefined,
});
for (const evaluation of outputEvaluations) {
outputEvaluationsToCreate.push({
...evaluation,
id: uuidv4(),
modelResponseId: newModelResponseId,
evaluationId: existingToNewEvaluationIds.get(evaluation.evaluationId) ?? "",
});
}
}
}
}
const templateVariablesToCreate: Prisma.TemplateVariableCreateManyInput[] = [];
for (const templateVariable of templateVariables) {
templateVariablesToCreate.push({
...templateVariable,
id: uuidv4(),
experimentId: newExperimentId,
});
}
const templateVariablesToCreate: Prisma.TemplateVariableCreateManyInput[] = [];
for (const templateVariable of templateVariables) {
templateVariablesToCreate.push({
...templateVariable,
id: uuidv4(),
experimentId: newExperimentId,
});
}
const maxSortIndex =
(
await prisma.experiment.aggregate({
_max: {
sortIndex: true,
const maxSortIndex =
(
await prisma.experiment.aggregate({
_max: {
sortIndex: true,
},
})
)._max?.sortIndex ?? 0;
await prisma.$transaction([
prisma.experiment.create({
data: {
id: newExperimentId,
sortIndex: maxSortIndex + 1,
label: `${existingExp.label} (forked)`,
organizationId: input.organizationId,
},
})
)._max?.sortIndex ?? 0;
}),
prisma.promptVariant.createMany({
data: variantsToCreate,
}),
prisma.testScenario.createMany({
data: scenariosToCreate,
}),
prisma.scenarioVariantCell.createMany({
data: cellsToCreate,
}),
prisma.modelResponse.createMany({
data: modelResponsesToCreate,
}),
prisma.evaluation.createMany({
data: evaluationsToCreate,
}),
prisma.outputEvaluation.createMany({
data: outputEvaluationsToCreate,
}),
prisma.templateVariable.createMany({
data: templateVariablesToCreate,
}),
]);
await prisma.$transaction([
prisma.experiment.create({
return newExperimentId;
}),
create: protectedProcedure
.input(z.object({ organizationId: z.string() }))
.mutation(async ({ input, ctx }) => {
await requireCanModifyOrganization(input.organizationId, ctx);
const maxSortIndex =
(
await prisma.experiment.aggregate({
_max: {
sortIndex: true,
},
where: { organizationId: input.organizationId },
})
)._max?.sortIndex ?? 0;
const exp = await prisma.experiment.create({
data: {
id: newExperimentId,
sortIndex: maxSortIndex + 1,
label: `${existingExp.label} (forked)`,
organizationId: (await userOrg(ctx.session.user.id)).id,
label: `Experiment ${maxSortIndex + 1}`,
organizationId: input.organizationId,
},
}),
prisma.promptVariant.createMany({
data: variantsToCreate,
}),
prisma.testScenario.createMany({
data: scenariosToCreate,
}),
prisma.scenarioVariantCell.createMany({
data: cellsToCreate,
}),
prisma.modelResponse.createMany({
data: modelResponsesToCreate,
}),
prisma.evaluation.createMany({
data: evaluationsToCreate,
}),
prisma.outputEvaluation.createMany({
data: outputEvaluationsToCreate,
}),
prisma.templateVariable.createMany({
data: templateVariablesToCreate,
}),
]);
});
return newExperimentId;
}),
create: protectedProcedure.input(z.object({})).mutation(async ({ ctx }) => {
// Anyone can create an experiment
requireNothing(ctx);
const organizationId = (await userOrg(ctx.session.user.id)).id;
const maxSortIndex =
(
await prisma.experiment.aggregate({
_max: {
sortIndex: true,
},
where: { organizationId },
})
)._max?.sortIndex ?? 0;
const exp = await prisma.experiment.create({
data: {
sortIndex: maxSortIndex + 1,
label: `Experiment ${maxSortIndex + 1}`,
organizationId,
},
});
const [variant, _, scenario1, scenario2, scenario3] = await prisma.$transaction([
prisma.promptVariant.create({
data: {
experimentId: exp.id,
label: "Prompt Variant 1",
sortIndex: 0,
// The interpolated $ is necessary until dedent incorporates
// https://github.com/dmnd/dedent/pull/46
promptConstructor: dedent`
const [variant, _, scenario1, scenario2, scenario3] = await prisma.$transaction([
prisma.promptVariant.create({
data: {
experimentId: exp.id,
label: "Prompt Variant 1",
sortIndex: 0,
// The interpolated $ is necessary until dedent incorporates
// https://github.com/dmnd/dedent/pull/46
promptConstructor: dedent`
/**
* Use Javascript to define an OpenAI chat completion
* (https://platform.openai.com/docs/api-reference/chat/create).
@@ -341,49 +343,49 @@ export const experimentsRouter = createTRPCRouter({
},
],
});`,
model: "gpt-3.5-turbo-0613",
modelProvider: "openai/ChatCompletion",
promptConstructorVersion,
},
}),
prisma.templateVariable.create({
data: {
experimentId: exp.id,
label: "language",
},
}),
prisma.testScenario.create({
data: {
experimentId: exp.id,
variableValues: {
language: "English",
model: "gpt-3.5-turbo-0613",
modelProvider: "openai/ChatCompletion",
promptConstructorVersion,
},
},
}),
prisma.testScenario.create({
data: {
experimentId: exp.id,
variableValues: {
language: "Spanish",
}),
prisma.templateVariable.create({
data: {
experimentId: exp.id,
label: "language",
},
},
}),
prisma.testScenario.create({
data: {
experimentId: exp.id,
variableValues: {
language: "German",
}),
prisma.testScenario.create({
data: {
experimentId: exp.id,
variableValues: {
language: "English",
},
},
},
}),
]);
}),
prisma.testScenario.create({
data: {
experimentId: exp.id,
variableValues: {
language: "Spanish",
},
},
}),
prisma.testScenario.create({
data: {
experimentId: exp.id,
variableValues: {
language: "German",
},
},
}),
]);
await generateNewCell(variant.id, scenario1.id);
await generateNewCell(variant.id, scenario2.id);
await generateNewCell(variant.id, scenario3.id);
await generateNewCell(variant.id, scenario1.id);
await generateNewCell(variant.id, scenario2.id);
await generateNewCell(variant.id, scenario3.id);
return exp;
}),
return exp;
}),
update: protectedProcedure
.input(z.object({ id: z.string(), updates: z.object({ label: z.string() }) }))

205
app/src/server/api/routers/externalApi.router.ts Normal file
View File

@@ -0,0 +1,205 @@
import { type Prisma } from "@prisma/client";
import { type JsonValue } from "type-fest";
import { z } from "zod";
import { v4 as uuidv4 } from "uuid";
import { TRPCError } from "@trpc/server";
import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
import { prisma } from "~/server/db";
import { hashRequest } from "~/server/utils/hashObject";
const reqValidator = z.object({
model: z.string(),
messages: z.array(z.any()),
});
const respValidator = z.object({
id: z.string(),
model: z.string(),
usage: z.object({
total_tokens: z.number(),
prompt_tokens: z.number(),
completion_tokens: z.number(),
}),
choices: z.array(
z.object({
finish_reason: z.string(),
}),
),
});
export const externalApiRouter = createTRPCRouter({
checkCache: publicProcedure
.meta({
openapi: {
method: "POST",
path: "/v1/check-cache",
description: "Check if a prompt is cached",
},
})
.input(
z.object({
startTime: z.number().describe("Unix timestamp in milliseconds"),
reqPayload: z.unknown().describe("JSON-encoded request payload"),
tags: z
.record(z.string())
.optional()
.describe(
'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
),
}),
)
.output(
z.object({
respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
}),
)
.mutation(async ({ input, ctx }) => {
const apiKey = ctx.apiKey;
if (!apiKey) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const key = await prisma.apiKey.findUnique({
where: { apiKey },
});
if (!key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const reqPayload = await reqValidator.spa(input.reqPayload);
const cacheKey = hashRequest(key.organizationId, reqPayload as JsonValue);
const existingResponse = await prisma.loggedCallModelResponse.findFirst({
where: {
cacheKey,
},
include: {
originalLoggedCall: true,
},
orderBy: {
startTime: "desc",
},
});
if (!existingResponse) return { respPayload: null };
await prisma.loggedCall.create({
data: {
organizationId: key.organizationId,
startTime: new Date(input.startTime),
cacheHit: true,
modelResponseId: existingResponse.id,
},
});
return {
respPayload: existingResponse.respPayload,
};
}),
report: publicProcedure
.meta({
openapi: {
method: "POST",
path: "/v1/report",
description: "Report an API call",
},
})
.input(
z.object({
startTime: z.number().describe("Unix timestamp in milliseconds"),
endTime: z.number().describe("Unix timestamp in milliseconds"),
reqPayload: z.unknown().describe("JSON-encoded request payload"),
respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
respStatus: z.number().optional().describe("HTTP status code of response"),
error: z.string().optional().describe("User-friendly error message"),
tags: z
.record(z.string())
.optional()
.describe(
'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
),
}),
)
.output(z.void())
.mutation(async ({ input, ctx }) => {
const apiKey = ctx.apiKey;
if (!apiKey) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const key = await prisma.apiKey.findUnique({
where: { apiKey },
});
if (!key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const reqPayload = await reqValidator.spa(input.reqPayload);
const respPayload = await respValidator.spa(input.respPayload);
const requestHash = hashRequest(key.organizationId, reqPayload as JsonValue);
const newLoggedCallId = uuidv4();
const newModelResponseId = uuidv4();
const usage = respPayload.success ? respPayload.data.usage : undefined;
await prisma.$transaction([
prisma.loggedCall.create({
data: {
id: newLoggedCallId,
organizationId: key.organizationId,
startTime: new Date(input.startTime),
cacheHit: false,
},
}),
prisma.loggedCallModelResponse.create({
data: {
id: newModelResponseId,
originalLoggedCallId: newLoggedCallId,
startTime: new Date(input.startTime),
endTime: new Date(input.endTime),
reqPayload: input.reqPayload as Prisma.InputJsonValue,
respPayload: input.respPayload as Prisma.InputJsonValue,
respStatus: input.respStatus,
error: input.error,
durationMs: input.endTime - input.startTime,
...(respPayload.success
? {
cacheKey: requestHash,
inputTokens: usage ? usage.prompt_tokens : undefined,
outputTokens: usage ? usage.completion_tokens : undefined,
}
: null),
},
}),
// Avoid foreign key constraint error by updating the logged call after the model response is created
prisma.loggedCall.update({
where: {
id: newLoggedCallId,
},
data: {
modelResponseId: newModelResponseId,
},
}),
]);
if (input.tags) {
const tagsToCreate = Object.entries(input.tags).map(([name, value]) => ({
loggedCallId: newLoggedCallId,
// sanitize tags
name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
value,
}));
if (reqPayload.success) {
tagsToCreate.push({
loggedCallId: newLoggedCallId,
name: "$model",
value: reqPayload.data.model,
});
}
await prisma.loggedCallTag.createMany({
data: tagsToCreate,
});
}
}),
});

128
app/src/server/api/routers/organizations.router.ts Normal file
View File

@@ -0,0 +1,128 @@
import { TRPCError } from "@trpc/server";
import { v4 as uuidv4 } from "uuid";
import { z } from "zod";
import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
import { prisma } from "~/server/db";
import { generateApiKey } from "~/server/utils/generateApiKey";
import userOrg from "~/server/utils/userOrg";
import {
requireCanModifyOrganization,
requireCanViewOrganization,
requireIsOrgAdmin,
requireNothing,
} from "~/utils/accessControl";
export const organizationsRouter = createTRPCRouter({
list: protectedProcedure.query(async ({ ctx }) => {
const userId = ctx.session.user.id;
requireNothing(ctx);
if (!userId) {
return null;
}
const organizations = await prisma.organization.findMany({
where: {
organizationUsers: {
some: { userId: ctx.session.user.id },
},
},
orderBy: {
createdAt: "asc",
},
});
if (!organizations.length) {
// TODO: We should move this to a separate endpoint that is called on sign up
const personalOrg = await userOrg(userId);
organizations.push(personalOrg);
}
return organizations;
}),
get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
await requireCanViewOrganization(input.id, ctx);
const [org, userRole] = await prisma.$transaction([
prisma.organization.findUnique({
where: {
id: input.id,
},
include: {
apiKeys: true,
personalOrgUser: true,
},
}),
prisma.organizationUser.findFirst({
where: {
userId: ctx.session.user.id,
organizationId: input.id,
role: {
in: ["ADMIN", "MEMBER"],
},
},
}),
]);
if (!org) {
throw new TRPCError({ code: "NOT_FOUND" });
}
return {
...org,
role: userRole?.role ?? null,
};
}),
update: protectedProcedure
.input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))
.mutation(async ({ input, ctx }) => {
await requireCanModifyOrganization(input.id, ctx);
return await prisma.organization.update({
where: {
id: input.id,
},
data: {
name: input.updates.name,
},
});
}),
create: protectedProcedure
.input(z.object({ name: z.string() }))
.mutation(async ({ input, ctx }) => {
requireNothing(ctx);
const newOrgId = uuidv4();
const [newOrg] = await prisma.$transaction([
prisma.organization.create({
data: {
id: newOrgId,
name: input.name,
},
}),
prisma.organizationUser.create({
data: {
userId: ctx.session.user.id,
organizationId: newOrgId,
role: "ADMIN",
},
}),
prisma.apiKey.create({
data: {
name: "Default API Key",
organizationId: newOrgId,
apiKey: generateApiKey(),
},
}),
]);
return newOrg;
}),
delete: protectedProcedure
.input(z.object({ id: z.string() }))
.mutation(async ({ input, ctx }) => {
await requireIsOrgAdmin(input.id, ctx);
return await prisma.organization.delete({
where: {
id: input.id,
},
});
}),
});

33
app/src/server/api/trpc.ts
View File

@@ -11,6 +11,7 @@ import { initTRPC, TRPCError } from "@trpc/server";
import { type CreateNextContextOptions } from "@trpc/server/adapters/next";
import { type Session } from "next-auth";
import superjson from "superjson";
import { type OpenApiMeta } from "trpc-openapi";
import { ZodError } from "zod";
import { getServerAuthSession } from "~/server/auth";
import { prisma } from "~/server/db";
@@ -26,6 +27,7 @@ import { capturePath } from "~/utils/analytics/serverAnalytics";
type CreateContextOptions = {
session: Session | null;
apiKey: string | null;
};
// eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -44,6 +46,7 @@ const noOp = () => {};
export const createInnerTRPCContext = (opts: CreateContextOptions) => {
return {
session: opts.session,
apiKey: opts.apiKey,
prisma,
markAccessControlRun: noOp,
};
@@ -61,8 +64,11 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
// Get the session from the server using the getServerSession wrapper function
const session = await getServerAuthSession({ req, res });
const apiKey = req.headers["x-openpipe-api-key"] as string | null;
return createInnerTRPCContext({
session,
apiKey,
});
};
@@ -76,18 +82,21 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
export type TRPCContext = Awaited<ReturnType<typeof createTRPCContext>>;
const t = initTRPC.context<typeof createTRPCContext>().create({
transformer: superjson,
errorFormatter({ shape, error }) {
return {
...shape,
data: {
...shape.data,
zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
},
};
},
});
const t = initTRPC
.context<typeof createTRPCContext>()
.meta<OpenApiMeta>()
.create({
transformer: superjson,
errorFormatter({ shape, error }) {
return {
...shape,
data: {
...shape.data,
zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
},
};
},
});
/**
* 3. ROUTER & PROCEDURE (THE IMPORTANT BIT)

9
app/src/server/auth.ts
View File

@@ -2,9 +2,16 @@ import { PrismaAdapter } from "@next-auth/prisma-adapter";
import { type GetServerSidePropsContext } from "next";
import { getServerSession, type NextAuthOptions, type DefaultSession } from "next-auth";
import { prisma } from "~/server/db";
import GitHubProvider from "next-auth/providers/github";
import GitHubModule from "next-auth/providers/github";
import { env } from "~/env.mjs";
// The client codegen script doesn't properly read the default export
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const untypedGitHubModule = GitHubModule as unknown as any;
const GitHubProvider: typeof GitHubModule = untypedGitHubModule.default
? untypedGitHubModule.default
: untypedGitHubModule;
/**
* Module augmentation for `next-auth` types. Allows us to add custom properties to the `session`
* object and keep type safety.

65
app/src/server/db.ts
View File

@@ -1,6 +1,61 @@
import { PrismaClient } from "@prisma/client";
import {
type Experiment,
type PromptVariant,
type TestScenario,
type TemplateVariable,
type ScenarioVariantCell,
type ModelResponse,
type Evaluation,
type OutputEvaluation,
type Dataset,
type DatasetEntry,
type Organization,
type OrganizationUser,
type WorldChampEntrant,
type LoggedCall,
type LoggedCallModelResponse,
type LoggedCallTag,
type ApiKey,
type Account,
type Session,
type User,
type VerificationToken,
PrismaClient,
} from "@prisma/client";
import { Kysely, PostgresDialect } from "kysely";
// TODO: Revert to normal import when our tsconfig.json is fixed
// import { Pool } from "pg";
import PGModule from "pg";
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const UntypedPool = PGModule.Pool as any;
const Pool = (UntypedPool.default ? UntypedPool.default : UntypedPool) as typeof PGModule.Pool;
import { env } from "~/env.mjs";
interface DB {
Experiment: Experiment;
PromptVariant: PromptVariant;
TestScenario: TestScenario;
TemplateVariable: TemplateVariable;
ScenarioVariantCell: ScenarioVariantCell;
ModelResponse: ModelResponse;
Evaluation: Evaluation;
OutputEvaluation: OutputEvaluation;
Dataset: Dataset;
DatasetEntry: DatasetEntry;
Organization: Organization;
OrganizationUser: OrganizationUser;
WorldChampEntrant: WorldChampEntrant;
LoggedCall: LoggedCall;
LoggedCallModelResponse: LoggedCallModelResponse;
LoggedCallTag: LoggedCallTag;
ApiKey: ApiKey;
Account: Account;
Session: Session;
User: User;
VerificationToken: VerificationToken;
}
const globalForPrisma = globalThis as unknown as {
prisma: PrismaClient | undefined;
};
@@ -14,4 +69,12 @@ export const prisma =
: ["error"],
});
export const kysely = new Kysely<DB>({
dialect: new PostgresDialect({
pool: new Pool({
connectionString: env.DATABASE_URL,
}),
}),
});
if (env.NODE_ENV !== "production") globalForPrisma.prisma = prisma;

33
app/src/server/scripts/backfillApiKeys.ts Normal file
View File

@@ -0,0 +1,33 @@
import { type Prisma } from "@prisma/client";
import { prisma } from "~/server/db";
import { generateApiKey } from "~/server/utils/generateApiKey";
console.log("backfilling api keys");
const organizations = await prisma.organization.findMany({
include: {
apiKeys: true,
},
});
console.log(`found ${organizations.length} organizations`);
const apiKeysToCreate: Prisma.ApiKeyCreateManyInput[] = [];
for (const org of organizations) {
if (!org.apiKeys.length) {
apiKeysToCreate.push({
name: "Default API Key",
organizationId: org.id,
apiKey: generateApiKey(),
});
}
}
console.log(`creating ${apiKeysToCreate.length} api keys`);
await prisma.apiKey.createMany({
data: apiKeysToCreate,
});
console.log("done");

32
app/src/server/scripts/client-codegen.ts Normal file
View File

@@ -0,0 +1,32 @@
import "dotenv/config";
import { openApiDocument } from "~/pages/api/openapi.json";
import fs from "fs";
import path from "path";
import { execSync } from "child_process";
console.log("Exporting public OpenAPI schema to client-libs/schema.json");
const scriptPath = import.meta.url.replace("file://", "");
const clientLibsPath = path.join(path.dirname(scriptPath), "../../../../client-libs");
const schemaPath = path.join(clientLibsPath, "schema.json");
console.log("Exporting schema");
fs.writeFileSync(schemaPath, JSON.stringify(openApiDocument, null, 2), "utf-8");
console.log("Generating Typescript client");
const tsClientPath = path.join(clientLibsPath, "typescript/codegen");
fs.rmSync(tsClientPath, { recursive: true, force: true });
execSync(
`pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
{
stdio: "inherit",
},
);
console.log("Done!");
process.exit(0);

63
app/src/server/scripts/test-queries.ts Normal file
View File

@@ -0,0 +1,63 @@
import dayjs from "dayjs";
import { prisma } from "../db";
const projectId = "1234";
// Find all calls in the last 24 hours
const responses = await prisma.loggedCall.findMany({
where: {
organizationId: projectId,
startTime: {
gt: dayjs()
.subtract(24 * 3600)
.toDate(),
},
},
include: {
modelResponse: true,
},
orderBy: {
startTime: "desc",
},
});
// Find all calls in the last 24 hours with promptId 'hello-world'
const helloWorld = await prisma.loggedCall.findMany({
where: {
organizationId: projectId,
startTime: {
gt: dayjs()
.subtract(24 * 3600)
.toDate(),
},
tags: {
some: {
name: "promptId",
value: "hello-world",
},
},
},
include: {
modelResponse: true,
},
orderBy: {
startTime: "desc",
},
});
// Total spent on OpenAI in the last month
const totalSpent = await prisma.loggedCallModelResponse.aggregate({
_sum: {
totalCost: true,
},
where: {
originalLoggedCall: {
organizationId: projectId,
},
startTime: {
gt: dayjs()
.subtract(30 * 24 * 3600)
.toDate(),
},
},
});

6
app/src/server/tasks/queryModel.task.ts
View File

@@ -1,10 +1,10 @@
import { type Prisma } from "@prisma/client";
import { type JsonObject } from "type-fest";
import { type JsonValue, type JsonObject } from "type-fest";
import modelProviders from "~/modelProviders/modelProviders";
import { prisma } from "~/server/db";
import { wsConnection } from "~/utils/wsConnection";
import { runEvalsForOutput } from "../utils/evaluations";
import hashPrompt from "../utils/hashPrompt";
import hashObject from "../utils/hashObject";
import defineTask from "./defineTask";
import parsePromptConstructor from "~/promptConstructor/parse";
@@ -99,7 +99,7 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
}
: null;
const inputHash = hashPrompt(prompt);
const inputHash = hashObject(prompt as JsonValue);
let modelResponse = await prisma.modelResponse.create({
data: {

2
app/src/server/tasks/worker.ts
View File

@@ -17,7 +17,7 @@ const taskList = registeredTasks.reduce((acc, task) => {
// Run a worker to execute jobs:
const runner = await run({
connectionString: env.DATABASE_URL,
concurrency: 50,
concurrency: 10,
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
noHandleSignals: false,
pollInterval: 1000,

5
app/src/server/utils/generateApiKey.ts Normal file
View File

@@ -0,0 +1,5 @@
import cryptoRandomString from "crypto-random-string";
const KEY_LENGTH = 42;
export const generateApiKey = () => `opc_${cryptoRandomString({ length: KEY_LENGTH })}`;

4
app/src/server/utils/generateNewCell.ts
View File

@@ -1,7 +1,7 @@
import { Prisma } from "@prisma/client";
import { prisma } from "../db";
import { type JsonObject } from "type-fest";
import hashPrompt from "./hashPrompt";
import hashObject from "./hashObject";
import { omit } from "lodash-es";
import { queueQueryModel } from "../tasks/queryModel.task";
import parsePromptConstructor from "~/promptConstructor/parse";
@@ -57,7 +57,7 @@ export const generateNewCell = async (
return;
}
const inputHash = hashPrompt(parsedConstructFn);
const inputHash = hashObject(parsedConstructFn);
cell = await prisma.scenarioVariantCell.create({
data: {

13
app/src/server/utils/hashPrompt.ts → app/src/server/utils/hashObject.ts
View File

@@ -1,6 +1,5 @@
import crypto from "crypto";
import { type JsonValue } from "type-fest";
import { ParsedPromptConstructor } from "~/promptConstructor/parse";
function sortKeys(obj: JsonValue): JsonValue {
if (typeof obj !== "object" || obj === null) {
@@ -25,9 +24,17 @@ function sortKeys(obj: JsonValue): JsonValue {
return sortedObj;
}
export default function hashPrompt(prompt: ParsedPromptConstructor<any>): string {
export function hashRequest(organizationId: string, reqPayload: JsonValue): string {
const obj = {
organizationId,
reqPayload,
};
return hashObject(obj);
}
export default function hashObject(obj: JsonValue): string {
// Sort object keys recursively
const sortedObj = sortKeys(prompt as unknown as JsonValue);
const sortedObj = sortKeys(obj);
// Convert to JSON and hash it
const str = JSON.stringify(sortedObj);

11
app/src/server/utils/openai.ts
View File

@@ -1,6 +1,13 @@
import { env } from "~/env.mjs";
import OpenAI from "openai";
import { default as OriginalOpenAI } from "openai";
// import { OpenAI } from "openpipe";
const openAIConfig = { apiKey: env.OPENAI_API_KEY ?? "dummy-key" };
// Set a dummy key so it doesn't fail at build time
export const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY ?? "dummy-key" });
// export const openai = env.OPENPIPE_API_KEY
// ? new OpenAI.OpenAI(openAIConfig)
// : new OriginalOpenAI(openAIConfig);
export const openai = new OriginalOpenAI(openAIConfig);

9
app/src/server/utils/userOrg.ts
View File

@@ -1,4 +1,5 @@
import { prisma } from "~/server/db";
import { generateApiKey } from "./generateApiKey";
export default async function userOrg(userId: string) {
return await prisma.organization.upsert({
@@ -14,6 +15,14 @@ export default async function userOrg(userId: string) {
role: "ADMIN",
},
},
apiKeys: {
create: [
{
name: "Default API Key",
apiKey: generateApiKey(),
},
],
},
},
});
}

7
app/src/state/store.ts
View File

@@ -14,6 +14,8 @@ export type State = {
api: APIClient | null;
setApi: (api: APIClient) => void;
sharedVariantEditor: SharedVariantEditorSlice;
selectedOrgId: string | null;
setSelectedOrgId: (orgId: string) => void;
};
export type SliceCreator<T> = StateCreator<State, [["zustand/immer", never]], [], T>;
@@ -39,6 +41,11 @@ const useBaseStore = create<State, [["zustand/immer", never]]>(
state.drawerOpen = false;
}),
sharedVariantEditor: createVariantEditorSlice(set, get, ...rest),
selectedOrgId: null,
setSelectedOrgId: (orgId: string) =>
set((state) => {
state.selectedOrgId = orgId;
}),
})),
);

11
app/src/theme/ChakraThemeProvider.tsx
View File

@@ -1,6 +1,5 @@
import { extendTheme } from "@chakra-ui/react";
import { extendTheme, defineStyleConfig, ChakraProvider } from "@chakra-ui/react";
import "@fontsource/inconsolata";
import { ChakraProvider } from "@chakra-ui/react";
import { modalAnatomy } from "@chakra-ui/anatomy";
import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system";
@@ -18,6 +17,13 @@ const modalTheme = defineMultiStyleConfig({
}),
});
const Divider = defineStyleConfig({
baseStyle: {
borderColor: "gray.300",
backgroundColor: "gray.300",
},
});
const theme = extendTheme({
styles: {
global: (props: { colorMode: "dark" | "light" }) => ({
@@ -53,6 +59,7 @@ const theme = extendTheme({
},
},
Modal: modalTheme,
Divider,
},
});

62
app/src/utils/accessControl.ts
View File

@@ -16,6 +16,68 @@ export const requireNothing = (ctx: TRPCContext) => {
ctx.markAccessControlRun();
};
export const requireIsOrgAdmin = async (organizationId: string, ctx: TRPCContext) => {
const userId = ctx.session?.user.id;
if (!userId) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const isAdmin = await prisma.organizationUser.findFirst({
where: {
userId,
organizationId,
role: "ADMIN",
},
});
if (!isAdmin) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
ctx.markAccessControlRun();
};
export const requireCanViewOrganization = async (organizationId: string, ctx: TRPCContext) => {
const userId = ctx.session?.user.id;
if (!userId) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const canView = await prisma.organizationUser.findFirst({
where: {
userId,
organizationId,
},
});
if (!canView) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
ctx.markAccessControlRun();
};
export const requireCanModifyOrganization = async (organizationId: string, ctx: TRPCContext) => {
const userId = ctx.session?.user.id;
if (!userId) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const canModify = await prisma.organizationUser.findFirst({
where: {
userId,
organizationId,
role: { in: [OrganizationUserRole.ADMIN, OrganizationUserRole.MEMBER] },
},
});
if (!canModify) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
ctx.markAccessControlRun();
};
export const requireCanViewDataset = async (datasetId: string, ctx: TRPCContext) => {
const dataset = await prisma.dataset.findFirst({
where: {

2
app/src/utils/dayjs.ts
View File

@@ -1,9 +1,11 @@
import dayjs from "dayjs";
import duration from "dayjs/plugin/duration";
import relativeTime from "dayjs/plugin/relativeTime";
import timezone from "dayjs/plugin/timezone";
dayjs.extend(duration);
dayjs.extend(relativeTime);
dayjs.extend(timezone);
export const formatTimePast = (date: Date) =>
dayjs.duration(dayjs(date).diff(dayjs())).humanize(true);

22
app/src/utils/hooks.ts
View File

@@ -2,6 +2,15 @@ import { useRouter } from "next/router";
import { type RefObject, useCallback, useEffect, useRef, useState } from "react";
import { api } from "~/utils/api";
import { NumberParam, useQueryParam, withDefault } from "use-query-params";
import { useAppStore } from "~/state/store";
export const useExperiments = () => {
const selectedOrgId = useAppStore((state) => state.selectedOrgId);
return api.experiments.list.useQuery(
{ organizationId: selectedOrgId ?? "" },
{ enabled: !!selectedOrgId },
);
};
export const useExperiment = () => {
const router = useRouter();
@@ -17,6 +26,14 @@ export const useExperimentAccess = () => {
return useExperiment().data?.access ?? { canView: false, canModify: false };
};
export const useDatasets = () => {
const selectedOrgId = useAppStore((state) => state.selectedOrgId);
return api.datasets.list.useQuery(
{ organizationId: selectedOrgId ?? "" },
{ enabled: !!selectedOrgId },
);
};
export const useDataset = () => {
const router = useRouter();
const dataset = api.datasets.get.useQuery(
@@ -132,3 +149,8 @@ export const useScenario = (scenarioId: string) => {
};
export const useVisibleScenarioIds = () => useScenarios().data?.scenarios.map((s) => s.id) ?? [];
export const useSelectedOrg = () => {
const selectedOrgId = useAppStore((state) => state.selectedOrgId);
return api.organizations.get.useQuery({ id: selectedOrgId ?? "" }, { enabled: !!selectedOrgId });
};

1
app/src/utils/useSocket.ts
View File

@@ -11,7 +11,6 @@ export default function useSocket<T>(channel?: string | null) {
useEffect(() => {
if (!channel) return;
console.log("connecting to channel", channel);
// Create websocket connection
socketRef.current = io(url);