alihan/OpenPipe-llm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

can add scenarios and it mostly works

Browse Source
This commit is contained in:
Kyle Corbitt
2023-06-23 20:00:46 -07:00
parent c497b74208
commit 8534477236
13 changed files with 278 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
package.json
View File

@@ -16,6 +16,8 @@
"@emotion/react": "^11.11.1",
"@emotion/server": "^11.11.0",
"@emotion/styled": "^11.11.0",
"@fontsource/poppins": "^5.0.3",
"@fontsource/roboto": "^5.0.3",
"@monaco-editor/react": "^4.5.1",
"@next-auth/prisma-adapter": "^1.0.5",
"@prisma/client": "^4.14.0",
@@ -34,6 +36,8 @@
"next-auth": "^4.22.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-icons": "^4.10.1",
"react-textarea-autosize": "^8.5.0",
"superjson": "1.12.2",
"tsx": "^3.12.7",
"zod": "^3.21.4"

77
pnpm-lock.yaml generated
View File

@@ -20,6 +20,12 @@ dependencies:
'@emotion/styled':
specifier: ^11.11.0
version: 11.11.0(@emotion/react@11.11.1)(@types/react@18.2.6)(react@18.2.0)
'@fontsource/poppins':
specifier: ^5.0.3
version: 5.0.3
'@fontsource/roboto':
specifier: ^5.0.3
version: 5.0.3
'@monaco-editor/react':
specifier: ^4.5.1
version: 4.5.1(monaco-editor@0.39.0)(react-dom@18.2.0)(react@18.2.0)
@@ -74,6 +80,12 @@ dependencies:
react-dom:
specifier: 18.2.0
version: 18.2.0(react@18.2.0)
react-icons:
specifier: ^4.10.1
version: 4.10.1(react@18.2.0)
react-textarea-autosize:
specifier: ^8.5.0
version: 8.5.0(@types/react@18.2.6)(react@18.2.0)
superjson:
specifier: 1.12.2
version: 1.12.2
@@ -1670,6 +1682,14 @@ packages:
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
dev: true
/@fontsource/poppins@5.0.3:
resolution: {integrity: sha512-5lL2vmvmh4zbknTh1nVH9pTWyhYqAlYMHIAnkNhqo5qwMZGlr+coM1dtMwiQHRBgmHAl3ZvJ35Bj0s8cpmXZbg==}
dev: false
/@fontsource/roboto@5.0.3:
resolution: {integrity: sha512-jbZDFwEFARDlo8TqG7th/xjhuq87GYfFpFb+uxuy+0Ng1bhRVgBRWlLj8+WIKhCTOr+h4QXbjpybLWFLUirOwQ==}
dev: false
/@humanwhocodes/config-array@0.11.10:
resolution: {integrity: sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ==}
engines: {node: '>=10.10.0'}
@@ -4135,6 +4155,14 @@ packages:
use-sidecar: 1.1.2(@types/react@18.2.6)(react@18.2.0)
dev: false
/react-icons@4.10.1(react@18.2.0):
resolution: {integrity: sha512-/ngzDP/77tlCfqthiiGNZeYFACw85fUjZtLbedmJ5DTlNDIwETxhwBzdOJ21zj4iJdvc0J3y7yOsX3PpxAJzrw==}
peerDependencies:
react: '*'
dependencies:
react: 18.2.0
dev: false
/react-is@16.13.1:
resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==}
@@ -4198,6 +4226,20 @@ packages:
tslib: 2.5.3
dev: false
/react-textarea-autosize@8.5.0(@types/react@18.2.6)(react@18.2.0):
resolution: {integrity: sha512-cp488su3U9RygmHmGpJp0KEt0i/+57KCK33XVPH+50swVRBhIZYh0fGduz2YLKXwl9vSKBZ9HUXcg9PQXUXqIw==}
engines: {node: '>=10'}
peerDependencies:
react: ^16.8.0 || ^17.0.0 || ^18.0.0
dependencies:
'@babel/runtime': 7.22.5
react: 18.2.0
use-composed-ref: 1.3.0(react@18.2.0)
use-latest: 1.2.1(@types/react@18.2.6)(react@18.2.0)
transitivePeerDependencies:
- '@types/react'
dev: false
/react@18.2.0:
resolution: {integrity: sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==}
engines: {node: '>=0.10.0'}
@@ -4658,6 +4700,41 @@ packages:
tslib: 2.5.3
dev: false
/use-composed-ref@1.3.0(react@18.2.0):
resolution: {integrity: sha512-GLMG0Jc/jiKov/3Ulid1wbv3r54K9HlMW29IWcDFPEqFkSO2nS0MuefWgMJpeHQ9YJeXDL3ZUF+P3jdXlZX/cQ==}
peerDependencies:
react: ^16.8.0 || ^17.0.0 || ^18.0.0
dependencies:
react: 18.2.0
dev: false
/use-isomorphic-layout-effect@1.1.2(@types/react@18.2.6)(react@18.2.0):
resolution: {integrity: sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==}
peerDependencies:
'@types/react': '*'
react: ^16.8.0 || ^17.0.0 || ^18.0.0
peerDependenciesMeta:
'@types/react':
optional: true
dependencies:
'@types/react': 18.2.6
react: 18.2.0
dev: false
/use-latest@1.2.1(@types/react@18.2.6)(react@18.2.0):
resolution: {integrity: sha512-xA+AVm/Wlg3e2P/JiItTziwS7FK92LWrDB0p+hgXloIMuVCeJJ8v6f0eeHyPZaJrM+usM1FkFfbNCrJGs8A/zw==}
peerDependencies:
'@types/react': '*'
react: ^16.8.0 || ^17.0.0 || ^18.0.0
peerDependenciesMeta:
'@types/react':
optional: true
dependencies:
'@types/react': 18.2.6
react: 18.2.0
use-isomorphic-layout-effect: 1.1.2(@types/react@18.2.6)(react@18.2.0)
dev: false
/use-sidecar@1.1.2(@types/react@18.2.6)(react@18.2.0):
resolution: {integrity: sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==}
engines: {node: '>=10'}

5
src/components/OutputsTable/EditableVariantLabel.tsx
View File

@@ -26,7 +26,12 @@ export default function EditableVariantLabel(props: { variant: PromptVariant })
ref={labelRef}
contentEditable
suppressContentEditableWarning
borderWidth={1}
borderColor="transparent"
_hover={{ borderColor: "gray.300" }}
_focus={{ borderColor: "blue.500", outline: "none" }}
onBlur={onBlur}
py={2}
>
{props.variant.label}
</Heading>

34
src/components/OutputsTable/NewScenarioButton.tsx Normal file
View File

@@ -0,0 +1,34 @@
import { Button } from "@chakra-ui/react";
import { BsPlus } from "react-icons/bs";
import { api } from "~/utils/api";
import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
export default function NewScenarioButton() {
const experiment = useExperiment();
const mutation = api.scenarios.create.useMutation();
const utils = api.useContext();
const [onClick] = useHandledAsyncCallback(async () => {
await mutation.mutateAsync({
experimentId: experiment.data!.id,
});
await utils.scenarios.list.invalidate();
}, [mutation]);
return (
<Button
w="100%"
borderRadius={0}
alignItems="center"
justifyContent="flex-start"
fontWeight="normal"
bgColor="transparent"
_hover={{ bgColor: "gray.100" }}
px={2}
onClick={onClick}
>
<BsPlus size={24} />
New Scenario
</Button>
);
}

29
src/components/OutputsTable/OutputCell.tsx
View File

@@ -1,6 +1,7 @@
import { api } from "~/utils/api";
import { PromptVariant, Scenario } from "./types";
import { Center } from "@chakra-ui/react";
import { Center, Text } from "@chakra-ui/react";
import { useExperiment } from "~/utils/hooks";
export default function OutputCell({
scenario,
@@ -9,12 +10,32 @@ export default function OutputCell({
scenario: Scenario;
variant: PromptVariant;
}) {
const output = api.outputs.get.useQuery({
const experiment = useExperiment();
const experimentVariables = experiment.data?.TemplateVariable.map((v) => v.label) ?? [];
const scenarioVariables = scenario.variableValues as Record<string, string>;
const templateHasVariables =
experimentVariables.length === 0 ||
experimentVariables.some((v) => scenarioVariables[v] !== undefined);
const output = api.outputs.get.useQuery(
{
scenarioId: scenario.id,
variantId: variant.id,
});
},
{ enabled: templateHasVariables }
);
if (!output.data) return null;
if (!templateHasVariables)
return (
<Center h="100%">
<Text color="gray.500">Add a scenario variable to see output</Text>
</Center>
);
if (output.isLoading) return <Center h="100%">Loading...</Center>;
if (!output.data) return <Center h="100%">No output</Center>;
return (
<Center h="100%">

49
src/components/OutputsTable/ScenarioHeader.tsx
View File

@@ -1,9 +1,11 @@
import { api } from "~/utils/api";
import { isEqual } from "lodash";
import { PromptVariant, Scenario } from "./types";
import { type Scenario } from "./types";
import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
import { useState } from "react";
import { Badge, Button, Flex, HStack, Stack, Textarea } from "@chakra-ui/react";
import ResizeTextarea from "react-textarea-autosize";
import { Box, Button, Flex, HStack, Stack, Textarea } from "@chakra-ui/react";
export default function ScenarioHeader({ scenario }: { scenario: Scenario }) {
const savedValues = scenario.variableValues as Record<string, string>;
@@ -30,34 +32,53 @@ export default function ScenarioHeader({ scenario }: { scenario: Scenario }) {
return (
<Stack>
{variableLabels.map((key) => {
const value = values[key] ?? "";
const layoutDirection = value.length > 20 ? "column" : "row";
return (
<Flex key={key}>
<Badge>{key}</Badge>
<Textarea
<Flex
key={key}
value={values[key] ?? ""}
direction={layoutDirection}
alignItems={layoutDirection === "column" ? "flex-start" : "center"}
flexWrap="wrap"
>
<Box bgColor="blue.100" color="blue.600" px={2} fontSize="xs" fontWeight="bold">
{key}
</Box>
<Textarea
borderRadius={0}
px={2}
py={1}
placeholder="empty"
value={value}
onChange={(e) => {
setValues((prev) => ({ ...prev, [key]: e.target.value }));
}}
rows={1}
// TODO: autosize
maxRows={20}
resize="none"
overflow="hidden"
minRows={1}
minH="unset"
as={ResizeTextarea}
flex={layoutDirection === "row" ? 1 : undefined}
borderColor={hasChanged ? "blue.300" : "transparent"}
_hover={{ borderColor: "gray.300" }}
_focus={{ borderColor: "blue.500", outline: "none" }}
/>
</Flex>
);
})}
{hasChanged && (
<HStack spacing={4}>
<HStack justify="right">
<Button
size="xs"
onClick={() => {
size="sm"
borderRadius={0}
onMouseDown={() => {
setValues(savedValues);
}}
color="gray"
colorScheme="gray"
>
Reset
</Button>
<Button size="xs" onClick={onSave}>
<Button size="sm" borderRadius={0} onMouseDown={onSave} colorScheme="blue">
Save
</Button>
</HStack>

7
src/components/OutputsTable/VariantConfigEditor.tsx
View File

@@ -108,19 +108,20 @@ export default function VariantConfigEditor(props: {
<Box w="100%" pos="relative">
<div id={editorId} style={{ height: "300px", width: "100%" }}></div>
{isChanged && (
<HStack pos="absolute" bottom={0} right={0} spacing={4}>
<HStack pos="absolute" bottom={0} right={0}>
<Button
colorScheme="gray"
size="xs"
size="sm"
onClick={() => {
editorRef.current?.setValue(props.savedConfig);
checkForChanges();
}}
borderRadius={0}
>
Reset
</Button>
<Tooltip label={`${modifierKey} + Enter`}>
<Button size="xs" onClick={onSave} colorScheme="blue">
<Button size="sm" onClick={onSave} colorScheme="blue" borderRadius={0}>
Save
</Button>
</Tooltip>

10
src/components/OutputsTable/VariantHeader.tsx
View File

@@ -21,17 +21,15 @@ export default function VariantHeader({ variant }: { variant: PromptVariant }) {
title: "Invalid JSON",
description: "Please fix the JSON before saving.",
status: "error",
duration: 5000,
position: "top",
});
return;
}
if (parsedConfig === null) {
notifications.show({
toast({
title: "Invalid JSON",
message: "Please fix the JSON before saving.",
color: "red",
description: "Please fix the JSON before saving.",
status: "error",
});
return;
}
@@ -43,7 +41,7 @@ export default function VariantHeader({ variant }: { variant: PromptVariant }) {
await utils.promptVariants.list.invalidate();
},
[variant.id, replaceWithConfig, utils.promptVariants.list]
[variant.id, replaceWithConfig, utils.promptVariants.list, toast]
);
return (

50
src/components/OutputsTable/index.tsx
View File

@@ -1,14 +1,11 @@
import { useMemo } from "react";
import { RouterOutputs, api } from "~/utils/api";
import { type PromptVariant } from "./types";
import VariantHeader from "./VariantHeader";
import OutputCell from "./OutputCell";
import ScenarioHeader from "./ScenarioHeader";
import React from "react";
import { Box, Heading } from "@chakra-ui/react";
const cellPaddingX = 4;
const cellPaddingY = 2;
import { Box, Grid, GridItem, Heading } from "@chakra-ui/react";
import NewScenarioButton from "./NewScenarioButton";
export default function OutputsTable({ experimentId }: { experimentId: string | undefined }) {
const variants = api.promptVariants.list.useQuery(
@@ -24,37 +21,48 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
if (!variants.data || !scenarios.data) return null;
return (
<Box p={4}>
<div
style={{
display: "grid",
gridTemplateColumns: `200px repeat(${variants.data.length}, minmax(300px, 1fr))`,
overflowX: "auto",
<Grid
p={4}
display="grid"
gridTemplateColumns={`200px repeat(${variants.data.length}, minmax(300px, 1fr))`}
overflowX="auto"
sx={{
"> *": {
borderColor: "gray.300",
borderBottomWidth: 1,
paddingX: 4,
paddingY: 2,
},
"> *:last-child": {
borderRightWidth: 0,
},
}}
>
<Box px={cellPaddingX} py={cellPaddingY} display="flex" sx={{}}>
<GridItem display="flex" alignItems="flex-end">
<Heading size="md" fontWeight="bold">
Scenario
</Heading>
</Box>
</GridItem>
{variants.data.map((variant) => (
<Box key={variant.uiId} px={cellPaddingX} py={cellPaddingY}>
<GridItem key={variant.uiId}>
<VariantHeader key={variant.uiId} variant={variant} />
</Box>
</GridItem>
))}
{scenarios.data.map((scenario) => (
<React.Fragment key={scenario.uiId}>
<Box px={cellPaddingX} py={cellPaddingY}>
<GridItem>
<ScenarioHeader scenario={scenario} />
</Box>
</GridItem>
{variants.data.map((variant) => (
<Box key={variant.id} px={cellPaddingX} py={cellPaddingY}>
<GridItem key={variant.id}>
<OutputCell key={variant.id} scenario={scenario} variant={variant} />
</Box>
</GridItem>
))}
</React.Fragment>
))}
</div>
</Box>
<GridItem borderBottomWidth={0} w="100%" colSpan={variants.data.length + 1} px={0} py={0}>
<NewScenarioButton />
</GridItem>
</Grid>
);
}

3
src/pages/_app.tsx
View File

@@ -3,6 +3,7 @@ import { SessionProvider } from "next-auth/react";
import { type AppType } from "next/app";
import { api } from "~/utils/api";
import { ChakraProvider } from "@chakra-ui/react";
import theme from "~/utils/theme";
const MyApp: AppType<{ session: Session | null }> = ({
Component,
@@ -10,7 +11,7 @@ const MyApp: AppType<{ session: Session | null }> = ({
}) => {
return (
<SessionProvider session={session}>
<ChakraProvider>
<ChakraProvider theme={theme}>
<Component {...pageProps} />
</ChakraProvider>
</SessionProvider>

2
src/pages/experiments/[id].tsx
View File

@@ -24,7 +24,7 @@ export default function Experiment() {
return (
<AppNav title={experiment.data?.label}>
<Box sx={{ minHeight: "100vh" }}>
<Box minH="100vh" mb={50}>
<OutputsTable experimentId={router.query.id as string | undefined} />
</Box>
</AppNav>

28
src/server/api/routers/scenarios.router.ts
View File

@@ -15,6 +15,34 @@ export const scenariosRouter = createTRPCRouter({
});
}),
create: publicProcedure
.input(
z.object({
experimentId: z.string(),
})
)
.mutation(async ({ input }) => {
const maxSortIndex =
(
await prisma.testScenario.aggregate({
where: {
experimentId: input.experimentId,
},
_max: {
sortIndex: true,
},
})
)._max.sortIndex ?? 0;
const newScenario = await prisma.testScenario.create({
data: {
experimentId: input.experimentId,
sortIndex: maxSortIndex + 1,
variableValues: {},
},
});
}),
replaceWithValues: publicProcedure
.input(
z.object({

12
src/utils/theme.ts Normal file
View File

@@ -0,0 +1,12 @@
import { extendTheme } from "@chakra-ui/react";
import "@fontsource/poppins";
import "@fontsource/roboto";
const theme = extendTheme({
fonts: {
heading: "Poppins, sans-serif",
body: "Roboto, sans-serif",
},
});
export default theme;