alihan/OpenPipe-llm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use javascript functions for prompt completions instead of templated json

Browse Source
This commit is contained in:
Kyle Corbitt
2023-07-13 18:01:07 -07:00
parent 1776da937a
commit 4770ea34a8
33 changed files with 1654 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/components/OutputsTable/OutputCell/OutputCell.tsx
View File

@@ -10,8 +10,6 @@ import { type ChatCompletion } from "openai/resources/chat";
import { generateChannel } from "~/utils/generateChannel";
import { isObject } from "lodash";
import useSocket from "~/utils/useSocket";
import { type JSONSerializable } from "~/server/types";
import { getModelName } from "~/server/utils/getModelName";
import { OutputStats } from "./OutputStats";
import { ErrorHandler } from "./ErrorHandler";
@@ -36,10 +34,12 @@ export default function OutputCell({
if (!templateHasVariables) disabledReason = "Add a value to the scenario variables to see output";
if (variant.config === null || Object.keys(variant.config).length === 0)
disabledReason = "Save your prompt variant to see output";
// if (variant.config === null || Object.keys(variant.config).length === 0)
// disabledReason = "Save your prompt variant to see output";
const model = getModelName(variant.config as JSONSerializable);
// const model = getModelName(variant.config as JSONSerializable);
// TODO: Temporarily hardcoding this while we get other stuff working
const model = "gpt-3.5-turbo";
const outputMutation = api.outputs.get.useMutation();

2
src/components/OutputsTable/ScenarioEditor.tsx
View File

@@ -107,7 +107,7 @@ export default function ScenarioEditor({
cursor: "pointer",
}}
>
<Icon as={hidingInProgress ? Spinner :BsX} boxSize={6} />
<Icon as={hidingInProgress ? Spinner : BsX} boxSize={6} />
</Button>
</Tooltip>
<Icon

6
src/components/OutputsTable/SettingsDrawer.tsx
View File

@@ -8,13 +8,13 @@ import {
Heading,
Stack,
} from "@chakra-ui/react";
import { useStore } from "~/utils/store";
import EditScenarioVars from "./EditScenarioVars";
import EditEvaluations from "./EditEvaluations";
import { useAppStore } from "~/state/store";
export default function SettingsDrawer() {
const isOpen = useStore((state) => state.drawerOpen);
const closeDrawer = useStore((state) => state.closeDrawer);
const isOpen = useAppStore((state) => state.drawerOpen);
const closeDrawer = useAppStore((state) => state.closeDrawer);
return (
<Drawer isOpen={isOpen} placement="right" onClose={closeDrawer} size="md">

106
src/components/OutputsTable/VariantConfigEditor.tsx → src/components/OutputsTable/VariantEditor.tsx
View File

@@ -1,66 +1,67 @@
import { Box, Button, HStack, Tooltip, useToast } from "@chakra-ui/react";
import { useMonaco } from "@monaco-editor/react";
import { useRef, useEffect, useState, useCallback, useMemo } from "react";
import { useRef, useEffect, useState, useCallback } from "react";
import { useHandledAsyncCallback, useModifierKeyLabel } from "~/utils/hooks";
import { type PromptVariant } from "./types";
import { type JSONSerializable } from "~/server/types";
import { api } from "~/utils/api";
import openaiSchema from "~/codegen/openai.schema.json";
let isEditorConfigured = false;
import { useAppStore } from "~/state/store";
// import openAITypes from "~/codegen/openai.types.ts.txt";
export default function VariantConfigEditor(props: { variant: PromptVariant }) {
const monaco = useMonaco();
const monaco = useAppStore.use.variantEditor.monaco();
const editorRef = useRef<ReturnType<NonNullable<typeof monaco>["editor"]["create"]> | null>(null);
const [editorId] = useState(() => `editor_${Math.random().toString(36).substring(7)}`);
const [isChanged, setIsChanged] = useState(false);
const savedConfig = useMemo(
() => JSON.stringify(props.variant.config, null, 2),
[props.variant.config],
);
const savedConfigRef = useRef(savedConfig);
const lastSavedFn = props.variant.constructFn;
const modifierKey = useModifierKeyLabel();
const checkForChanges = useCallback(() => {
if (!editorRef.current) return;
const currentConfig = editorRef.current.getValue();
setIsChanged(currentConfig !== savedConfigRef.current);
}, []);
setIsChanged(currentConfig !== lastSavedFn);
}, [lastSavedFn]);
const replaceWithConfig = api.promptVariants.replaceWithConfig.useMutation();
const replaceVariant = api.promptVariants.replaceVariant.useMutation();
const utils = api.useContext();
const toast = useToast();
const [onSave] = useHandledAsyncCallback(async () => {
const currentConfig = editorRef.current?.getValue();
if (!currentConfig) return;
const currentFn = editorRef.current?.getValue();
if (!currentFn) return;
let parsedConfig: JSONSerializable;
try {
parsedConfig = JSON.parse(currentConfig) as JSONSerializable;
} catch (e) {
// Check if the editor has any typescript errors
const model = editorRef.current?.getModel();
if (!model) return;
const markers = monaco?.editor.getModelMarkers({ resource: model.uri });
const hasErrors = markers?.some((m) => m.severity === monaco?.MarkerSeverity.Error);
if (hasErrors) {
toast({
title: "Invalid JSON",
description: "Please fix the JSON before saving.",
title: "Invalid TypeScript",
description: "Please fix the TypeScript errors before saving.",
status: "error",
});
return;
}
if (parsedConfig === null) {
// Make sure the user defined the prompt with the string "prompt\w*=" somewhere
const promptRegex = /prompt\s*=/;
if (!promptRegex.test(currentFn)) {
console.log("no prompt");
console.log(currentFn);
toast({
title: "Invalid JSON",
description: "Please fix the JSON before saving.",
title: "Missing prompt",
description: "Please define the prompt (eg. `prompt = { ...`).",
status: "error",
});
return;
}
await replaceWithConfig.mutateAsync({
await replaceVariant.mutateAsync({
id: props.variant.id,
config: currentConfig,
constructFn: currentFn,
});
await utils.promptVariants.list.invalidate();
@@ -70,37 +71,11 @@ export default function VariantConfigEditor(props: { variant: PromptVariant }) {
useEffect(() => {
if (monaco) {
if (!isEditorConfigured) {
monaco.editor.defineTheme("customTheme", {
base: "vs",
inherit: true,
rules: [],
colors: {
"editor.background": "#fafafa",
},
});
monaco.languages.json.jsonDefaults.setDiagnosticsOptions({
validate: true,
schemas: [
{
uri: "https://api.openai.com/v1",
fileMatch: ["*"],
schema: {
$schema: "http://json-schema.org/draft-07/schema#",
$ref: "#/components/schemas/CreateChatCompletionRequest",
components: openaiSchema.components,
},
},
],
});
isEditorConfigured = true;
}
const container = document.getElementById(editorId) as HTMLElement;
editorRef.current = monaco.editor.create(container, {
value: savedConfig,
language: "json",
value: lastSavedFn,
language: "typescript",
theme: "customTheme",
lineNumbers: "off",
minimap: { enabled: false },
@@ -114,6 +89,7 @@ export default function VariantConfigEditor(props: { variant: PromptVariant }) {
},
wordWrapBreakAfterCharacters: "",
wordWrapBreakBeforeCharacters: "",
quickSuggestions: true,
});
editorRef.current.onDidFocusEditorText(() => {
@@ -141,17 +117,17 @@ export default function VariantConfigEditor(props: { variant: PromptVariant }) {
/* eslint-disable-next-line react-hooks/exhaustive-deps */
}, [monaco, editorId]);
useEffect(() => {
const savedConfigChanged = savedConfigRef.current !== savedConfig;
// useEffect(() => {
// const savedConfigChanged = lastSavedFn !== savedConfig;
savedConfigRef.current = savedConfig;
// lastSavedFn = savedConfig;
if (savedConfigChanged && editorRef.current?.getValue() !== savedConfig) {
editorRef.current?.setValue(savedConfig);
}
// if (savedConfigChanged && editorRef.current?.getValue() !== savedConfig) {
// editorRef.current?.setValue(savedConfig);
// }
checkForChanges();
}, [savedConfig, checkForChanges]);
// checkForChanges();
// }, [savedConfig, checkForChanges]);
return (
<Box w="100%" pos="relative">
@@ -162,7 +138,7 @@ export default function VariantConfigEditor(props: { variant: PromptVariant }) {
colorScheme="gray"
size="sm"
onClick={() => {
editorRef.current?.setValue(savedConfig);
editorRef.current?.setValue(lastSavedFn);
checkForChanges();
}}
>

8
src/components/OutputsTable/index.tsx
View File

@@ -3,12 +3,12 @@ import { api } from "~/utils/api";
import NewScenarioButton from "./NewScenarioButton";
import NewVariantButton from "./NewVariantButton";
import ScenarioRow from "./ScenarioRow";
import VariantConfigEditor from "./VariantConfigEditor";
import VariantConfigEditor from "./VariantEditor";
import VariantHeader from "./VariantHeader";
import { cellPadding } from "../constants";
import { BsPencil } from "react-icons/bs";
import { useStore } from "~/utils/store";
import VariantStats from "./VariantStats";
import { useAppStore } from "~/state/store";
const stickyHeaderStyle: SystemStyleObject = {
position: "sticky",
@@ -22,7 +22,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
{ experimentId: experimentId as string },
{ enabled: !!experimentId },
);
const openDrawer = useStore((s) => s.openDrawer);
const openDrawer = useAppStore((s) => s.openDrawer);
const scenarios = api.scenarios.list.useQuery(
{ experimentId: experimentId as string },
@@ -57,7 +57,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
py={cellPadding.y}
// TODO: This is a hack to get the sticky header to work. It's not ideal because it's not responsive to the height of the header,
// so if the header height changes, this will need to be updated.
sx={{...stickyHeaderStyle, top: "-337px"}}
sx={{ ...stickyHeaderStyle, top: "-337px" }}
>
<HStack w="100%">
<Heading size="xs" fontWeight="bold" flex={1}>