diff --git a/README.md b/README.md index f52d83d..5833c34 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,15 @@ -(Note: this repository practices [Readme Driven Development](https://tom.preston-werner.com/2010/08/23/readme-driven-development.html). This README documents what we **want** our UX to be, not what it is right now. We'll remove this note once the repository is ready for outside testing. Thanks for your patience! ๐Ÿ™) +# ๐Ÿงช Prompt Lab -# ๐Ÿ›  Prompt Lab +Prompt Lab is a flexible playground for comparing and optimizing LLM prompts. It lets you quickly generate, test and compare candidate prompts with realistic sample data. -Prompt Lab is a powerful toolset to optimize your LLM prompts. It lets you quickly generate, test and compare candidate prompts with realistic sample data. +[] + +Currently there's a public playground available at [https://promptlab.corbt.com/](https://promptlab.corbt.com/), but the recommended approach is to but the recommended approach is to [run locally](#running-locally). ## High-Level Features **Configure Multiple Prompts** -Set up multiple prompt configurations and compare their output side-by-side. Each configuration can use a different model, template string, and temperature/top_p. +Set up multiple prompt configurations and compare their output side-by-side. Each configuration can be configured independently. **Visualize Responses** Inspect prompt completions side-by-side. @@ -15,34 +17,29 @@ Inspect prompt completions side-by-side. **Test Many Inputs** Prompt Lab lets you *template* a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broader coverage of your problem space than you'd get with manual testing. -**Automatically Evaluate Prompt Quality** -1. If you're extracting structured data, Prompt Lab lets you define the expected output for each input sample and will automatically score each prompt variant for accuracy. -2. If you're generating free-form text, Prompt Lab lets you either (a) manually review outputs side-by-side to compare quality, or (b) configure a GPT-4 based evaluator to compare and score your completions automatically. ๐Ÿงžโ€โ™‚๏ธ +**๐Ÿช„ Auto-generate Test Scenarios** +Prompt Lab includes a tool to generate new test scenarios based on your existing prompts and scenarios. Just click "Autogenerate Scenario" to try it out! -**๐Ÿช„ Auto-generate Prompts and Data** -Prompt Lab includes easy tools to generate both new prompt variants and new test inputs. It can even use the test inputs with incorrect results to guide the variant generation more intelligently! +**Prompt Validation and Typeahead** +We use OpenAI's OpenAPI spec to automatically provide typeahead and validate prompts. + +[] + +**Function Call Support** +Natively supports [OpenAI function calls](https://openai.com/blog/function-calling-and-other-api-updates) on supported models. + +[] ## Supported Models Prompt Lab currently supports GPT-3.5 and GPT-4. Wider model support is planned. -## More Features +## Running Locally - - [x] Fully open source - - [x] GPT-4 and 3.5 function call support - -## Future Work - - - [ ] Hosted version for easier onboarding - - [ ] Additional model support - - [ ] Automatic context augmentation and fine-tuning (for feature extraction and classification use-cases) - -# Running Locally - -We'll have a hosted version of Prompt Lab soon to make onboarding easier but for now you can run it locally. - -1. Install [NodeJS 20](https://nodejs.org/en/download/current) (earlier versions will likely work but aren't tested) -2. Install `pnpm`: `npm i -g pnpm` -3. Clone this repository: `git clone https://github.com/prompt-lab/prompt-lab` -4. Install the dependencies: `cd prompt-lab && pnpm install` -5. Start the app: `pnpm start` -6. Navigate to [http://localhost:3000](http://localhost:3000) \ No newline at end of file +1. Install [Postgresql](https://www.postgresql.org/download/). +2. Install [NodeJS 20](https://nodejs.org/en/download/current) (earlier versions will very likely work but aren't tested). +3. Install `pnpm`: `npm i -g pnpm` +4. Clone this repository: `git clone https://github.com/prompt-lab/prompt-lab` +5. Install the dependencies: `cd prompt-lab && pnpm install` +6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`. +7. Start the app: `pnpm start` +8. Navigate to [http://localhost:3000](http://localhost:3000) \ No newline at end of file diff --git a/prisma/seed.ts b/prisma/seed.ts index 4b487bf..36dfeda 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -30,7 +30,7 @@ await prisma.promptVariant.deleteMany({ }, }); -const resp = await prisma.promptVariant.createMany({ +await prisma.promptVariant.createMany({ data: [ { experimentId, @@ -87,14 +87,14 @@ await prisma.testScenario.createMany({ experimentId, sortIndex: 0, variableValues: { - country: "USA", + country: "Spain", }, }, { experimentId, sortIndex: 1, variableValues: { - country: "Spain", + country: "USA", }, }, { diff --git a/src/pages/experiments/[id].tsx b/src/pages/experiments/[id].tsx index 3e2e4a3..40589a5 100644 --- a/src/pages/experiments/[id].tsx +++ b/src/pages/experiments/[id].tsx @@ -64,7 +64,7 @@ export default function Experiment() { if (!experiment.isLoading && !experiment.data) { return ( -
+
Experiment not found ๐Ÿ˜•
@@ -73,7 +73,7 @@ export default function Experiment() { return ( - + diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 8b8e0a4..f7e5976 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -4,7 +4,7 @@ import AppShell from "~/components/nav/AppShell"; export default function Home() { return ( -
+
Select an experiment from the sidebar to get started!
diff --git a/src/server/api/routers/promptVariants.router.ts b/src/server/api/routers/promptVariants.router.ts index 28f0e7f..557c896 100644 --- a/src/server/api/routers/promptVariants.router.ts +++ b/src/server/api/routers/promptVariants.router.ts @@ -26,6 +26,7 @@ export const promptVariantsRouter = createTRPCRouter({ const lastScenario = await prisma.promptVariant.findFirst({ where: { experimentId: input.experimentId, + visible: true, }, orderBy: { sortIndex: "desc",