From b4cb931f6c6bb6c1514c85b7744448e499e58404 Mon Sep 17 00:00:00 2001 From: Kyle Corbitt Date: Fri, 25 Aug 2023 06:37:06 +0000 Subject: [PATCH] first version of example ready --- examples/classify-recipes/.env.example | 3 + examples/classify-recipes/README.md | 10 + examples/classify-recipes/benchmark.ipynb | 493 +--------------------- examples/classify-recipes/evaluate.ipynb | 17 +- 4 files changed, 31 insertions(+), 492 deletions(-) create mode 100644 examples/classify-recipes/README.md diff --git a/examples/classify-recipes/.env.example b/examples/classify-recipes/.env.example index 819c9fc..430ec47 100644 --- a/examples/classify-recipes/.env.example +++ b/examples/classify-recipes/.env.example @@ -1,4 +1,7 @@ OPENAI_API_KEY="[your OpenAI API key]" OPENPIPE_API_KEY="[your OpenPipe API key from https://app.openpipe.ai/project/settings]" +# You'll need this to download the Llama 2 weights from Hugging Face +HUGGING_FACE_HUB_TOKEN="[Your Hugging Face Hub token]" + WANDB_API_KEY="[Optionally, you can set a Weights & Biases API key to track your training run. Create it at https://wandb.ai/settings]" \ No newline at end of file diff --git a/examples/classify-recipes/README.md b/examples/classify-recipes/README.md new file mode 100644 index 0000000..067fc92 --- /dev/null +++ b/examples/classify-recipes/README.md @@ -0,0 +1,10 @@ +# OpenPipe demo: fine-tuning your own model + +Hi there! This repository should give you a brief overview of how to fine-tune a competitive model from start to finish. You should review the notebooks in this directory in the following order: + +1. [./generate-data.ipynb](./generate-data.ipynb): Demonstrates how to generate a sample dataset of GPT-4 completions, store it using OpenPipe, and then export it in a format suitable for training a model. +2. [./train.ipynb](./train.ipynb): Trains a Llama 2 7B model on the dataset from step (1). +3. [./evaluate.ipynb](./evaluate.ipynb): Evaluates the model we trained using a special test set that we set aside in step (1). +4. [./benchmark.ipynb](./benchmark.ipynb): A script to compare costs and completion latencies between our fine-tuned model, GPT-3.5, and GPT-4. + +If you want to follow along yourself, I recommend using [RunPod](https://www.runpod.io/). The training scripts we use will run on any of their GPUs with 24GB of vRAM or more. \ No newline at end of file diff --git a/examples/classify-recipes/benchmark.ipynb b/examples/classify-recipes/benchmark.ipynb index f22bc68..8849632 100644 --- a/examples/classify-recipes/benchmark.ipynb +++ b/examples/classify-recipes/benchmark.ipynb @@ -1,468 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Current Time: 2023-08-24 22:02:07\n", - "Current Time: 2023-08-24 22:02:17\n", - "Current Time: 2023-08-24 22:02:27\n", - "Current Time: 2023-08-24 22:02:37\n", - "Current Time: 2023-08-24 22:02:47\n", - "Current Time: 2023-08-24 22:02:57\n", - "Current Time: 2023-08-24 22:03:08\n", - "Current Time: 2023-08-24 22:03:18\n", - "Current Time: 2023-08-24 22:03:28\n", - "Current Time: 2023-08-24 22:03:38\n", - "Current Time: 2023-08-24 22:03:48\n", - "Current Time: 2023-08-24 22:03:58\n", - "Current Time: 2023-08-24 22:04:08\n", - "Current Time: 2023-08-24 22:04:18\n", - "Current Time: 2023-08-24 22:04:28\n", - "Current Time: 2023-08-24 22:04:38\n", - "Current Time: 2023-08-24 22:04:48\n", - "Current Time: 2023-08-24 22:04:58\n", - "Current Time: 2023-08-24 22:05:08\n", - "Current Time: 2023-08-24 22:05:18\n", - "Current Time: 2023-08-24 22:05:28\n", - "Current Time: 2023-08-24 22:05:38\n", - "Current Time: 2023-08-24 22:05:48\n", - "Current Time: 2023-08-24 22:05:58\n", - "Current Time: 2023-08-24 22:06:08\n", - "Current Time: 2023-08-24 22:06:18\n", - "Current Time: 2023-08-24 22:06:28\n", - "Current Time: 2023-08-24 22:06:38\n", - "Current Time: 2023-08-24 22:06:48\n", - "Current Time: 2023-08-24 22:06:58\n", - "Current Time: 2023-08-24 22:07:08\n", - "Current Time: 2023-08-24 22:07:18\n", - "Current Time: 2023-08-24 22:07:28\n", - "Current Time: 2023-08-24 22:07:38\n", - "Current Time: 2023-08-24 22:07:48\n", - "Current Time: 2023-08-24 22:07:58\n", - "Current Time: 2023-08-24 22:08:08\n", - "Current Time: 2023-08-24 22:08:18\n", - "Current Time: 2023-08-24 22:08:28\n", - "Current Time: 2023-08-24 22:08:38\n", - "Current Time: 2023-08-24 22:08:48\n", - "Current Time: 2023-08-24 22:08:58\n", - "Current Time: 2023-08-24 22:09:08\n", - "Current Time: 2023-08-24 22:09:18\n", - "Current Time: 2023-08-24 22:09:28\n", - "Current Time: 2023-08-24 22:09:38\n", - "Current Time: 2023-08-24 22:09:48\n", - "Current Time: 2023-08-24 22:09:58\n", - "Current Time: 2023-08-24 22:10:08\n", - "Current Time: 2023-08-24 22:10:18\n", - "Current Time: 2023-08-24 22:10:28\n", - "Current Time: 2023-08-24 22:10:38\n", - "Current Time: 2023-08-24 22:10:48\n", - "Current Time: 2023-08-24 22:10:58\n", - "Current Time: 2023-08-24 22:11:08\n", - "Current Time: 2023-08-24 22:11:18\n", - "Current Time: 2023-08-24 22:11:28\n", - "Current Time: 2023-08-24 22:11:38\n", - "Current Time: 2023-08-24 22:11:48\n", - "Current Time: 2023-08-24 22:11:58\n", - "Current Time: 2023-08-24 22:12:08\n", - "Current Time: 2023-08-24 22:12:18\n", - "Current Time: 2023-08-24 22:12:28\n", - "Current Time: 2023-08-24 22:12:38\n", - "Current Time: 2023-08-24 22:12:48\n", - "Current Time: 2023-08-24 22:12:58\n", - "Current Time: 2023-08-24 22:13:08\n", - "Current Time: 2023-08-24 22:13:18\n", - "Current Time: 2023-08-24 22:13:28\n", - "Current Time: 2023-08-24 22:13:38\n", - "Current Time: 2023-08-24 22:13:48\n", - "Current Time: 2023-08-24 22:13:58\n", - "Current Time: 2023-08-24 22:14:08\n", - "Current Time: 2023-08-24 22:14:18\n", - "Current Time: 2023-08-24 22:14:28\n", - "Current Time: 2023-08-24 22:14:38\n", - "Current Time: 2023-08-24 22:14:48\n", - "Current Time: 2023-08-24 22:14:58\n", - "Current Time: 2023-08-24 22:15:08\n", - "Current Time: 2023-08-24 22:15:18\n", - "Current Time: 2023-08-24 22:15:28\n", - "Current Time: 2023-08-24 22:15:38\n", - "Current Time: 2023-08-24 22:15:48\n", - "Current Time: 2023-08-24 22:15:58\n", - "Current Time: 2023-08-24 22:16:08\n", - "Current Time: 2023-08-24 22:16:18\n", - "Current Time: 2023-08-24 22:16:28\n", - "Current Time: 2023-08-24 22:16:38\n", - "Current Time: 2023-08-24 22:16:48\n", - "Current Time: 2023-08-24 22:16:58\n", - "Current Time: 2023-08-24 22:17:08\n", - "Current Time: 2023-08-24 22:17:18\n", - "Current Time: 2023-08-24 22:17:28\n", - "Current Time: 2023-08-24 22:17:38\n", - "Current Time: 2023-08-24 22:17:48\n", - "Current Time: 2023-08-24 22:17:58\n", - "Current Time: 2023-08-24 22:18:08\n", - "Current Time: 2023-08-24 22:18:18\n", - "Current Time: 2023-08-24 22:18:28\n", - "Current Time: 2023-08-24 22:18:38\n", - "Current Time: 2023-08-24 22:18:48\n", - "Current Time: 2023-08-24 22:18:58\n", - "Current Time: 2023-08-24 22:19:08\n", - "Current Time: 2023-08-24 22:19:18\n", - "Current Time: 2023-08-24 22:19:28\n", - "Current Time: 2023-08-24 22:19:38\n", - "Current Time: 2023-08-24 22:19:48\n", - "Current Time: 2023-08-24 22:19:58\n", - "Current Time: 2023-08-24 22:20:08\n", - "Current Time: 2023-08-24 22:20:18\n", - "Current Time: 2023-08-24 22:20:28\n", - "Current Time: 2023-08-24 22:20:39\n", - "Current Time: 2023-08-24 22:20:49\n", - "Current Time: 2023-08-24 22:20:59\n", - "Current Time: 2023-08-24 22:21:09\n", - "Current Time: 2023-08-24 22:21:19\n", - "Current Time: 2023-08-24 22:21:29\n", - "Current Time: 2023-08-24 22:21:39\n", - "Current Time: 2023-08-24 22:21:49\n", - "Current Time: 2023-08-24 22:21:59\n", - "Current Time: 2023-08-24 22:22:09\n", - "Current Time: 2023-08-24 22:22:19\n", - "Current Time: 2023-08-24 22:22:29\n", - "Current Time: 2023-08-24 22:22:39\n", - "Current Time: 2023-08-24 22:22:49\n", - "Current Time: 2023-08-24 22:22:59\n", - "Current Time: 2023-08-24 22:23:09\n", - "Current Time: 2023-08-24 22:23:19\n", - "Current Time: 2023-08-24 22:23:29\n", - "Current Time: 2023-08-24 22:23:39\n", - "Current Time: 2023-08-24 22:23:49\n", - "Current Time: 2023-08-24 22:23:59\n", - "Current Time: 2023-08-24 22:24:09\n", - "Current Time: 2023-08-24 22:24:19\n", - "Current Time: 2023-08-24 22:24:29\n", - "Current Time: 2023-08-24 22:24:39\n", - "Current Time: 2023-08-24 22:24:49\n", - "Current Time: 2023-08-24 22:24:59\n", - "Current Time: 2023-08-24 22:25:09\n", - "Current Time: 2023-08-24 22:25:19\n", - "Current Time: 2023-08-24 22:25:29\n", - "Current Time: 2023-08-24 22:25:39\n", - "Current Time: 2023-08-24 22:25:49\n", - "Current Time: 2023-08-24 22:25:59\n", - "Current Time: 2023-08-24 22:26:09\n", - "Current Time: 2023-08-24 22:26:19\n", - "Current Time: 2023-08-24 22:26:29\n", - "Current Time: 2023-08-24 22:26:39\n", - "Current Time: 2023-08-24 22:26:49\n", - "Current Time: 2023-08-24 22:26:59\n", - "Current Time: 2023-08-24 22:27:09\n", - "Current Time: 2023-08-24 22:27:19\n", - "Current Time: 2023-08-24 22:27:29\n", - "Current Time: 2023-08-24 22:27:39\n", - "Current Time: 2023-08-24 22:27:49\n", - "Current Time: 2023-08-24 22:27:59\n", - "Current Time: 2023-08-24 22:28:09\n", - "Current Time: 2023-08-24 22:28:19\n", - "Current Time: 2023-08-24 22:28:29\n", - "Current Time: 2023-08-24 22:28:39\n", - "Current Time: 2023-08-24 22:28:49\n", - "Current Time: 2023-08-24 22:28:59\n", - "Current Time: 2023-08-24 22:29:09\n", - "Current Time: 2023-08-24 22:29:19\n", - "Current Time: 2023-08-24 22:29:29\n", - "Current Time: 2023-08-24 22:29:39\n", - "Current Time: 2023-08-24 22:29:49\n", - "Current Time: 2023-08-24 22:29:59\n", - "Current Time: 2023-08-24 22:30:09\n", - "Current Time: 2023-08-24 22:30:19\n", - "Current Time: 2023-08-24 22:30:29\n", - "Current Time: 2023-08-24 22:30:39\n", - "Current Time: 2023-08-24 22:30:49\n", - "Current Time: 2023-08-24 22:30:59\n", - "Current Time: 2023-08-24 22:31:09\n", - "Current Time: 2023-08-24 22:31:19\n", - "Current Time: 2023-08-24 22:31:29\n", - "Current Time: 2023-08-24 22:31:39\n", - "Current Time: 2023-08-24 22:31:49\n", - "Current Time: 2023-08-24 22:31:59\n", - "Current Time: 2023-08-24 22:32:09\n", - "Current Time: 2023-08-24 22:32:19\n", - "Current Time: 2023-08-24 22:32:29\n", - "Current Time: 2023-08-24 22:32:39\n", - "Current Time: 2023-08-24 22:32:49\n", - "Current Time: 2023-08-24 22:32:59\n", - "Current Time: 2023-08-24 22:33:09\n", - "Current Time: 2023-08-24 22:33:19\n", - "Current Time: 2023-08-24 22:33:29\n", - "Current Time: 2023-08-24 22:33:39\n", - "Current Time: 2023-08-24 22:33:49\n", - "Current Time: 2023-08-24 22:33:59\n", - "Current Time: 2023-08-24 22:34:09\n", - "Current Time: 2023-08-24 22:34:19\n", - "Current Time: 2023-08-24 22:34:29\n", - "Current Time: 2023-08-24 22:34:39\n", - "Current Time: 2023-08-24 22:34:49\n", - "Current Time: 2023-08-24 22:34:59\n", - "Current Time: 2023-08-24 22:35:09\n", - "Current Time: 2023-08-24 22:35:19\n", - "Current Time: 2023-08-24 22:35:29\n", - "Current Time: 2023-08-24 22:35:39\n", - "Current Time: 2023-08-24 22:35:49\n", - "Current Time: 2023-08-24 22:35:59\n", - "Current Time: 2023-08-24 22:36:09\n", - "Current Time: 2023-08-24 22:36:19\n", - "Current Time: 2023-08-24 22:36:29\n", - "Current Time: 2023-08-24 22:36:39\n", - "Current Time: 2023-08-24 22:36:49\n", - "Current Time: 2023-08-24 22:36:59\n", - "Current Time: 2023-08-24 22:37:09\n", - "Current Time: 2023-08-24 22:37:19\n", - "Current Time: 2023-08-24 22:37:30\n", - "Current Time: 2023-08-24 22:37:40\n", - "Current Time: 2023-08-24 22:37:50\n", - "Current Time: 2023-08-24 22:38:00\n", - "Current Time: 2023-08-24 22:38:10\n", - "Current Time: 2023-08-24 22:38:20\n", - "Current Time: 2023-08-24 22:38:30\n", - "Current Time: 2023-08-24 22:38:40\n", - "Current Time: 2023-08-24 22:38:50\n", - "Current Time: 2023-08-24 22:39:00\n", - "Current Time: 2023-08-24 22:39:10\n", - "Current Time: 2023-08-24 22:39:20\n", - "Current Time: 2023-08-24 22:39:30\n", - "Current Time: 2023-08-24 22:39:40\n", - "Current Time: 2023-08-24 22:39:50\n", - "Current Time: 2023-08-24 22:40:00\n", - "Current Time: 2023-08-24 22:40:10\n", - "Current Time: 2023-08-24 22:40:20\n", - "Current Time: 2023-08-24 22:40:30\n", - "Current Time: 2023-08-24 22:40:40\n", - "Current Time: 2023-08-24 22:40:50\n", - "Current Time: 2023-08-24 22:41:00\n", - "Current Time: 2023-08-24 22:41:10\n", - "Current Time: 2023-08-24 22:41:20\n", - "Current Time: 2023-08-24 22:41:30\n", - "Current Time: 2023-08-24 22:41:40\n", - "Current Time: 2023-08-24 22:41:50\n", - "Current Time: 2023-08-24 22:42:00\n", - "Current Time: 2023-08-24 22:42:10\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Current Time: 2023-08-24 22:42:20\n", - "Current Time: 2023-08-24 22:42:30\n", - "Current Time: 2023-08-24 22:42:40\n", - "Current Time: 2023-08-24 22:42:50\n", - "Current Time: 2023-08-24 22:43:00\n", - "Current Time: 2023-08-24 22:43:10\n", - "Current Time: 2023-08-24 22:43:20\n", - "Current Time: 2023-08-24 22:43:30\n", - "Current Time: 2023-08-24 22:43:40\n", - "Current Time: 2023-08-24 22:43:50\n", - "Current Time: 2023-08-24 22:44:00\n", - "Current Time: 2023-08-24 22:44:10\n", - "Current Time: 2023-08-24 22:44:20\n", - "Current Time: 2023-08-24 22:44:30\n", - "Current Time: 2023-08-24 22:44:40\n", - "Current Time: 2023-08-24 22:44:50\n", - "Current Time: 2023-08-24 22:45:00\n", - "Current Time: 2023-08-24 22:45:10\n", - "Current Time: 2023-08-24 22:45:20\n", - "Current Time: 2023-08-24 22:45:30\n", - "Current Time: 2023-08-24 22:45:40\n", - "Current Time: 2023-08-24 22:45:50\n", - "Current Time: 2023-08-24 22:46:00\n", - "Current Time: 2023-08-24 22:46:10\n", - "Current Time: 2023-08-24 22:46:20\n", - "Current Time: 2023-08-24 22:46:30\n", - "Current Time: 2023-08-24 22:46:40\n", - "Current Time: 2023-08-24 22:46:50\n", - "Current Time: 2023-08-24 22:47:00\n", - "Current Time: 2023-08-24 22:47:10\n", - "Current Time: 2023-08-24 22:47:20\n", - "Current Time: 2023-08-24 22:47:30\n", - "Current Time: 2023-08-24 22:47:40\n", - "Current Time: 2023-08-24 22:47:50\n", - "Current Time: 2023-08-24 22:48:00\n", - "Current Time: 2023-08-24 22:48:10\n", - "Current Time: 2023-08-24 22:48:20\n", - "Current Time: 2023-08-24 22:48:30\n", - "Current Time: 2023-08-24 22:48:40\n", - "Current Time: 2023-08-24 22:48:50\n", - "Current Time: 2023-08-24 22:49:00\n", - "Current Time: 2023-08-24 22:49:10\n", - "Current Time: 2023-08-24 22:49:20\n", - "Current Time: 2023-08-24 22:49:30\n", - "Current Time: 2023-08-24 22:49:40\n", - "Current Time: 2023-08-24 22:49:50\n", - "Current Time: 2023-08-24 22:50:00\n", - "Current Time: 2023-08-24 22:50:10\n", - "Current Time: 2023-08-24 22:50:20\n", - "Current Time: 2023-08-24 22:50:30\n", - "Current Time: 2023-08-24 22:50:40\n", - "Current Time: 2023-08-24 22:50:50\n", - "Current Time: 2023-08-24 22:51:00\n", - "Current Time: 2023-08-24 22:51:10\n", - "Current Time: 2023-08-24 22:51:20\n", - "Current Time: 2023-08-24 22:51:30\n", - "Current Time: 2023-08-24 22:51:40\n", - "Current Time: 2023-08-24 22:51:50\n", - "Current Time: 2023-08-24 22:52:00\n", - "Current Time: 2023-08-24 22:52:10\n", - "Current Time: 2023-08-24 22:52:20\n", - "Current Time: 2023-08-24 22:52:30\n", - "Current Time: 2023-08-24 22:52:40\n", - "Current Time: 2023-08-24 22:52:50\n", - "Current Time: 2023-08-24 22:53:00\n", - "Current Time: 2023-08-24 22:53:10\n", - "Current Time: 2023-08-24 22:53:20\n", - "Current Time: 2023-08-24 22:53:30\n", - "Current Time: 2023-08-24 22:53:40\n", - "Current Time: 2023-08-24 22:53:50\n", - "Current Time: 2023-08-24 22:54:00\n", - "Current Time: 2023-08-24 22:54:11\n", - "Current Time: 2023-08-24 22:54:21\n", - "Current Time: 2023-08-24 22:54:31\n", - "Current Time: 2023-08-24 22:54:41\n", - "Current Time: 2023-08-24 22:54:51\n", - "Current Time: 2023-08-24 22:55:01\n", - "Current Time: 2023-08-24 22:55:11\n", - "Current Time: 2023-08-24 22:55:21\n", - "Current Time: 2023-08-24 22:55:31\n", - "Current Time: 2023-08-24 22:55:41\n", - "Current Time: 2023-08-24 22:55:51\n", - "Current Time: 2023-08-24 22:56:01\n", - "Current Time: 2023-08-24 22:56:11\n", - "Current Time: 2023-08-24 22:56:21\n", - "Current Time: 2023-08-24 22:56:31\n", - "Current Time: 2023-08-24 22:56:41\n", - "Current Time: 2023-08-24 22:56:51\n", - "Current Time: 2023-08-24 22:57:01\n", - "Current Time: 2023-08-24 22:57:11\n", - "Current Time: 2023-08-24 22:57:21\n", - "Current Time: 2023-08-24 22:57:31\n", - "Current Time: 2023-08-24 22:57:41\n", - "Current Time: 2023-08-24 22:57:51\n", - "Current Time: 2023-08-24 22:58:01\n", - "Current Time: 2023-08-24 22:58:11\n", - "Current Time: 2023-08-24 22:58:21\n", - "Current Time: 2023-08-24 22:58:31\n", - "Current Time: 2023-08-24 22:58:41\n", - "Current Time: 2023-08-24 22:58:51\n", - "Current Time: 2023-08-24 22:59:01\n", - "Current Time: 2023-08-24 22:59:11\n", - "Current Time: 2023-08-24 22:59:21\n", - "Current Time: 2023-08-24 22:59:31\n", - "Current Time: 2023-08-24 22:59:41\n", - "Current Time: 2023-08-24 22:59:51\n", - "Current Time: 2023-08-24 23:00:01\n", - "Current Time: 2023-08-24 23:00:11\n", - "Current Time: 2023-08-24 23:00:21\n", - "Current Time: 2023-08-24 23:00:31\n", - "Current Time: 2023-08-24 23:00:41\n", - "Current Time: 2023-08-24 23:00:51\n", - "Current Time: 2023-08-24 23:01:01\n", - "Current Time: 2023-08-24 23:01:11\n", - "Current Time: 2023-08-24 23:01:21\n", - "Current Time: 2023-08-24 23:01:31\n", - "Current Time: 2023-08-24 23:01:41\n", - "Current Time: 2023-08-24 23:01:51\n", - "Current Time: 2023-08-24 23:02:01\n", - "Current Time: 2023-08-24 23:02:11\n", - "Current Time: 2023-08-24 23:02:21\n", - "Current Time: 2023-08-24 23:02:31\n", - "Current Time: 2023-08-24 23:02:41\n", - "Current Time: 2023-08-24 23:02:51\n", - "Current Time: 2023-08-24 23:03:01\n", - "Current Time: 2023-08-24 23:03:11\n", - "Current Time: 2023-08-24 23:03:21\n", - "Current Time: 2023-08-24 23:03:31\n", - "Current Time: 2023-08-24 23:03:41\n", - "Current Time: 2023-08-24 23:03:51\n", - "Current Time: 2023-08-24 23:04:01\n", - "Current Time: 2023-08-24 23:04:11\n", - "Current Time: 2023-08-24 23:04:21\n", - "Current Time: 2023-08-24 23:04:31\n", - "Current Time: 2023-08-24 23:04:41\n", - "Current Time: 2023-08-24 23:04:51\n", - "Current Time: 2023-08-24 23:05:01\n", - "Current Time: 2023-08-24 23:05:11\n", - "Current Time: 2023-08-24 23:05:21\n", - "Current Time: 2023-08-24 23:05:31\n", - "Current Time: 2023-08-24 23:05:41\n", - "Current Time: 2023-08-24 23:05:51\n", - "Current Time: 2023-08-24 23:06:01\n", - "Current Time: 2023-08-24 23:06:11\n", - "Current Time: 2023-08-24 23:06:21\n", - "Current Time: 2023-08-24 23:06:31\n", - "Current Time: 2023-08-24 23:06:41\n", - "Current Time: 2023-08-24 23:06:51\n", - "Current Time: 2023-08-24 23:07:01\n", - "Current Time: 2023-08-24 23:07:12\n", - "Current Time: 2023-08-24 23:07:22\n", - "Current Time: 2023-08-24 23:07:32\n", - "Current Time: 2023-08-24 23:07:42\n", - "Current Time: 2023-08-24 23:07:52\n", - "Current Time: 2023-08-24 23:08:02\n", - "Current Time: 2023-08-24 23:08:12\n", - "Current Time: 2023-08-24 23:08:22\n", - "Current Time: 2023-08-24 23:08:32\n", - "Current Time: 2023-08-24 23:08:42\n", - "Current Time: 2023-08-24 23:08:52\n", - "Current Time: 2023-08-24 23:09:02\n", - "Current Time: 2023-08-24 23:09:12\n", - "Current Time: 2023-08-24 23:09:22\n", - "Current Time: 2023-08-24 23:09:32\n", - "Current Time: 2023-08-24 23:09:42\n", - "Current Time: 2023-08-24 23:09:52\n", - "Current Time: 2023-08-24 23:10:02\n", - "Current Time: 2023-08-24 23:10:12\n", - "Current Time: 2023-08-24 23:10:22\n", - "Current Time: 2023-08-24 23:10:32\n", - "Current Time: 2023-08-24 23:10:42\n", - "Current Time: 2023-08-24 23:10:52\n", - "Current Time: 2023-08-24 23:11:02\n", - "Current Time: 2023-08-24 23:11:12\n", - "Current Time: 2023-08-24 23:11:22\n", - "Current Time: 2023-08-24 23:11:32\n", - "Current Time: 2023-08-24 23:11:42\n", - "Current Time: 2023-08-24 23:11:52\n", - "Current Time: 2023-08-24 23:12:02\n", - "Current Time: 2023-08-24 23:12:12\n", - "Current Time: 2023-08-24 23:12:22\n", - "Current Time: 2023-08-24 23:12:32\n", - "Current Time: 2023-08-24 23:12:42\n", - "Current Time: 2023-08-24 23:12:52\n", - "Current Time: 2023-08-24 23:13:02\n", - "Current Time: 2023-08-24 23:13:12\n", - "Current Time: 2023-08-24 23:13:22\n", - "Current Time: 2023-08-24 23:13:32\n", - "Current Time: 2023-08-24 23:13:42\n", - "Current Time: 2023-08-24 23:13:52\n", - "Current Time: 2023-08-24 23:14:02\n", - "Current Time: 2023-08-24 23:14:12\n", - "Current Time: 2023-08-24 23:14:22\n", - "Current Time: 2023-08-24 23:14:32\n", - "Current Time: 2023-08-24 23:14:42\n" - ] - } - ], - "source": [ - "import time\n", - "\n", - "while True:\n", - " current_time = time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())\n", - " print(f\"Current Time: {current_time}\")\n", - " time.sleep(10)\n" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -730,7 +267,9 @@ "\n", "end_time = time.time()\n", "print(f\"End time: {end_time}\")\n", - "print(f\"Total hours: {((end_time - start_time) / 3600):.2f}\")\n" + "print(f\"Total hours: {((end_time - start_time) / 3600):.2f}\")\n", + "\n", + "# Ended up running this in a separate script to leave it running in the background.\n" ] }, { @@ -825,11 +364,12 @@ "# I used an on-demand Nvidia L40 on RunPod for this, at an hourly cost of $1.14.\n", "finetuned_hourly_cost = 1.14\n", "\n", - "finetuned_total_hours = 16.54\n", + "finetuned_total_hours = 16.5\n", "\n", "finetuned_avg_cost = finetuned_hourly_cost * finetuned_total_hours / len(all_recipes)\n", "\n", - "# The average input and output tokens calculated by OpenAI, based on the 5000 recipes I sent them\n", + "# The average input and output tokens for OpenAI, based on the 5000 recipes I\n", + "# sent them when generating training data.\n", "avg_input_tokens = 276\n", "avg_output_tokens = 42\n", "\n", @@ -842,13 +382,6 @@ " avg_input_tokens * 0.012 / 1000 + avg_output_tokens * 0.016 / 1000 + 0.06 / 1000\n", ")\n", "\n", - "# Multiply the number of recipes\n", - "# gpt_4_cost = len(all_recipes) * gpt_4_avg_cost\n", - "# gpt_35_cost = len(all_recipes) * gpt_35_avg_cost\n", - "# gpt_35_finetuned_cost = len(all_recipes) * gpt_35_finetuned_avg_cost\n", - "\n", - "# Let's put this in a dataframe for easier comparison.\n", - "\n", "costs = pd.DataFrame(\n", " {\n", " \"Model\": [\n", @@ -873,20 +406,6 @@ "\n", "costs\n" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "...and just for fun, let's figure out how many recipes my pescatarian basement-dwelling brother can make! 😂" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/examples/classify-recipes/evaluate.ipynb b/examples/classify-recipes/evaluate.ipynb index 03829e1..2811271 100644 --- a/examples/classify-recipes/evaluate.ipynb +++ b/examples/classify-recipes/evaluate.ipynb @@ -159,15 +159,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "INFO 08-24 22:01:58 llm_engine.py:70] Initializing an LLM engine with config: model='./models/run1/merged', tokenizer='./models/run1/merged', tokenizer_mode=auto, trust_remote_code=False, dtype=torch.float16, use_dummy_weights=False, download_dir=None, use_np_weights=False, tensor_parallel_size=1, seed=0)\n", - "INFO 08-24 22:02:46 llm_engine.py:196] # GPU blocks: 3419, # CPU blocks: 512\n" + "INFO 08-25 03:58:49 llm_engine.py:70] Initializing an LLM engine with config: model='./models/run1/merged', tokenizer='./models/run1/merged', tokenizer_mode=auto, trust_remote_code=False, dtype=torch.float16, use_dummy_weights=False, download_dir=None, use_np_weights=False, tensor_parallel_size=1, seed=0)\n", + "INFO 08-25 03:59:40 llm_engine.py:196] # GPU blocks: 3419, # CPU blocks: 512\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Processed prompts: 100%|██████████| 500/500 [00:37<00:00, 13.44it/s]" + "Processed prompts: 100%|██████████| 500/500 [00:37<00:00, 13.42it/s]" ] }, { @@ -216,7 +216,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -266,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -629,6 +629,13 @@ "\n", "Interested in cost/latency benchmarking? You can check out [./benchmarking.ipynb](./benchmarking.ipynb) for an overview of my findings!" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {