Add newline to publish.sh

Library is now published at https://www.npmjs.com/package/openpipe; see README for details.

app/package.json

@@ -79,7 +79,8 @@
     "nextjs-routes": "^2.0.1",
     "nodemailer": "^6.9.4",
     "openai": "4.0.0-beta.7",
-    "openpipe": "workspace:*",
+    "openpipe": "^0.3.0",
+    "openpipe-dev": "workspace:^",
     "pg": "^8.11.2",
     "pluralize": "^8.0.0",
     "posthog-js": "^1.75.3",

app/src/modelProviders/openai-ChatCompletion/getCompletion.ts

@@ -2,7 +2,7 @@
 import { isArray, isString } from "lodash-es";
 import { APIError } from "openai";
 import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
-import mergeChunks from "openpipe/src/openai/mergeChunks";
+import mergeChunks from "openpipe/openai/mergeChunks";
 import { openai } from "~/server/utils/openai";
 import { type CompletionResponse } from "../types";
 

app/src/server/utils/openai.ts

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import OpenAI, { type ClientOptions } from "openpipe/src/openai";
+import OpenAI, { type ClientOptions } from "openpipe/openai";
 
 import { env } from "~/env.mjs";
 

client-libs/typescript/README.md

@@ -0,0 +1,70 @@
+# OpenPipe Node API Library
+
+[![NPM version](https://img.shields.io/npm/v/openpipe.svg)](https://npmjs.org/package/openpipe)
+
+This library wraps TypeScript or Javascript OpenAI API calls and logs additional data to the configured `OPENPIPE_BASE_URL` for further processing.
+
+It is fully compatible with OpenAI's sdk and logs both streaming and non-streaming requests and responses.
+
+<!-- To learn more about using OpenPipe, check out our [Documentation](https://docs.openpipe.ai/docs/api). -->
+
+## Installation
+
+```sh
+npm install --save openpipe
+# or
+yarn add openpipe
+```
+
+## Usage
+
+1. Create a project at https://app.openpipe.ai
+2. Find your project's API key at https://app.openpipe.ai/project/settings
+3. Configure the OpenPipe client as shown below.
+
+```js
+// import OpenAI from 'openai'
+import OpenAI from "openpipe/openai";
+
+// Fully compatible with original OpenAI initialization
+const openai = new OpenAI({
+  apiKey: "my api key", // defaults to process.env["OPENAI_API_KEY"]
+  // openpipe key is optional
+  openpipe: {
+    apiKey: "my api key", // defaults to process.env["OPENPIPE_API_KEY"]
+    baseUrl: "my url", // defaults to process.env["OPENPIPE_BASE_URL"] or https://app.openpipe.ai/api/v1 if not set
+  },
+});
+
+async function main() {
+  // Allows optional openpipe object
+  const completion = await openai.chat.completions.create({
+    messages: [{ role: "user", content: "Say this is a test" }],
+    model: "gpt-3.5-turbo",
+    // optional
+    openpipe: {
+      // Add custom searchable tags
+      tags: {
+        prompt_id: "getCompletion",
+        any_key: "any_value",
+      },
+    },
+  });
+
+  console.log(completion.choices);
+}
+
+main();
+```
+
+## FAQ
+
+<i>How do I report calls to my self-hosted instance?</i>
+
+Start an instance by following the instructions on [Running Locally](https://github.com/OpenPipe/OpenPipe#running-locally). Once it's running, point your `OPENPIPE_BASE_URL` to your self-hosted instance.
+
+<i>What if my `OPENPIPE_BASE_URL` is misconfigured or my instance goes down? Will my OpenAI calls stop working?</i>
+
+Your OpenAI calls will continue to function as expected no matter what. The sdk handles logging errors gracefully without affecting OpenAI inference.
+
+See the [GitHub repo](https://github.com/OpenPipe/OpenPipe) for more details.

client-libs/typescript/build.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Adapted from https://github.com/openai/openai-node/blob/master/build
+
+set -exuo pipefail
+
+rm -rf dist /tmp/openpipe-build-dist
+
+mkdir /tmp/openpipe-build-dist
+
+cp -rp * /tmp/openpipe-build-dist
+
+# Rename package name in package.json
+python3 -c "
+import json
+with open('/tmp/openpipe-build-dist/package.json', 'r') as f:
+    data = json.load(f)
+data['name'] = 'openpipe'
+with open('/tmp/openpipe-build-dist/package.json', 'w') as f:
+    json.dump(data, f, indent=4)
+"
+
+rm -rf /tmp/openpipe-build-dist/node_modules
+mv /tmp/openpipe-build-dist dist
+
+# build to .js files
+(cd dist && npm exec tsc -- --noEmit false)

client-libs/typescript/index.ts

@@ -1,3 +1 @@
-// main.ts or index.ts at the root level
-export * as OpenAI from "./src/openai";
-export * as OpenAILegacy from "./src/openai-legacy";
+export * as openai from "./openai";

client-libs/typescript/openai/index.test.ts

@@ -80,6 +80,7 @@ test("bad call streaming", async () => {
       stream: true,
     });
   } catch (e) {
+    // @ts-expect-error need to check for error type
     await e.openpipe.reportingFinished;
     const lastLogged = await lastLoggedCall();
     expect(lastLogged?.modelResponse?.errorMessage).toEqual(
@@ -96,7 +97,9 @@ test("bad call", async () => {
       messages: [{ role: "system", content: "count to 10" }],
     });
   } catch (e) {
+    // @ts-expect-error need to check for error type
     assert("openpipe" in e);
+    // @ts-expect-error need to check for error type
     await e.openpipe.reportingFinished;
     const lastLogged = await lastLoggedCall();
     expect(lastLogged?.modelResponse?.errorMessage).toEqual(
@@ -120,7 +123,8 @@ test("caching", async () => {
 
   await completion.openpipe.reportingFinished;
   const firstLogged = await lastLoggedCall();
-  expect(completion.choices[0].message.content).toEqual(
+
+  expect(completion.choices[0]?.message.content).toEqual(
     firstLogged?.modelResponse?.respPayload.choices[0].message.content,
   );
 

client-libs/typescript/package.json

@@ -1,14 +1,17 @@
 {
-  "name": "openpipe",
-  "version": "0.1.0",
+  "name": "openpipe-dev",
+  "version": "0.3.3",
   "type": "module",
   "description": "Metrics and auto-evaluation for LLM calls",
   "scripts": {
-    "build": "tsc",
+    "build": "./build.sh",
     "test": "vitest"
   },
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
+  "main": "./index.ts",
+  "publishConfig": {
+    "access": "public",
+    "main": "./index.js"
+  },
   "keywords": [],
   "author": "",
   "license": "Apache-2.0",

client-libs/typescript/publish.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Adapted from https://github.com/openai/openai-node/blob/master/build
+
+set -exuo pipefail
+
+./build.sh
+
+(cd dist && pnpm publish --access public)

client-libs/typescript/shared.ts

@@ -1,4 +1,5 @@
-import pkg from "../package.json";
+import pkg from "./package.json";
+
 import { DefaultService } from "./codegen";
 
 export type OpenPipeConfig = {

client-libs/typescript/src/openai-legacy/index.ts

@@ -1,85 +0,0 @@
-import * as openPipeClient from "../codegen";
-import * as openai from "openai-legacy";
-import { version } from "../../package.json";
-
-// Anything we don't override we want to pass through to openai directly
-export * as openAILegacy from "openai-legacy";
-
-type OPConfigurationParameters = {
-  apiKey?: string;
-  basePath?: string;
-};
-
-export class Configuration extends openai.Configuration {
-  public qkConfig?: openPipeClient.Configuration;
-
-  constructor(
-    config: openai.ConfigurationParameters & {
-      opParameters?: OPConfigurationParameters;
-    }
-  ) {
-    super(config);
-    if (config.opParameters) {
-      this.qkConfig = new openPipeClient.Configuration(config.opParameters);
-    }
-  }
-}
-
-type CreateChatCompletion = InstanceType<typeof openai.OpenAIApi>["createChatCompletion"];
-
-export class OpenAIApi extends openai.OpenAIApi {
-  public openPipeApi?: openPipeClient.DefaultApi;
-
-  constructor(config: Configuration) {
-    super(config);
-    if (config.qkConfig) {
-      this.openPipeApi = new openPipeClient.DefaultApi(config.qkConfig);
-    }
-  }
-
-  public async createChatCompletion(
-    createChatCompletionRequest: Parameters<CreateChatCompletion>[0],
-    options?: Parameters<CreateChatCompletion>[1]
-  ): ReturnType<CreateChatCompletion> {
-    const requestedAt = Date.now();
-    let resp: Awaited<ReturnType<CreateChatCompletion>> | null = null;
-    let respPayload: openai.CreateChatCompletionResponse | null = null;
-    let statusCode: number | undefined = undefined;
-    let errorMessage: string | undefined;
-    try {
-      resp = await super.createChatCompletion(createChatCompletionRequest, options);
-      respPayload = resp.data;
-      statusCode = resp.status;
-    } catch (err) {
-      console.error("Error in createChatCompletion");
-      if ("isAxiosError" in err && err.isAxiosError) {
-        errorMessage = err.response?.data?.error?.message;
-        respPayload = err.response?.data;
-        statusCode = err.response?.status;
-      } else if ("message" in err) {
-        errorMessage = err.message.toString();
-      }
-      throw err;
-    } finally {
-      this.openPipeApi
-        ?.externalApiReport({
-          requestedAt,
-          receivedAt: Date.now(),
-          reqPayload: createChatCompletionRequest,
-          respPayload: respPayload,
-          statusCode: statusCode,
-          errorMessage,
-          tags: {
-            client: "openai-js",
-            clientVersion: version,
-          },
-        })
-        .catch((err) => {
-          console.error("Error reporting to OP", err);
-        });
-    }
-
-    console.log("done");
-    return resp;
-  }
-}

client-libs/typescript/tsconfig.json

@@ -14,9 +14,12 @@
     "isolatedModules": true,
     "incremental": true,
     "noUncheckedIndexedAccess": true,
-    "baseUrl": ".",
-    "outDir": "dist"
+    "noEmit": true,
+    "sourceMap": true,
+    "declaration": true,
+    "declarationMap": true,
+    "rootDir": "."
   },
-  "include": ["src/**/*.ts"],
+  "include": ["**/*.ts"],
   "exclude": ["node_modules"]
 }

examples/classify-recipes/benchmark-tmp.py

@@ -1,123 +0,0 @@
-# %% [markdown]
-# I'm pretty happy with my model's accuracy relative to GPT-4. How does it compare cost-wise?
-#
-# I'll really push this to its limits -- let's see how quickly our poor model can classify the [full 2-million-recipe dataset](https://huggingface.co/datasets/corbt/all-recipes) 😈.
-
-# %%
-
-# %%
-from datasets import load_dataset
-
-all_recipes = load_dataset("corbt/all-recipes")["train"]["input"]
-
-print(f"Number of recipes: {len(all_recipes):,}")
-
-
-# %%
-from vllm import LLM, SamplingParams
-
-llm = LLM(model="./models/run1/merged", max_num_batched_tokens=4096)
-
-sampling_params = SamplingParams(
-    # 120 should be fine for the work we're doing here.
-    max_tokens=120,
-    # This is a deterministic task so temperature=0 is best.
-    temperature=0,
-)
-
-
-# %%
-import os
-import time
-import json
-
-BATCH_SIZE = 10000
-start_time = time.time()
-print(f"Start time: {start_time}")
-
-for i in range(0, len(all_recipes), BATCH_SIZE):
-    # File name for the current batch
-    file_name = f"./data/benchmark_batch_{int(i/BATCH_SIZE)}.txt"
-
-    # Check if the file already exists; if so, skip to the next batch
-    if os.path.exists(file_name):
-        print(f"File {file_name} exists, skipping recipes {i:,} to {i+BATCH_SIZE:,}...")
-        continue
-
-    print(f"Processing recipes {i:,} to {i+BATCH_SIZE:,}...")
-    outputs = llm.generate(
-        all_recipes[i : i + BATCH_SIZE], sampling_params=sampling_params
-    )
-
-    outputs = [o.outputs[0].text for o in outputs]
-
-    # Write the generated outputs to the file as a JSON list
-    json.dump(outputs, open(file_name, "w"))
-
-end_time = time.time()
-print(f"End time: {end_time}")
-print(f"Total hours: {((end_time - start_time) / 3600):.2f}")
-
-
-# %% [markdown]
-# Nice! I've processed all 2,147,248 recipes in under 17 hours. Let's do a cost comparison with GPT-3.5 and GPT-4. I'll use the GPT-4 latency/cost numbers based on the 5000 samples used to generate our model's training data.
-
-# %%
-import pandas as pd
-
-# I used an on-demand Nvidia L40 on RunPod for this, at an hourly cost of $1.14.
-finetuned_hourly_cost = 1.14
-
-finetuned_total_hours = 17
-
-finetuned_avg_cost = finetuned_hourly_cost * finetuned_total_hours / len(all_recipes)
-
-# The average input and output tokens calculated by OpenAI, based on the 5000 recipes I sent them
-avg_input_tokens = 276
-avg_output_tokens = 42
-
-# Token pricing from https://openai.com/pricing
-gpt_4_avg_cost = avg_input_tokens * 0.03 / 1000 + avg_output_tokens * 0.06 / 1000
-
-gpt_35_avg_cost = avg_input_tokens * 0.0015 / 1000 + avg_output_tokens * 0.0016 / 1000
-
-gpt_35_finetuned_avg_cost = (
-    avg_input_tokens * 0.012 / 1000 + avg_output_tokens * 0.016 / 1000 + 0.06 / 1000
-)
-
-# Multiply the number of recipes
-# gpt_4_cost = len(all_recipes) * gpt_4_avg_cost
-# gpt_35_cost = len(all_recipes) * gpt_35_avg_cost
-# gpt_35_finetuned_cost = len(all_recipes) * gpt_35_finetuned_avg_cost
-
-# Let's put this in a dataframe for easier comparison.
-
-costs = pd.DataFrame(
-    {
-        "Model": [
-            "Llama 2 7B (finetuned)",
-            "GPT-3.5",
-            "GPT-3.5 (finetuned)",
-            "GPT-4",
-        ],
-        "Cost to Classify One Recipe": [
-            finetuned_avg_cost,
-            gpt_35_avg_cost,
-            gpt_35_finetuned_avg_cost,
-            gpt_4_avg_cost,
-        ],
-    }
-)
-
-costs["Cost to Classify Entire Dataset"] = (
-    costs["Cost to Classify One Recipe"] * len(all_recipes)
-).map(lambda x: f"{x:,.2f}")
-
-
-costs
-
-
-# %% [markdown]
-# ...and just for fun, let's figure out how many recipes my pescatarian basement-dwelling brother can make! 😂
-
-# %%

pnpm-lock.yaml

@@ -174,7 +174,10 @@ importers:
         specifier: 4.0.0-beta.7
         version: 4.0.0-beta.7(encoding@0.1.13)
       openpipe:
-        specifier: workspace:*
+        specifier: ^0.3.0
+        version: 0.3.0
+      openpipe-dev:
+        specifier: workspace:^
         version: link:../client-libs/typescript
       pg:
         specifier: ^8.11.2
@@ -7247,6 +7250,19 @@ packages:
       oidc-token-hash: 5.0.3
     dev: false
 
+  /openpipe@0.3.0:
+    resolution: {integrity: sha512-0hhk3Aq0kUxzvNb36vm9vssxMHYZvgJOg5wKeepRhVthW4ygBWftHZjR4PHyOtvjcRmnJ/v4h8xd/IINu5ypnQ==}
+    dependencies:
+      encoding: 0.1.13
+      form-data: 4.0.0
+      lodash-es: 4.17.21
+      node-fetch: 2.6.12(encoding@0.1.13)
+      openai-beta: /openai@4.0.0-beta.7(encoding@0.1.13)
+      openai-legacy: /openai@3.3.0
+    transitivePeerDependencies:
+      - debug
+    dev: false
+
   /optionator@0.9.3:
     resolution: {integrity: sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==}
     engines: {node: '>= 0.8.0'}