alihan/OpenPipe-llm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into log-filters

Browse Source
This commit is contained in:
David Corbitt
2023-08-15 00:41:28 -07:00
parent 0b4e116783 b00ab933b3
commit dfee8a0ed7
67 changed files with 2449 additions and 1337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/.env.example
View File

@@ -32,5 +32,5 @@ NEXT_PUBLIC_HOST="http://localhost:3000"
GITHUB_CLIENT_ID="your_client_id"
GITHUB_CLIENT_SECRET="your_secret"
OPENPIPE_BASE_URL="http://localhost:3000/api"
OPENPIPE_BASE_URL="http://localhost:3000/api/v1"
OPENPIPE_API_KEY="your_key"

5
app/@types/nextjs-routes.d.ts vendored
View File

@@ -12,12 +12,11 @@ declare module "nextjs-routes" {
export type Route =
| StaticRoute<"/account/signin">
| DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
| DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
| StaticRoute<"/api/experiments/og-image">
| StaticRoute<"/api/openapi">
| StaticRoute<"/api/sentry-example-api">
| DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
| DynamicRoute<"/api/v1/[...trpc]", { "trpc": string[] }>
| StaticRoute<"/api/v1/openapi">
| StaticRoute<"/dashboard">
| DynamicRoute<"/data/[id]", { "id": string }>
| StaticRoute<"/data">

5
app/package.json
View File

@@ -72,6 +72,7 @@
"nextjs-cors": "^2.1.2",
"nextjs-routes": "^2.0.1",
"openai": "4.0.0-beta.7",
"openpipe": "workspace:*",
"pg": "^8.11.2",
"pluralize": "^8.0.0",
"posthog-js": "^1.75.3",
@@ -100,8 +101,7 @@
"uuid": "^9.0.0",
"vite-tsconfig-paths": "^4.2.0",
"zod": "^3.21.4",
"zustand": "^4.3.9",
"openpipe": "workspace:*"
"zustand": "^4.3.9"
},
"devDependencies": {
"@openapi-contrib/openapi-schema-to-json-schema": "^4.0.5",
@@ -129,6 +129,7 @@
"eslint-plugin-unused-imports": "^2.0.0",
"monaco-editor": "^0.40.0",
"openapi-typescript": "^6.3.4",
"openapi-typescript-codegen": "^0.25.0",
"prisma": "^4.14.0",
"raw-loader": "^4.0.2",
"typescript": "^5.0.4",

32
app/prisma/schema.prisma
View File

@@ -112,17 +112,17 @@ model ScenarioVariantCell {
model ModelResponse {
id String @id @default(uuid()) @db.Uuid
cacheKey String
requestedAt DateTime?
receivedAt DateTime?
respPayload Json?
cost Float?
inputTokens Int?
outputTokens Int?
statusCode Int?
errorMessage String?
retryTime DateTime?
outdated Boolean @default(false)
cacheKey String
requestedAt DateTime?
receivedAt DateTime?
respPayload Json?
cost Float?
inputTokens Int?
outputTokens Int?
statusCode Int?
errorMessage String?
retryTime DateTime?
outdated Boolean @default(false)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@ -273,8 +273,8 @@ model LoggedCall {
projectId String @db.Uuid
project Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
model String?
tags LoggedCallTag[]
model String?
tags LoggedCallTag[]
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@ -295,7 +295,7 @@ model LoggedCallModelResponse {
errorMessage String?
requestedAt DateTime
receivedAt DateTime
receivedAt DateTime
// Note: the function to calculate the cacheKey should include the project
// ID so we don't share cached responses between projects, which could be an
@@ -341,8 +341,8 @@ model ApiKey {
name String
apiKey String @unique
projectId String @db.Uuid
project Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
projectId String @db.Uuid
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt

11
app/prisma/seed.ts
View File

@@ -2,6 +2,7 @@ import { prisma } from "~/server/db";
import dedent from "dedent";
import { generateNewCell } from "~/server/utils/generateNewCell";
import { promptConstructorVersion } from "~/promptConstructor/version";
import { env } from "~/env.mjs";
const defaultId = "11111111-1111-1111-1111-111111111111";
@@ -16,6 +17,16 @@ const project =
data: { id: defaultId },
}));
if (env.OPENPIPE_API_KEY) {
await prisma.apiKey.create({
data: {
projectId: project.id,
name: "Default API Key",
apiKey: env.OPENPIPE_API_KEY,
},
});
}
await prisma.experiment.deleteMany({
where: {
id: defaultId,

59
app/src/modelProviders/openai-ChatCompletion/getCompletion.ts
View File

@@ -1,54 +1,10 @@
/* eslint-disable @typescript-eslint/no-unsafe-call */
import {
type ChatCompletionChunk,
type ChatCompletion,
type CompletionCreateParams,
} from "openai/resources/chat";
import { type CompletionResponse } from "../types";
import { isArray, isString, omit } from "lodash-es";
import { openai } from "~/server/utils/openai";
import { isArray, isString } from "lodash-es";
import { APIError } from "openai";
const mergeStreamedChunks = (
base: ChatCompletion | null,
chunk: ChatCompletionChunk,
): ChatCompletion => {
if (base === null) {
return mergeStreamedChunks({ ...chunk, choices: [] }, chunk);
}
const choices = [...base.choices];
for (const choice of chunk.choices) {
const baseChoice = choices.find((c) => c.index === choice.index);
if (baseChoice) {
baseChoice.finish_reason = choice.finish_reason ?? baseChoice.finish_reason;
baseChoice.message = baseChoice.message ?? { role: "assistant" };
if (choice.delta?.content)
baseChoice.message.content =
((baseChoice.message.content as string) ?? "") + (choice.delta.content ?? "");
if (choice.delta?.function_call) {
const fnCall = baseChoice.message.function_call ?? {};
fnCall.name =
((fnCall.name as string) ?? "") + ((choice.delta.function_call.name as string) ?? "");
fnCall.arguments =
((fnCall.arguments as string) ?? "") +
((choice.delta.function_call.arguments as string) ?? "");
}
} else {
// @ts-expect-error the types are correctly telling us that finish_reason
// could be null, but don't want to fix it right now.
choices.push({ ...omit(choice, "delta"), message: { role: "assistant", ...choice.delta } });
}
}
const merged: ChatCompletion = {
...base,
choices,
};
return merged;
};
import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
import mergeChunks from "openpipe/src/openai/mergeChunks";
import { openai } from "~/server/utils/openai";
import { type CompletionResponse } from "../types";
export async function getCompletion(
input: CompletionCreateParams,
@@ -59,7 +15,6 @@ export async function getCompletion(
try {
if (onStream) {
console.log("got started");
const resp = await openai.chat.completions.create(
{ ...input, stream: true },
{
@@ -67,11 +22,9 @@ export async function getCompletion(
},
);
for await (const part of resp) {
console.log("got part", part);
finalCompletion = mergeStreamedChunks(finalCompletion, part);
finalCompletion = mergeChunks(finalCompletion, part);
onStream(finalCompletion);
}
console.log("got final", finalCompletion);
if (!finalCompletion) {
return {
type: "error",

6
app/src/pages/api/sentry-example-api.js
View File

@@ -1,6 +0,0 @@
// A faulty API route to test Sentry's error monitoring
// @ts-expect-error just a test file, don't care about types
export default function handler(_req, res) {
throw new Error("Sentry Example API Route Error");
res.status(200).json({ name: "John Doe" });
}

11
app/src/pages/api/[...trpc].ts → app/src/pages/api/v1/[...trpc].ts
View File

@@ -1,17 +1,14 @@
import { type NextApiRequest, type NextApiResponse } from "next";
import cors from "nextjs-cors";
import { createOpenApiNextHandler } from "trpc-openapi";
import { createProcedureCache } from "trpc-openapi/dist/adapters/node-http/procedures";
import { appRouter } from "~/server/api/root.router";
import { createTRPCContext } from "~/server/api/trpc";
import { v1ApiRouter } from "~/server/api/external/v1Api.router";
import { createOpenApiContext } from "~/server/api/external/openApiTrpc";
const openApiHandler = createOpenApiNextHandler({
router: appRouter,
createContext: createTRPCContext,
router: v1ApiRouter,
createContext: createOpenApiContext,
});
const cache = createProcedureCache(appRouter);
const handler = async (req: NextApiRequest, res: NextApiResponse) => {
// Setup CORS
await cors(req, res);

8
app/src/pages/api/openapi.json.ts → app/src/pages/api/v1/openapi.json.ts
View File

@@ -1,12 +1,12 @@
import { type NextApiRequest, type NextApiResponse } from "next";
import { generateOpenApiDocument } from "trpc-openapi";
import { appRouter } from "~/server/api/root.router";
import { v1ApiRouter } from "~/server/api/external/v1Api.router";
export const openApiDocument = generateOpenApiDocument(appRouter, {
export const openApiDocument = generateOpenApiDocument(v1ApiRouter, {
title: "OpenPipe API",
description: "The public API for reporting API calls to OpenPipe",
version: "0.1.0",
baseUrl: "https://app.openpipe.ai/api",
version: "0.1.1",
baseUrl: "https://app.openpipe.ai/api/v1",
});
// Respond with our OpenAPI schema
const hander = (req: NextApiRequest, res: NextApiResponse) => {

95
app/src/server/api/external/openApiTrpc.ts vendored Normal file
View File

@@ -0,0 +1,95 @@
import type { ApiKey, Project } from "@prisma/client";
import { TRPCError, initTRPC } from "@trpc/server";
import { type CreateNextContextOptions } from "@trpc/server/adapters/next";
import superjson from "superjson";
import { type OpenApiMeta } from "trpc-openapi";
import { ZodError } from "zod";
import { prisma } from "~/server/db";
type CreateContextOptions = {
key:
| (ApiKey & {
project: Project;
})
| null;
};
/**
* This helper generates the "internals" for a tRPC context. If you need to use it, you can export
* it from here.
*
* Examples of things you may need it for:
* - testing, so we don't have to mock Next.js' req/res
* - tRPC's `createSSGHelpers`, where we don't have req/res
*
* @see https://create.t3.gg/en/usage/trpc#-serverapitrpcts
*/
export const createInnerTRPCContext = (opts: CreateContextOptions) => {
return {
key: opts.key,
};
};
export const createOpenApiContext = async (opts: CreateNextContextOptions) => {
const { req, res } = opts;
const apiKey = req.headers.authorization?.split(" ")[1] as string | null;
if (!apiKey) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const key = await prisma.apiKey.findUnique({
where: { apiKey },
include: { project: true },
});
if (!key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
return createInnerTRPCContext({
key,
});
};
export type TRPCContext = Awaited<ReturnType<typeof createOpenApiContext>>;
const t = initTRPC
.context<typeof createOpenApiContext>()
.meta<OpenApiMeta>()
.create({
transformer: superjson,
errorFormatter({ shape, error }) {
return {
...shape,
data: {
...shape.data,
zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
},
};
},
});
export const createOpenApiRouter = t.router;
export const openApiPublicProc = t.procedure;
/** Reusable middleware that enforces users are logged in before running the procedure. */
const enforceApiKey = t.middleware(async ({ ctx, next }) => {
if (!ctx.key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
return next({
ctx: { key: ctx.key },
});
});
/**
* Protected (authenticated) procedure
*
* If you want a query or mutation to ONLY be accessible to logged in users, use this. It verifies
* the session is valid and guarantees `ctx.session.user` is not null.
*
* @see https://trpc.io/docs/procedures
*/
export const openApiProtectedProc = t.procedure.use(enforceApiKey);

137
app/src/server/api/routers/externalApi.router.ts → app/src/server/api/external/v1Api.router.ts vendored
View File

@@ -2,9 +2,6 @@ import { type Prisma } from "@prisma/client";
import { type JsonValue } from "type-fest";
import { z } from "zod";
import { v4 as uuidv4 } from "uuid";
import { TRPCError } from "@trpc/server";
import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
import { prisma } from "~/server/db";
import { hashRequest } from "~/server/utils/hashObject";
import modelProvider from "~/modelProviders/openai-ChatCompletion";
@@ -12,6 +9,7 @@ import {
type ChatCompletion,
type CompletionCreateParams,
} from "openai/resources/chat/completions";
import { createOpenApiRouter, openApiProtectedProc } from "./openApiTrpc";
const reqValidator = z.object({
model: z.string(),
@@ -28,12 +26,12 @@ const respValidator = z.object({
),
});
export const externalApiRouter = createTRPCRouter({
checkCache: publicProcedure
export const v1ApiRouter = createOpenApiRouter({
checkCache: openApiProtectedProc
.meta({
openapi: {
method: "POST",
path: "/v1/check-cache",
path: "/check-cache",
description: "Check if a prompt is cached",
protect: true,
},
@@ -47,7 +45,8 @@ export const externalApiRouter = createTRPCRouter({
.optional()
.describe(
'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
),
)
.default({}),
}),
)
.output(
@@ -56,18 +55,8 @@ export const externalApiRouter = createTRPCRouter({
}),
)
.mutation(async ({ input, ctx }) => {
const apiKey = ctx.apiKey;
if (!apiKey) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const key = await prisma.apiKey.findUnique({
where: { apiKey },
});
if (!key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const reqPayload = await reqValidator.spa(input.reqPayload);
const cacheKey = hashRequest(key.projectId, reqPayload as JsonValue);
const cacheKey = hashRequest(ctx.key.projectId, reqPayload as JsonValue);
const existingResponse = await prisma.loggedCallModelResponse.findFirst({
where: { cacheKey },
@@ -79,23 +68,28 @@ export const externalApiRouter = createTRPCRouter({
await prisma.loggedCall.create({
data: {
projectId: key.projectId,
projectId: ctx.key.projectId,
requestedAt: new Date(input.requestedAt),
cacheHit: true,
modelResponseId: existingResponse.id,
},
});
await createTags(
existingResponse.originalLoggedCall.projectId,
existingResponse.originalLoggedCallId,
input.tags,
);
return {
respPayload: existingResponse.respPayload,
};
}),
report: publicProcedure
report: openApiProtectedProc
.meta({
openapi: {
method: "POST",
path: "/v1/report",
path: "/report",
description: "Report an API call",
protect: true,
},
@@ -113,26 +107,16 @@ export const externalApiRouter = createTRPCRouter({
.optional()
.describe(
'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
),
)
.default({}),
}),
)
.output(z.void())
.output(z.object({ status: z.literal("ok") }))
.mutation(async ({ input, ctx }) => {
console.log("GOT TAGS", input.tags);
const apiKey = ctx.apiKey;
if (!apiKey) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const key = await prisma.apiKey.findUnique({
where: { apiKey },
});
if (!key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const reqPayload = await reqValidator.spa(input.reqPayload);
const respPayload = await respValidator.spa(input.respPayload);
const requestHash = hashRequest(key.projectId, reqPayload as JsonValue);
const requestHash = hashRequest(ctx.key.projectId, reqPayload as JsonValue);
const newLoggedCallId = uuidv4();
const newModelResponseId = uuidv4();
@@ -151,7 +135,7 @@ export const externalApiRouter = createTRPCRouter({
prisma.loggedCall.create({
data: {
id: newLoggedCallId,
projectId: key.projectId,
projectId: ctx.key.projectId,
requestedAt: new Date(input.requestedAt),
cacheHit: false,
model,
@@ -185,15 +169,78 @@ export const externalApiRouter = createTRPCRouter({
}),
]);
const tagsToCreate = Object.entries(input.tags ?? {}).map(([name, value]) => ({
projectId: key.projectId,
loggedCallId: newLoggedCallId,
// sanitize tags
name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
value,
}));
await prisma.loggedCallTag.createMany({
data: tagsToCreate,
await createTags(ctx.key.projectId, newLoggedCallId, input.tags);
return { status: "ok" };
}),
localTestingOnlyGetLatestLoggedCall: openApiProtectedProc
.meta({
openapi: {
method: "GET",
path: "/local-testing-only-get-latest-logged-call",
description: "Get the latest logged call (only for local testing)",
protect: true, // Make sure to protect this endpoint
},
})
.input(z.void())
.output(
z
.object({
createdAt: z.date(),
cacheHit: z.boolean(),
tags: z.record(z.string().nullable()),
modelResponse: z
.object({
id: z.string(),
statusCode: z.number().nullable(),
errorMessage: z.string().nullable(),
reqPayload: z.unknown(),
respPayload: z.unknown(),
})
.nullable(),
})
.nullable(),
)
.mutation(async ({ ctx }) => {
if (process.env.NODE_ENV === "production") {
throw new Error("This operation is not allowed in production environment");
}
const latestLoggedCall = await prisma.loggedCall.findFirst({
where: { projectId: ctx.key.projectId },
orderBy: { requestedAt: "desc" },
select: {
createdAt: true,
cacheHit: true,
tags: true,
modelResponse: {
select: {
id: true,
statusCode: true,
errorMessage: true,
reqPayload: true,
respPayload: true,
},
},
},
});
return (
latestLoggedCall && {
...latestLoggedCall,
tags: Object.fromEntries(latestLoggedCall.tags.map((tag) => [tag.name, tag.value])),
}
);
}),
});
async function createTags(projectId: string, loggedCallId: string, tags: Record<string, string>) {
const tagsToCreate = Object.entries(tags).map(([name, value]) => ({
projectId,
loggedCallId,
name: name.replaceAll(/[^a-zA-Z0-9_$]/g, "_"),
value,
}));
await prisma.loggedCallTag.createMany({
data: tagsToCreate,
});
}

2
app/src/server/api/root.router.ts
View File

@@ -8,7 +8,6 @@ import { evaluationsRouter } from "./routers/evaluations.router";
import { worldChampsRouter } from "./routers/worldChamps.router";
import { datasetsRouter } from "./routers/datasets.router";
import { datasetEntries } from "./routers/datasetEntries.router";
import { externalApiRouter } from "./routers/externalApi.router";
import { projectsRouter } from "./routers/projects.router";
import { dashboardRouter } from "./routers/dashboard.router";
import { loggedCallsRouter } from "./routers/loggedCalls.router";
@@ -31,7 +30,6 @@ export const appRouter = createTRPCRouter({
projects: projectsRouter,
dashboard: dashboardRouter,
loggedCalls: loggedCallsRouter,
externalApi: externalApiRouter,
});
// export type definition of API

5
app/src/server/api/trpc.ts
View File

@@ -27,7 +27,6 @@ import { capturePath } from "~/utils/analytics/serverAnalytics";
type CreateContextOptions = {
session: Session | null;
apiKey: string | null;
};
// eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -46,7 +45,6 @@ const noOp = () => {};
export const createInnerTRPCContext = (opts: CreateContextOptions) => {
return {
session: opts.session,
apiKey: opts.apiKey,
prisma,
markAccessControlRun: noOp,
};
@@ -64,11 +62,8 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
// Get the session from the server using the getServerSession wrapper function
const session = await getServerAuthSession({ req, res });
const apiKey = req.headers.authorization?.split(" ")[1] as string | null;
return createInnerTRPCContext({
session,
apiKey,
});
};

22
app/src/server/scripts/client-codegen.ts
View File

@@ -1,8 +1,9 @@
import "dotenv/config";
import { openApiDocument } from "~/pages/api/openapi.json";
import { openApiDocument } from "~/pages/api/v1/openapi.json";
import fs from "fs";
import path from "path";
import { execSync } from "child_process";
import { generate } from "openapi-typescript-codegen";
const scriptPath = import.meta.url.replace("file://", "");
const clientLibsPath = path.join(path.dirname(scriptPath), "../../../../client-libs");
@@ -18,13 +19,20 @@ console.log("Generating TypeScript client");
const tsClientPath = path.join(clientLibsPath, "typescript/src/codegen");
fs.rmSync(tsClientPath, { recursive: true, force: true });
fs.mkdirSync(tsClientPath, { recursive: true });
execSync(
`pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
{
stdio: "inherit",
},
);
await generate({
input: openApiDocument,
output: tsClientPath,
clientName: "OPClient",
httpClient: "node",
});
// execSync(
// `pnpm run openapi generate --input "${schemaPath}" --output "${tsClientPath}" --name OPClient --client node`,
// {
// stdio: "inherit",
// },
// );
console.log("Generating Python client");

2
app/src/server/utils/generateApiKey.ts
View File

@@ -2,4 +2,4 @@ import cryptoRandomString from "crypto-random-string";
const KEY_LENGTH = 42;
export const generateApiKey = () => `opc_${cryptoRandomString({ length: KEY_LENGTH })}`;
export const generateApiKey = () => `opk_${cryptoRandomString({ length: KEY_LENGTH })}`;

11
app/src/server/utils/openai.ts
View File

@@ -1,7 +1,6 @@
import { type ClientOptions } from "openai";
import fs from "fs";
import path from "path";
import OpenAI from "openpipe/src/openai";
import OpenAI, { type ClientOptions } from "openpipe/src/openai";
import { env } from "~/env.mjs";
@@ -16,7 +15,13 @@ try {
config = JSON.parse(jsonData.toString());
} catch (error) {
// Set a dummy key so it doesn't fail at build time
config = { apiKey: env.OPENAI_API_KEY ?? "dummy-key" };
config = {
apiKey: env.OPENAI_API_KEY ?? "dummy-key",
openpipe: {
apiKey: env.OPENPIPE_API_KEY,
baseUrl: "http://localhost:3000/api/v1",
},
};
}
// export const openai = env.OPENPIPE_API_KEY ? new OpenAI.OpenAI(config) : new OriginalOpenAI(config);