alihan/OpenPipe-llm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into log-filters

Browse Source
This commit is contained in:
David Corbitt
2023-08-15 00:41:28 -07:00
parent 0b4e116783 b00ab933b3
commit dfee8a0ed7
67 changed files with 2449 additions and 1337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
app/src/server/api/external/openApiTrpc.ts vendored Normal file
View File

@@ -0,0 +1,95 @@
import type { ApiKey, Project } from "@prisma/client";
import { TRPCError, initTRPC } from "@trpc/server";
import { type CreateNextContextOptions } from "@trpc/server/adapters/next";
import superjson from "superjson";
import { type OpenApiMeta } from "trpc-openapi";
import { ZodError } from "zod";
import { prisma } from "~/server/db";
type CreateContextOptions = {
key:
| (ApiKey & {
project: Project;
})
| null;
};
/**
* This helper generates the "internals" for a tRPC context. If you need to use it, you can export
* it from here.
*
* Examples of things you may need it for:
* - testing, so we don't have to mock Next.js' req/res
* - tRPC's `createSSGHelpers`, where we don't have req/res
*
* @see https://create.t3.gg/en/usage/trpc#-serverapitrpcts
*/
export const createInnerTRPCContext = (opts: CreateContextOptions) => {
return {
key: opts.key,
};
};
export const createOpenApiContext = async (opts: CreateNextContextOptions) => {
const { req, res } = opts;
const apiKey = req.headers.authorization?.split(" ")[1] as string | null;
if (!apiKey) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const key = await prisma.apiKey.findUnique({
where: { apiKey },
include: { project: true },
});
if (!key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
return createInnerTRPCContext({
key,
});
};
export type TRPCContext = Awaited<ReturnType<typeof createOpenApiContext>>;
const t = initTRPC
.context<typeof createOpenApiContext>()
.meta<OpenApiMeta>()
.create({
transformer: superjson,
errorFormatter({ shape, error }) {
return {
...shape,
data: {
...shape.data,
zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
},
};
},
});
export const createOpenApiRouter = t.router;
export const openApiPublicProc = t.procedure;
/** Reusable middleware that enforces users are logged in before running the procedure. */
const enforceApiKey = t.middleware(async ({ ctx, next }) => {
if (!ctx.key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
return next({
ctx: { key: ctx.key },
});
});
/**
* Protected (authenticated) procedure
*
* If you want a query or mutation to ONLY be accessible to logged in users, use this. It verifies
* the session is valid and guarantees `ctx.session.user` is not null.
*
* @see https://trpc.io/docs/procedures
*/
export const openApiProtectedProc = t.procedure.use(enforceApiKey);

137
app/src/server/api/routers/externalApi.router.ts → app/src/server/api/external/v1Api.router.ts vendored
View File

@@ -2,9 +2,6 @@ import { type Prisma } from "@prisma/client";
import { type JsonValue } from "type-fest";
import { z } from "zod";
import { v4 as uuidv4 } from "uuid";
import { TRPCError } from "@trpc/server";
import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
import { prisma } from "~/server/db";
import { hashRequest } from "~/server/utils/hashObject";
import modelProvider from "~/modelProviders/openai-ChatCompletion";
@@ -12,6 +9,7 @@ import {
type ChatCompletion,
type CompletionCreateParams,
} from "openai/resources/chat/completions";
import { createOpenApiRouter, openApiProtectedProc } from "./openApiTrpc";
const reqValidator = z.object({
model: z.string(),
@@ -28,12 +26,12 @@ const respValidator = z.object({
),
});
export const externalApiRouter = createTRPCRouter({
checkCache: publicProcedure
export const v1ApiRouter = createOpenApiRouter({
checkCache: openApiProtectedProc
.meta({
openapi: {
method: "POST",
path: "/v1/check-cache",
path: "/check-cache",
description: "Check if a prompt is cached",
protect: true,
},
@@ -47,7 +45,8 @@ export const externalApiRouter = createTRPCRouter({
.optional()
.describe(
'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
),
)
.default({}),
}),
)
.output(
@@ -56,18 +55,8 @@ export const externalApiRouter = createTRPCRouter({
}),
)
.mutation(async ({ input, ctx }) => {
const apiKey = ctx.apiKey;
if (!apiKey) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const key = await prisma.apiKey.findUnique({
where: { apiKey },
});
if (!key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const reqPayload = await reqValidator.spa(input.reqPayload);
const cacheKey = hashRequest(key.projectId, reqPayload as JsonValue);
const cacheKey = hashRequest(ctx.key.projectId, reqPayload as JsonValue);
const existingResponse = await prisma.loggedCallModelResponse.findFirst({
where: { cacheKey },
@@ -79,23 +68,28 @@ export const externalApiRouter = createTRPCRouter({
await prisma.loggedCall.create({
data: {
projectId: key.projectId,
projectId: ctx.key.projectId,
requestedAt: new Date(input.requestedAt),
cacheHit: true,
modelResponseId: existingResponse.id,
},
});
await createTags(
existingResponse.originalLoggedCall.projectId,
existingResponse.originalLoggedCallId,
input.tags,
);
return {
respPayload: existingResponse.respPayload,
};
}),
report: publicProcedure
report: openApiProtectedProc
.meta({
openapi: {
method: "POST",
path: "/v1/report",
path: "/report",
description: "Report an API call",
protect: true,
},
@@ -113,26 +107,16 @@ export const externalApiRouter = createTRPCRouter({
.optional()
.describe(
'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
),
)
.default({}),
}),
)
.output(z.void())
.output(z.object({ status: z.literal("ok") }))
.mutation(async ({ input, ctx }) => {
console.log("GOT TAGS", input.tags);
const apiKey = ctx.apiKey;
if (!apiKey) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const key = await prisma.apiKey.findUnique({
where: { apiKey },
});
if (!key) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
const reqPayload = await reqValidator.spa(input.reqPayload);
const respPayload = await respValidator.spa(input.respPayload);
const requestHash = hashRequest(key.projectId, reqPayload as JsonValue);
const requestHash = hashRequest(ctx.key.projectId, reqPayload as JsonValue);
const newLoggedCallId = uuidv4();
const newModelResponseId = uuidv4();
@@ -151,7 +135,7 @@ export const externalApiRouter = createTRPCRouter({
prisma.loggedCall.create({
data: {
id: newLoggedCallId,
projectId: key.projectId,
projectId: ctx.key.projectId,
requestedAt: new Date(input.requestedAt),
cacheHit: false,
model,
@@ -185,15 +169,78 @@ export const externalApiRouter = createTRPCRouter({
}),
]);
const tagsToCreate = Object.entries(input.tags ?? {}).map(([name, value]) => ({
projectId: key.projectId,
loggedCallId: newLoggedCallId,
// sanitize tags
name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
value,
}));
await prisma.loggedCallTag.createMany({
data: tagsToCreate,
await createTags(ctx.key.projectId, newLoggedCallId, input.tags);
return { status: "ok" };
}),
localTestingOnlyGetLatestLoggedCall: openApiProtectedProc
.meta({
openapi: {
method: "GET",
path: "/local-testing-only-get-latest-logged-call",
description: "Get the latest logged call (only for local testing)",
protect: true, // Make sure to protect this endpoint
},
})
.input(z.void())
.output(
z
.object({
createdAt: z.date(),
cacheHit: z.boolean(),
tags: z.record(z.string().nullable()),
modelResponse: z
.object({
id: z.string(),
statusCode: z.number().nullable(),
errorMessage: z.string().nullable(),
reqPayload: z.unknown(),
respPayload: z.unknown(),
})
.nullable(),
})
.nullable(),
)
.mutation(async ({ ctx }) => {
if (process.env.NODE_ENV === "production") {
throw new Error("This operation is not allowed in production environment");
}
const latestLoggedCall = await prisma.loggedCall.findFirst({
where: { projectId: ctx.key.projectId },
orderBy: { requestedAt: "desc" },
select: {
createdAt: true,
cacheHit: true,
tags: true,
modelResponse: {
select: {
id: true,
statusCode: true,
errorMessage: true,
reqPayload: true,
respPayload: true,
},
},
},
});
return (
latestLoggedCall && {
...latestLoggedCall,
tags: Object.fromEntries(latestLoggedCall.tags.map((tag) => [tag.name, tag.value])),
}
);
}),
});
async function createTags(projectId: string, loggedCallId: string, tags: Record<string, string>) {
const tagsToCreate = Object.entries(tags).map(([name, value]) => ({
projectId,
loggedCallId,
name: name.replaceAll(/[^a-zA-Z0-9_$]/g, "_"),
value,
}));
await prisma.loggedCallTag.createMany({
data: tagsToCreate,
});
}

2
app/src/server/api/root.router.ts
View File

@@ -8,7 +8,6 @@ import { evaluationsRouter } from "./routers/evaluations.router";
import { worldChampsRouter } from "./routers/worldChamps.router";
import { datasetsRouter } from "./routers/datasets.router";
import { datasetEntries } from "./routers/datasetEntries.router";
import { externalApiRouter } from "./routers/externalApi.router";
import { projectsRouter } from "./routers/projects.router";
import { dashboardRouter } from "./routers/dashboard.router";
import { loggedCallsRouter } from "./routers/loggedCalls.router";
@@ -31,7 +30,6 @@ export const appRouter = createTRPCRouter({
projects: projectsRouter,
dashboard: dashboardRouter,
loggedCalls: loggedCallsRouter,
externalApi: externalApiRouter,
});
// export type definition of API

5
app/src/server/api/trpc.ts
View File

@@ -27,7 +27,6 @@ import { capturePath } from "~/utils/analytics/serverAnalytics";
type CreateContextOptions = {
session: Session | null;
apiKey: string | null;
};
// eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -46,7 +45,6 @@ const noOp = () => {};
export const createInnerTRPCContext = (opts: CreateContextOptions) => {
return {
session: opts.session,
apiKey: opts.apiKey,
prisma,
markAccessControlRun: noOp,
};
@@ -64,11 +62,8 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
// Get the session from the server using the getServerSession wrapper function
const session = await getServerAuthSession({ req, res });
const apiKey = req.headers.authorization?.split(" ")[1] as string | null;
return createInnerTRPCContext({
session,
apiKey,
});
};

22
app/src/server/scripts/client-codegen.ts
View File

@@ -1,8 +1,9 @@
import "dotenv/config";
import { openApiDocument } from "~/pages/api/openapi.json";
import { openApiDocument } from "~/pages/api/v1/openapi.json";
import fs from "fs";
import path from "path";
import { execSync } from "child_process";
import { generate } from "openapi-typescript-codegen";
const scriptPath = import.meta.url.replace("file://", "");
const clientLibsPath = path.join(path.dirname(scriptPath), "../../../../client-libs");
@@ -18,13 +19,20 @@ console.log("Generating TypeScript client");
const tsClientPath = path.join(clientLibsPath, "typescript/src/codegen");
fs.rmSync(tsClientPath, { recursive: true, force: true });
fs.mkdirSync(tsClientPath, { recursive: true });
execSync(
`pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
{
stdio: "inherit",
},
);
await generate({
input: openApiDocument,
output: tsClientPath,
clientName: "OPClient",
httpClient: "node",
});
// execSync(
// `pnpm run openapi generate --input "${schemaPath}" --output "${tsClientPath}" --name OPClient --client node`,
// {
// stdio: "inherit",
// },
// );
console.log("Generating Python client");

2
app/src/server/utils/generateApiKey.ts
View File

@@ -2,4 +2,4 @@ import cryptoRandomString from "crypto-random-string";
const KEY_LENGTH = 42;
export const generateApiKey = () => `opc_${cryptoRandomString({ length: KEY_LENGTH })}`;
export const generateApiKey = () => `opk_${cryptoRandomString({ length: KEY_LENGTH })}`;

11
app/src/server/utils/openai.ts
View File

@@ -1,7 +1,6 @@
import { type ClientOptions } from "openai";
import fs from "fs";
import path from "path";
import OpenAI from "openpipe/src/openai";
import OpenAI, { type ClientOptions } from "openpipe/src/openai";
import { env } from "~/env.mjs";
@@ -16,7 +15,13 @@ try {
config = JSON.parse(jsonData.toString());
} catch (error) {
// Set a dummy key so it doesn't fail at build time
config = { apiKey: env.OPENAI_API_KEY ?? "dummy-key" };
config = {
apiKey: env.OPENAI_API_KEY ?? "dummy-key",
openpipe: {
apiKey: env.OPENPIPE_API_KEY,
baseUrl: "http://localhost:3000/api/v1",
},
};
}
// export const openai = env.OPENPIPE_API_KEY ? new OpenAI.OpenAI(config) : new OriginalOpenAI(config);