OpenPipe-llm/app/prisma/schema.prisma

// This is your Prisma schema file,
// learn more about it in the docs: https://pris.ly/d/prisma-schema

generator client {
    provider = "prisma-client-js"
}

datasource db {
    provider = "postgresql"
    url      = env("DATABASE_URL")
}

model Experiment {
    id    String @id @default(uuid()) @db.Uuid
    label String

    sortIndex Int @default(0)

    projectId String   @db.Uuid
    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt

    templateVariables TemplateVariable[]
    promptVariants    PromptVariant[]
    testScenarios     TestScenario[]
    evaluations       Evaluation[]
}

model PromptVariant {
    id String @id @default(uuid()) @db.Uuid

    label                    String
    promptConstructor        String
    promptConstructorVersion Int
    model                    String
    modelProvider            String

    uiId      String  @default(uuid()) @db.Uuid
    visible   Boolean @default(true)
    sortIndex Int     @default(0)

    experimentId String     @db.Uuid
    experiment   Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade)

    createdAt            DateTime              @default(now())
    updatedAt            DateTime              @updatedAt
    scenarioVariantCells ScenarioVariantCell[]

    @@index([uiId])
}

model TestScenario {
    id String @id @default(uuid()) @db.Uuid

    variableValues Json

    uiId      String  @default(uuid()) @db.Uuid
    visible   Boolean @default(true)
    sortIndex Int     @default(0)

    experimentId String     @db.Uuid
    experiment   Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade)

    createdAt            DateTime              @default(now())
    updatedAt            DateTime              @updatedAt
    scenarioVariantCells ScenarioVariantCell[]
}

model TemplateVariable {
    id String @id @default(uuid()) @db.Uuid

    label String

    experimentId String     @db.Uuid
    experiment   Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt
}

enum CellRetrievalStatus {
    PENDING
    IN_PROGRESS
    COMPLETE
    ERROR
}

model ScenarioVariantCell {
    id String @id @default(uuid()) @db.Uuid

    retrievalStatus CellRetrievalStatus @default(COMPLETE)
    jobQueuedAt     DateTime?
    jobStartedAt    DateTime?
    modelResponses  ModelResponse[]
    errorMessage    String? // Contains errors that occurred independently of model responses

    promptVariantId String        @db.Uuid
    promptVariant   PromptVariant @relation(fields: [promptVariantId], references: [id], onDelete: Cascade)
    prompt          Json?

    testScenarioId String       @db.Uuid
    testScenario   TestScenario @relation(fields: [testScenarioId], references: [id], onDelete: Cascade)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt

    @@unique([promptVariantId, testScenarioId])
}

model ModelResponse {
    id String @id @default(uuid()) @db.Uuid

    cacheKey     String
    requestedAt  DateTime?
    receivedAt   DateTime?
    respPayload  Json?
    cost         Float?
    inputTokens  Int?
    outputTokens Int?
    statusCode   Int?
    errorMessage String?
    retryTime    DateTime?
    outdated     Boolean   @default(false)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt

    scenarioVariantCellId String              @db.Uuid
    scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
    outputEvaluations     OutputEvaluation[]

    @@index([cacheKey])
}

enum EvalType {
    CONTAINS
    DOES_NOT_CONTAIN
    GPT4_EVAL
}

model Evaluation {
    id String @id @default(uuid()) @db.Uuid

    label    String
    evalType EvalType
    value    String

    experimentId String     @db.Uuid
    experiment   Experiment @relation(fields: [experimentId], references: [id], onDelete: Cascade)

    createdAt         DateTime           @default(now())
    updatedAt         DateTime           @updatedAt
    outputEvaluations OutputEvaluation[]
}

model OutputEvaluation {
    id String @id @default(uuid()) @db.Uuid

    // Number between 0 (fail) and 1 (pass)
    result  Float
    details String?

    modelResponseId String        @db.Uuid
    modelResponse   ModelResponse @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)

    evaluationId String     @db.Uuid
    evaluation   Evaluation @relation(fields: [evaluationId], references: [id], onDelete: Cascade)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt

    @@unique([modelResponseId, evaluationId])
}

model Dataset {
    id String @id @default(uuid()) @db.Uuid

    name           String
    datasetEntries DatasetEntry[]

    projectId String  @db.Uuid
    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt
}

model DatasetEntry {
    id String @id @default(uuid()) @db.Uuid

    input  String
    output String?

    datasetId String   @db.Uuid
    dataset   Dataset? @relation(fields: [datasetId], references: [id], onDelete: Cascade)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt
}

model Project {
    id   String @id @default(uuid()) @db.Uuid
    name String @default("Project 1")

    personalProjectUserId String? @unique @db.Uuid
    personalProjectUser   User?   @relation(fields: [personalProjectUserId], references: [id], onDelete: Cascade)

    createdAt    DateTime      @default(now())
    updatedAt    DateTime      @updatedAt
    projectUsers ProjectUser[]
    experiments  Experiment[]
    datasets     Dataset[]
    loggedCalls  LoggedCall[]
    apiKeys      ApiKey[]
}

enum ProjectUserRole {
    ADMIN
    MEMBER
    VIEWER
}

model ProjectUser {
    id String @id @default(uuid()) @db.Uuid

    role ProjectUserRole

    projectId String   @db.Uuid
    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)

    userId String @db.Uuid
    user   User   @relation(fields: [userId], references: [id], onDelete: Cascade)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt

    @@unique([projectId, userId])
}

model WorldChampEntrant {
    id String @id @default(uuid()) @db.Uuid

    userId String @db.Uuid
    user   User   @relation(fields: [userId], references: [id], onDelete: Cascade)

    approved Boolean @default(false)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt

    @@unique([userId])
}

model LoggedCall {
    id String @id @default(uuid()) @db.Uuid

    requestedAt DateTime

    // True if this call was served from the cache, false otherwise
    cacheHit Boolean

    // A LoggedCall is always associated with a LoggedCallModelResponse. If this
    // is a cache miss, we create a new LoggedCallModelResponse.
    // If it's a cache hit, it's a pre-existing LoggedCallModelResponse.    
    modelResponseId String?                  @db.Uuid
    modelResponse   LoggedCallModelResponse? @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)

    // The responses created by this LoggedCall. Will be empty if this LoggedCall was a cache hit.
    createdResponses LoggedCallModelResponse[] @relation(name: "ModelResponseOriginalCall")

    projectId String   @db.Uuid
    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)

    model String?
    tags  LoggedCallTag[]

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt

    @@index([requestedAt])
}

model LoggedCallModelResponse {
    id String @id @default(uuid()) @db.Uuid

    reqPayload Json

    // The HTTP status returned by the model provider
    statusCode  Int?
    respPayload Json?

    // Should be null if the request was successful, and some string if the request failed.
    errorMessage String?

    requestedAt DateTime
    receivedAt  DateTime

    // Note: the function to calculate the cacheKey should include the project
    // ID so we don't share cached responses between projects, which could be an
    // attack vector. Also, we should only set the cacheKey on the model if the
    // request was successful.
    cacheKey String?

    // Derived fields
    durationMs   Int?
    inputTokens  Int?
    outputTokens Int?
    finishReason String?
    completionId String?
    cost         Decimal? @db.Decimal(18, 12)

    // The LoggedCall that created this LoggedCallModelResponse
    originalLoggedCallId String     @unique @db.Uuid
    originalLoggedCall   LoggedCall @relation(name: "ModelResponseOriginalCall", fields: [originalLoggedCallId], references: [id], onDelete: Cascade)

    createdAt   DateTime     @default(now())
    updatedAt   DateTime     @updatedAt
    loggedCalls LoggedCall[]

    @@index([cacheKey])
}

model LoggedCallTag {
    id    String  @id @default(uuid()) @db.Uuid
    name  String
    value String?

    loggedCallId String     @db.Uuid
    loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)

    @@index([name])
    @@index([name, value])
}

model ApiKey {
    id String @id @default(uuid()) @db.Uuid

    name   String
    apiKey String @unique

    projectId String  @db.Uuid
    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)

    createdAt DateTime @default(now())
    updatedAt DateTime @updatedAt
}

model Account {
    id                       String  @id @default(uuid()) @db.Uuid
    userId                   String  @db.Uuid
    type                     String
    provider                 String
    providerAccountId        String
    refresh_token            String? @db.Text
    refresh_token_expires_in Int?
    access_token             String? @db.Text
    expires_at               Int?
    token_type               String?
    scope                    String?
    id_token                 String? @db.Text
    session_state            String?
    user                     User    @relation(fields: [userId], references: [id], onDelete: Cascade)

    @@unique([provider, providerAccountId])
}

model Session {
    id           String   @id @default(uuid()) @db.Uuid
    sessionToken String   @unique
    userId       String   @db.Uuid
    expires      DateTime
    user         User     @relation(fields: [userId], references: [id], onDelete: Cascade)
}

enum UserRole {
    ADMIN
    USER
}

model User {
    id            String    @id @default(uuid()) @db.Uuid
    name          String?
    email         String?   @unique
    emailVerified DateTime?
    image         String?

    role UserRole @default(USER)

    accounts          Account[]
    sessions          Session[]
    projectUsers      ProjectUser[]
    projects          Project[]
    worldChampEntrant WorldChampEntrant?

    createdAt DateTime @default(now())
    updatedAt DateTime @default(now()) @updatedAt
}

model VerificationToken {
    identifier String
    token      String   @unique
    expires    DateTime

    @@unique([identifier, token])
}