Add smoketest (broken)

app/.gitignore

@@ -1,9 +1,11 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
+# App files
+
 # dependencies
 /node_modules
 /.pnp
-.pnp.js
+/.pnp.js
 
 # testing
 /coverage
@@ -15,28 +17,28 @@
 # next.js
 /.next/
 /out/
-next-env.d.ts
+/next-env.d.ts
 
 # production
 /build
 
 # misc
-.DS_Store
-*.pem
+/.DS_Store
+/*.pem
 
 # debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-.pnpm-debug.log*
+/npm-debug.log*
+/yarn-debug.log*
+/yarn-error.log*
+/.pnpm-debug.log*
 
 # local env files
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
-.env
-.env*.local
+/.env
+/.env*.local
 
 # vercel
-.vercel
+/.vercel
 
 # typescript
-*.tsbuildinfo
+/*.tsbuildinfo

app/@types/nextjs-routes.d.ts

@@ -12,7 +12,9 @@ declare module "nextjs-routes" {
 
   export type Route =
     | StaticRoute<"/account/signin">
+    | DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
+    | StaticRoute<"/api/openapi">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
     | DynamicRoute<"/experiments/[id]", { "id": string }>
     | StaticRoute<"/experiments">

app/openapitools.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json",
+  "spaces": 2,
+  "generator-cli": {
+    "version": "6.6.0"
+  }
+}

app/package.json

@@ -16,7 +16,7 @@
     "postinstall": "prisma generate",
     "lint": "next lint",
     "start": "next start",
-    "codegen": "tsx src/codegen/export-openai-types.ts",
+    "codegen": "tsx src/codegen/export-client-types.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'"
   },
@@ -61,6 +61,7 @@
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
     "next-query-params": "^4.2.3",
+    "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
     "openai": "4.0.0-beta.2",
     "pluralize": "^8.0.0",
@@ -79,6 +80,7 @@
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
     "superjson": "1.12.2",
+    "trpc-openapi": "^1.2.0",
     "tsx": "^3.12.7",
     "type-fest": "^4.0.0",
     "use-query-params": "^2.2.1",

app/pnpm-lock.yaml

@@ -1,4 +1,4 @@
-lockfileVersion: '6.1'
+lockfileVersion: '6.0'
 
 settings:
   autoInstallPeers: true
@@ -125,6 +125,9 @@ dependencies:
   next-query-params:
     specifier: ^4.2.3
     version: 4.2.3(next@13.4.2)(react@18.2.0)(use-query-params@2.2.1)
+  nextjs-cors:
+    specifier: ^2.1.2
+    version: 2.1.2(next@13.4.2)
   nextjs-routes:
     specifier: ^2.0.1
     version: 2.0.1(next@13.4.2)
@@ -179,6 +182,9 @@ dependencies:
   superjson:
     specifier: 1.12.2
     version: 1.12.2
+  trpc-openapi:
+    specifier: ^1.2.0
+    version: 1.2.0(@trpc/server@10.26.0)(zod@3.21.4)
   tsx:
     specifier: ^3.12.7
     version: 3.12.7
@@ -3861,6 +3867,15 @@ packages:
       wrap-ansi: 7.0.0
     dev: false
 
+  /co-body@6.1.0:
+    resolution: {integrity: sha512-m7pOT6CdLN7FuXUcpuz/8lfQ/L77x8SchHCF4G0RBTJO20Wzmhn5Sp4/5WsKy8OSpifBSUrmg83qEqaDHdyFuQ==}
+    dependencies:
+      inflation: 2.0.0
+      qs: 6.11.2
+      raw-body: 2.5.1
+      type-is: 1.6.18
+    dev: false
+
   /color-convert@1.9.3:
     resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==}
     dependencies:
@@ -3935,6 +3950,10 @@ packages:
   /convert-source-map@1.9.0:
     resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==}
 
+  /cookie-es@1.0.0:
+    resolution: {integrity: sha512-mWYvfOLrfEc996hlKcdABeIiPHUPC6DM2QYZdGGOvhOTbA3tjm2eBwqlJpoFdjC89NI4Qt6h0Pu06Mp+1Pj5OQ==}
+    dev: false
+
   /cookie-signature@1.0.6:
     resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==}
     dev: false
@@ -4137,11 +4156,20 @@ packages:
       has-property-descriptors: 1.0.0
       object-keys: 1.1.1
 
+  /defu@6.1.2:
+    resolution: {integrity: sha512-+uO4+qr7msjNNWKYPHqN/3+Dx3NFkmIzayk2L1MyZQlvgZb/J1A0fo410dpKrN2SnqFjt8n4JL8fDJE0wIgjFQ==}
+    dev: false
+
   /delayed-stream@1.0.0:
     resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
     engines: {node: '>=0.4.0'}
     dev: false
 
+  /depd@1.1.2:
+    resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==}
+    engines: {node: '>= 0.6'}
+    dev: false
+
   /depd@2.0.0:
     resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
     engines: {node: '>= 0.8'}
@@ -4152,6 +4180,10 @@ packages:
     engines: {node: '>=6'}
     dev: true
 
+  /destr@2.0.0:
+    resolution: {integrity: sha512-FJ9RDpf3GicEBvzI3jxc2XhHzbqD8p4ANw/1kPsFBfTvP1b7Gn/Lg1vO7R9J4IVgoMbyUmFrFGZafJ1hPZpvlg==}
+    dev: false
+
   /destroy@1.2.0:
     resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==}
     engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@@ -5273,6 +5305,18 @@ packages:
       - pg-native
     dev: false
 
+  /h3@1.7.1:
+    resolution: {integrity: sha512-A9V2NEDNHet7v1gCg7CMwerSigLi0SRbhTy7C3lGb0N4YKIpPmLDjedTUopqp4dnn7COHfqUjjaz3zbtz4QduA==}
+    dependencies:
+      cookie-es: 1.0.0
+      defu: 6.1.2
+      destr: 2.0.0
+      iron-webcrypto: 0.7.1
+      radix3: 1.0.1
+      ufo: 1.1.2
+      uncrypto: 0.1.3
+    dev: false
+
   /has-bigints@1.0.2:
     resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==}
     dev: true
@@ -5400,6 +5444,11 @@ packages:
     engines: {node: '>=0.8.19'}
     dev: true
 
+  /inflation@2.0.0:
+    resolution: {integrity: sha512-m3xv4hJYR2oXw4o4Y5l6P5P16WYmazYof+el6Al3f+YlggGj6qT9kImBAnzDelRALnP5d3h4jGBPKzYCizjZZw==}
+    engines: {node: '>= 0.8.0'}
+    dev: false
+
   /inflight@1.0.6:
     resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==}
     dependencies:
@@ -5429,6 +5478,10 @@ packages:
     engines: {node: '>= 0.10'}
     dev: false
 
+  /iron-webcrypto@0.7.1:
+    resolution: {integrity: sha512-K/UmlEhPCPXEHV5hAtH5C0tI5JnFuOrv4yO/j7ODPl3HaiiHBLbOLTde+ieUaAyfCATe4LoAnclyF+hmSCOVmQ==}
+    dev: false
+
   /is-alphabetical@1.0.4:
     resolution: {integrity: sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==}
     dev: false
@@ -6134,6 +6187,15 @@ packages:
       - babel-plugin-macros
     dev: false
 
+  /nextjs-cors@2.1.2(next@13.4.2):
+    resolution: {integrity: sha512-2yOVivaaf2ILe4f/qY32hnj3oC77VCOsUQJQfhVMGsXE/YMEWUY2zy78sH9FKUCM7eG42/l3pDofIzMD781XGA==}
+    peerDependencies:
+      next: ^8.1.1-canary.54 || ^9.0.0 || ^10.0.0-0 || ^11.0.0 || ^12.0.0 || ^13.0.0
+    dependencies:
+      cors: 2.8.5
+      next: 13.4.2(@babel/core@7.22.9)(react-dom@18.2.0)(react@18.2.0)
+    dev: false
+
   /nextjs-routes@2.0.1(next@13.4.2):
     resolution: {integrity: sha512-pBGRm6uR44zwUjWWYn6+gwz08BhBbqUYlIzsbNHAh1TWohHYKWFaa2YVsj8BxEo726MZYg87OJPnHpaaY1ia0w==}
     hasBin: true
@@ -6161,6 +6223,22 @@ packages:
       whatwg-url: 5.0.0
     dev: false
 
+  /node-mocks-http@1.12.2:
+    resolution: {integrity: sha512-xhWwC0dh35R9rf0j3bRZXuISXdHxxtMx0ywZQBwjrg3yl7KpRETzogfeCamUIjltpn0Fxvs/ZhGJul1vPLrdJQ==}
+    engines: {node: '>=0.6'}
+    dependencies:
+      accepts: 1.3.8
+      content-disposition: 0.5.4
+      depd: 1.1.2
+      fresh: 0.5.2
+      merge-descriptors: 1.0.1
+      methods: 1.1.2
+      mime: 1.6.0
+      parseurl: 1.3.3
+      range-parser: 1.2.1
+      type-is: 1.6.18
+    dev: false
+
   /node-releases@2.0.13:
     resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==}
 
@@ -6322,6 +6400,10 @@ packages:
       - supports-color
     dev: false
 
+  /openapi-types@12.1.3:
+    resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==}
+    dev: false
+
   /openapi-typescript@5.4.1:
     resolution: {integrity: sha512-AGB2QiZPz4rE7zIwV3dRHtoUC/CWHhUjuzGXvtmMQN2AFV8xCTLKcZUHLcdPQmt/83i22nRE7+TxXOXkK+gf4Q==}
     engines: {node: '>= 14.0.0'}
@@ -6745,6 +6827,10 @@ packages:
     resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
     dev: true
 
+  /radix3@1.0.1:
+    resolution: {integrity: sha512-y+AcwZ3HcUIGc9zGsNVf5+BY/LxL+z+4h4J3/pp8jxSmy1STaCocPS3qrj4tA5ehUSzqtqK+0Aygvz/r/8vy4g==}
+    dev: false
+
   /randombytes@2.1.0:
     resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==}
     dependencies:
@@ -7645,6 +7731,22 @@ packages:
     hasBin: true
     dev: false
 
+  /trpc-openapi@1.2.0(@trpc/server@10.26.0)(zod@3.21.4):
+    resolution: {integrity: sha512-pfYoCd/3KYXWXvUPZBKJw455OOwngKN/6SIcj7Yit19OMLJ+8yVZkEvGEeg5wUSwfsiTdRsKuvqkRPXVSwV7ew==}
+    peerDependencies:
+      '@trpc/server': ^10.0.0
+      zod: ^3.14.4
+    dependencies:
+      '@trpc/server': 10.26.0
+      co-body: 6.1.0
+      h3: 1.7.1
+      lodash.clonedeep: 4.5.0
+      node-mocks-http: 1.12.2
+      openapi-types: 12.1.3
+      zod: 3.21.4
+      zod-to-json-schema: 3.21.4(zod@3.21.4)
+    dev: false
+
   /tsconfck@2.1.2(typescript@5.0.4):
     resolution: {integrity: sha512-ghqN1b0puy3MhhviwO2kGF8SeMDNhEbnKxjK7h6+fvY9JAxqvXi8y5NAHSQv687OVboS2uZIByzGd45/YxrRHg==}
     engines: {node: ^14.13.1 || ^16 || >=18}
@@ -7782,7 +7884,6 @@ packages:
 
   /ufo@1.1.2:
     resolution: {integrity: sha512-TrY6DsjTQQgyS3E3dBaOXf0TpPD8u9FVrVYmKVegJuFw51n/YB9XPt+U6ydzFG5ZIN7+DIjPbNmXoBj9esYhgQ==}
-    dev: true
 
   /unbox-primitive@1.0.2:
     resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==}
@@ -7793,6 +7894,10 @@ packages:
       which-boxed-primitive: 1.0.2
     dev: true
 
+  /uncrypto@0.1.3:
+    resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==}
+    dev: false
+
   /undici@5.22.1:
     resolution: {integrity: sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==}
     engines: {node: '>=14.0'}
@@ -8305,6 +8410,14 @@ packages:
     engines: {node: '>=12.20'}
     dev: true
 
+  /zod-to-json-schema@3.21.4(zod@3.21.4):
+    resolution: {integrity: sha512-fjUZh4nQ1s6HMccgIeE0VP4QG/YRGPmyjO9sAh890aQKPEk3nqbfUXhMFaC+Dr5KvYBm8BCyvfpZf2jY9aGSsw==}
+    peerDependencies:
+      zod: ^3.21.4
+    dependencies:
+      zod: 3.21.4
+    dev: false
+
   /zod@3.21.4:
     resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==}
     dev: false

app/prisma/migrations/20230730061804_add_logged_calls_and_data_flows/migration.sql

@@ -0,0 +1,64 @@
+-- AlterTable
+ALTER TABLE "Experiment" ADD COLUMN     "dataFlowId" UUID;
+
+-- AlterTable
+ALTER TABLE "ModelResponse" ADD COLUMN     "loggedCallId" UUID,
+ALTER COLUMN "scenarioVariantCellId" DROP NOT NULL;
+
+-- CreateTable
+CREATE TABLE "LoggedCall" (
+    "id" UUID NOT NULL,
+    "promptFunctionHash" TEXT NOT NULL,
+    "promptFunction" TEXT NOT NULL,
+    "prompt" JSONB NOT NULL,
+    "responsePayload" JSONB,
+    "scenarioVariables" JSONB NOT NULL,
+    "model" TEXT NOT NULL,
+    "modelProvider" TEXT NOT NULL,
+    "dataFlowId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCall_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "DataFlow" (
+    "id" UUID NOT NULL,
+    "label" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "DataFlow_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "ApiKey" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "key" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "ApiKey_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ApiKey_key_key" ON "ApiKey"("key");
+
+-- AddForeignKey
+ALTER TABLE "Experiment" ADD CONSTRAINT "Experiment_dataFlowId_fkey" FOREIGN KEY ("dataFlowId") REFERENCES "DataFlow"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ModelResponse" ADD CONSTRAINT "ModelResponse_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_dataFlowId_fkey" FOREIGN KEY ("dataFlowId") REFERENCES "DataFlow"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "DataFlow" ADD CONSTRAINT "DataFlow_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/schema.prisma

@@ -19,6 +19,9 @@ model Experiment {
     organizationId String        @db.Uuid
     organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
 
+    dataFlowId String?     @db.Uuid
+    dataFlow   DataFlow?   @relation(fields: [dataFlowId], references: [id], onDelete: Cascade)
+
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
@@ -127,10 +130,13 @@ model ModelResponse {
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
 
-    scenarioVariantCellId String              @db.Uuid
-    scenarioVariantCell   ScenarioVariantCell @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
+    scenarioVariantCellId String?              @db.Uuid
+    scenarioVariantCell   ScenarioVariantCell? @relation(fields: [scenarioVariantCellId], references: [id], onDelete: Cascade)
     outputEvaluations      OutputEvaluation[]
 
+    loggedCallId String? @db.Uuid
+    loggedCall   LoggedCall? @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
+
     @@index([inputHash])
 }
 
@@ -174,10 +180,57 @@ model OutputEvaluation {
     @@unique([modelResponseId, evaluationId])
 }
 
+model LoggedCall {
+    id String @id @default(uuid()) @db.Uuid
+
+    promptFunctionHash String
+    promptFunction     String
+    prompt     Json
+    responsePayload    Json?
+    scenarioVariables Json
+    model              String
+    modelProvider      String
+
+    dataFlowId String     @db.Uuid
+    dataFlow   DataFlow   @relation(fields: [dataFlowId], references: [id], onDelete: Cascade)
+
+    modelResponse      ModelResponse[]
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
+model DataFlow {
+    id String @id @default(uuid()) @db.Uuid
+
+    label String
+    experiments   Experiment[]
+    loggedCalls      LoggedCall[]
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
+model ApiKey {
+    id        String  @id @default(uuid()) @db.Uuid
+
+    name      String
+    key     String  @unique
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
 model Organization {
     id                String  @id @default(uuid()) @db.Uuid
     personalOrgUserId String? @unique @db.Uuid
-    PersonalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+    personalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+    apiKeys         ApiKey[]
+    dataFlows         DataFlow[]
 
     createdAt        DateTime           @default(now())
     updatedAt        DateTime           @updatedAt

app/src/codegen/export-client-types.ts

@@ -0,0 +1,42 @@
+import "dotenv/config";
+import { openApiDocument } from "~/pages/api/openapi.json";
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+import { generatePromptTypes } from "~/modelProviders/generatePromptTypes";
+
+console.log("Exporting public OpenAPI schema to client-libs/schema.json");
+
+const executionDir = import.meta.url.replace("file://", "");
+const schemaPath = path.join(
+  path.dirname(executionDir),
+  "../../../client-libs/schema.json"
+);
+
+console.log("Exporting schema");
+fs.writeFileSync(schemaPath, JSON.stringify(openApiDocument, null, 2), "utf-8");
+
+console.log("Generating Typescript client");
+
+const tsClientPath = path.join(
+  path.dirname(executionDir),
+  "../../../client-libs/js/codegen"
+);
+
+fs.rmSync(tsClientPath, { recursive: true, force: true });
+
+execSync(
+  `pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
+  {
+    stdio: "inherit",
+  }
+);
+
+const promptTypes = await generatePromptTypes();
+
+const promptTypesPath = path.join(tsClientPath, "promptTypes.ts");
+
+console.log(`Writing promptTypes to ${promptTypesPath}`);
+fs.writeFileSync(promptTypesPath, promptTypes, "utf-8");
+
+console.log("Done!");

app/src/modelProviders/generatePromptTypes.ts

@@ -0,0 +1,41 @@
+import { type JSONSchema4Object } from "json-schema";
+import modelProviders from "./modelProviders";
+import { compile } from "json-schema-to-typescript";
+import dedent from "dedent";
+
+export const declarePromptTypes = async () => {
+  const promptTypes = (await generatePromptTypes()).replace(
+    /export interface PromptTypes/g,
+    "interface PromptTypes",
+  );
+
+  return dedent`
+    ${promptTypes}
+  
+    declare function definePrompt<T extends keyof PromptTypes>(modelProvider: T, input: PromptTypes[T])
+    `;
+};
+
+export const generatePromptTypes = async () => {
+  const combinedSchema = {
+    type: "object",
+    properties: {} as Record<string, JSONSchema4Object>,
+  };
+
+  Object.entries(modelProviders).forEach(([id, provider]) => {
+    combinedSchema.properties[id] = provider.inputSchema;
+  });
+
+  Object.entries(modelProviders).forEach(([id, provider]) => {
+    combinedSchema.properties[id] = provider.inputSchema;
+  });
+
+  return await compile(combinedSchema as JSONSchema4Object, "PromptTypes", {
+    additionalProperties: false,
+    bannerComment: dedent`
+  /**
+   * This type map defines the input types for each model provider. 
+   */ 
+  `,
+  });
+};