alihan/OpenPipe-llm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ensure evals run properly (#96)

Browse Source 
* Run evals against llama output

* Continue polling in OutputCell until evals complete

* Remove unnecessary check
This commit is contained in:
arcticfly
2023-07-25 20:01:58 -07:00
committed by GitHub
parent 98b231c8bd
commit d4fb8b689a
5 changed files with 66 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/server/tasks/queryModel.task.ts
View File

@@ -99,7 +99,7 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
const inputHash = hashPrompt(prompt);
for (let i = 0; true; i++) {
const modelResponse = await prisma.modelResponse.create({
let modelResponse = await prisma.modelResponse.create({
data: {
inputHash,
scenarioVariantCellId: cellId,
@@ -108,7 +108,7 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
});
const response = await provider.getCompletion(prompt.modelInput, onStream);
if (response.type === "success") {
await prisma.modelResponse.update({
modelResponse = await prisma.modelResponse.update({
where: { id: modelResponse.id },
data: {
output: response.value as Prisma.InputJsonObject,
@@ -127,7 +127,7 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
},
});
await runEvalsForOutput(variant.experimentId, scenario, modelResponse);
await runEvalsForOutput(variant.experimentId, scenario, modelResponse, prompt.modelProvider);
break;
} else {
const shouldRetry = response.autoRetry && i < MAX_AUTO_RETRIES;