26b6fa4f0c
* Continue polling stats until all evals complete * Return evaluation changes early, before it has run * Add task for running new eval * requeue rate-limited tasks * Fix prettier
18 lines
562 B
TypeScript
18 lines
562 B
TypeScript
import { runAllEvals } from "../utils/evaluations";
|
|
import defineTask from "./defineTask";
|
|
|
|
export type RunNewEvalJob = {
|
|
experimentId: string;
|
|
};
|
|
|
|
// When a new eval is created, we want to run it on all existing outputs, but return the new eval first
|
|
export const runNewEval = defineTask<RunNewEvalJob>("runNewEval", async (task) => {
|
|
console.log("RUNNING TASK", task);
|
|
const { experimentId } = task;
|
|
await runAllEvals(experimentId);
|
|
});
|
|
|
|
export const queueRunNewEval = async (experimentId: string) => {
|
|
await runNewEval.enqueue({ experimentId });
|
|
};
|