Run workers in a separate Docker container

We've outgrown the run-everything-on-one-machine setup. This change moves background jobs to a different Docker image in production. It also adds a `jobKey` to certain jobs so if we try to process the same cell multiple times it'll only actually run the job once.
This commit is contained in:
Kyle Corbitt
2023-08-18 11:16:00 -07:00
parent b1802fc04b
commit 10dd53e7f6
6 changed files with 24 additions and 13 deletions

View File

@@ -1,4 +1,4 @@
import { type Helpers, type Task, makeWorkerUtils } from "graphile-worker";
import { type Helpers, type Task, makeWorkerUtils, TaskSpec } from "graphile-worker";
import { env } from "~/env.mjs";
let workerUtilsPromise: ReturnType<typeof makeWorkerUtils> | null = null;
@@ -16,9 +16,11 @@ function defineTask<TPayload>(
taskIdentifier: string,
taskHandler: (payload: TPayload, helpers: Helpers) => Promise<void>,
) {
const enqueue = async (payload: TPayload, runAt?: Date) => {
const enqueue = async (payload: TPayload, spec?: TaskSpec) => {
console.log("Enqueuing task", taskIdentifier, payload);
await (await workerUtils()).addJob(taskIdentifier, payload, { runAt });
const utils = await workerUtils();
return await utils.addJob(taskIdentifier, payload, spec);
};
const handler = (payload: TPayload, helpers: Helpers) => {