Back to home

tl;dr
Queues with Kue.js, Node.js and Redis main image

Queues with Kue.js, Node.js and Redis

Reference these links:

https://medium.com/@NorbertdeLangen/communicating-between-nodejs-processes-4e68be42b917 https://medium.com/@niratattri/workers-and-node-kue-it-up-1c1215d9bddf https://github.com/Automattic/kue

tl;dr

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 const { fork } = require("child_process"); const kue = require("kue"); kue.app.listen(3050); const port = process.env.REDIS_PORT ? process.env.REDIS_PORT : "6379"; const host = process.env.REDIS_HOST ? process.env.REDIS_HOST : "127.0.0.1"; let queue = kue.createQueue({ redis: { host: host, port: port, }, }); queue.process("build", 1, (job, done) => { run(job, done); }); const run = async (job, done) => { try { const { data } = job.data; // process is a forked process const compute = fork("./processes/buildWeb.js"); compute.send(data); compute.on("message", (_) => { return done(); }); } catch (err) { console.error(err); return done(new Error(JSON.stringify(err))); } }; module.exports = (app) => { app.post("/", async function (req, res) { try { // Create a fork for a process const buildJob = queue .create("build", { // Job Type project: project, // Job Data data: req.body, }) .removeOnComplete(true) // REMOVE THE JOB FROM THE QUEUE ONCE IT'S COMPLETED .attempts(5) // The maximum number of retries you want the job to have .backoff({ delay: 60 * 1000, type: "exponential" }) // Time between retries. Read docs. .save(); // PERSIST THE DAMN JOB LOL buildJob.on("failed", function (errorMessage) { console.log("Job failed"); let error = JSON.parse(errorMessage); // error now contains the object passed from the worker when the job failed console.log(error); // Check it out for yourself // call pagerduty or whatever jazz you wanna do in case of failure }); res.status(200).send("Building " + project); } catch (err) { return res.status(500).send("Failed"); } }); }; // buildWeb.js const run = async (data) => { try { setTimeout(() => { process.send("Done"); }, 3000); } catch (err) { console.error(err); } }; process.on("message", (data) => { run(data); });

Dennis O'Keeffe

@dennisokeeffe92
  • Melbourne, Australia

Hi, I am a professional Software Engineer. Formerly of Culture Amp, UsabilityHub, Present Company and NightGuru.
I am currently working on workingoutloud.dev, Den Dribbles and LandPad .

Related articles


1,200+ PEOPLE ALREADY JOINED ❤️️

Get fresh posts + news direct to your inbox.

No spam. We only send you relevant content.