I want to write a small worker app in node.js.
This app would read from AWS-SQS, process the data in some way, and spit it out to another AWS-SQS.
So far I have written:
while(true){
readFromQueue()
.then(transform)
.then(writeToQueue);
}
function transform(data) {
console.log("> transforming...");
//transformation logic
return data;
}
//TODO: need to remove message from queue after read!
function readFromQueue() {
// var params = {
// QueueUrl: 'STRING_VALUE',
// WaitTimeSeconds: 2
// };
// return new Promise(function(resolve, reject) {
// sqs.receiveMessage(params, function(err, data) {
// if (err) reject(err);
// else resolve(data);
// });
// });
return new Promise(function(resolve, reject) {
console.log("> reading from queue...");
resolve({ data : "world" });
});
}
function writeToQueue(data) {
// var params = {
// MessageBody: data,
// QueueUrl: 'STRING_VALUE',
// };
// sqs.sendMessage(params, function(err, data) {
// if (err) console.log(err, err.stack);
// else console.log(data);
// });
console.log("> writing to queue...");
console.log(">> " + data);
}
As you can see everything is set up for AWS, but when I run it locally for the time being, I would just have some mock stuff inside, until I actually get my transformation logic tested etc...
The problems I have are:
So am I doing something wrong? I can understand that since promises are async, my while loop will go crazy and create thousands of them, so that concerns me... Nevertheless I want to initiate another loop once the previous read->transform->write has been finished. Is there some other pattern I should use here? Or just block and wait for readFromQueue to end...
--EDIT--
It does execute everything if not wrapped in while(true):
readFromQueue()
.then(transform)
.then(writeToQueue);
I also understand that since while(true) is being executed, it will essentially block the thread, and therefore the promise is not resolved. So is there a way around it?
I am falling back to my setInterval way. I know you said that reading queue has to start immediately after the writing is finished, but 10ms isn't much of delay if you ask me.
function someFunc(){
if(readingQueue) return;
readingQueue = true;
return readFromQueue()
.then(transform)
.then(writeToQueue)
.catch(someErrorHandler)
.then(function(){
readingQueue=false;
})
}
var readingQueue = false;
setInterval(someFunc, 10);
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With