How to handle UnprocessedItems using AWS JavaScript SDK (dynamoDB)?

Question

I am trying to use an AWS Lambda function to handle events from SendGrid. As I understand it, the event will be an array with a variable number of JSON objects, each representing a given event. I want to write these events to DynamoDB using batchWriteItem and loop the process until I'm not returned any UnprocessedItems. However, I'm getting stuck in an infinite loop. Here is my code now:

console.log('Loading function');

var aws = require('aws-sdk');
var dynamo = new aws.DynamoDB();
params = {};

exports.handler = function(sg_event, context) {

    var items = [];
    for(var i = 0; i < sg_event.length; i++) {
        var obj = sg_event[i];
        var request = {
            PutRequest: {
                Item: {
                    email: { S: obj.email },
                    timestamp: { S: obj.timestamp.toString() },
                    sg_message_id: { S: obj.sg_message_id },
                    event: { S: obj.event }
                }
            }
        };
        items.push(request);
    }

    params = {
        RequestItems: {
            sendgrid_response: items
        }
    }

    do {
        dynamo.batchWriteItem( params, function(err, data) {
            if(err)
                context.fail(err);
            else
                params.RequestItems = data.UnprocessedItems;
        });
    } while(!isEmpty(params.RequestItems));
};

function isEmpty(obj) {
    return (Object.keys(obj).length === 0);
}

I think the problem is trying to set the params in the callback function, but I don't know how else I'm supposed to do it...I know I could just call another batchWriteItem using UnprocessedItems within the callback of the original one, but I still need to be able to run the function as many times as needed to ensure all UnprocessedItems are written. How can I loop the batchWriteItem correctly?

Answer 1

@Daniela Miao, Thanks for sharing the solution.

We can add one code block in your posted code which will avoid exception from DynamoDB. This will check if params.RequestItems has Unprocessed data before requesting the DynamoDB for batch write again.

//db is AWS.DynamoDB Client
var processItemsCallback = function(err, data) {
  if (err) { 
     //fail
  } else {
    var params = {};
    params.RequestItems = data.UnprocessedItems;
    /*
    * Added Code block 
    */
    if(Object.keys(params.RequestItems).length != 0) {
      db.batchWriteItem(params, processItemsCallback);
    }
  }
};

db.batchWriteItem(/*initial params*/, processItemsCallback);