Read JSON from S3 file and Insert records into dynamoDB using Lambda with NodeJS runtime

Question

My DynamoDB table has the following partition key:

UserId  String

This is my Lambda function:

'use strict';

console.log('Loading function');
const doc = require('dynamodb-doc');
const dynamo = new doc.DynamoDB();

var AWS = require('aws-sdk');

var S3 = new AWS.S3({
    maxRetries: 0,
    region: 'us-east-1',
});

var insertSuccess = 0;
var insertErrors = 0;

function dynamoResultCallback(err, data) {
    if (err) {
        insertErrors++;
        console.log("Insert Error: \n");
        console.log(err, err.stack); // an error occurred
    } else {
        insertSuccess++;
    }
}

exports.handler = (event, context, callback) => {
    console.log('Received event:', JSON.stringify(event, null, 2));
    console.log("Init complete, running.. \n")

    var srcBucket = event.Records[0].s3.bucket.name;
    var srcKey = event.Records[0].s3.object.key;

    console.log("Params: srcBucket: " + srcBucket + " srcKey: " + srcKey + "\n")

    S3.getObject({
        Bucket: srcBucket,
        Key: srcKey,
    }, function(err, data) {
        if (err !== null) {
            return callback(err, null);
        }
        var fileData = data.Body.toString('utf-8');
        var recordsArray = fileData.split("\n");

        for (var i = 0; i < recordsArray.length; i++) {
             var record = recordsArray[i];
              console.log("Inserting record: " + record);

            var params = {
                Item: record,
                ReturnConsumedCapacity: "TOTAL",
                TableName: "PacketData"
            };
            dynamo.putItem(params, dynamoResultCallback);
        }
        console.log("Insert Result -- successCount: " + insertSuccess + " errorCount: " + insertErrors)
        return callback(null, data);
    });
};

What's happening currently is that, it reads the S3 file as expected, but I'm not able to pass the resulting records into dynamo, the cloud watchlog error is that it didn't get passed a UserId (the required parition key) despite it being in the data. The records print fine in CloudWatch, and actually if I replace the record variable on this line:

Item: record,

With the JSON string from CloudWatch it works as expected. Any suggestions?

P.S. I'm a JavaScript novice if that wasn't already apparent, but I believe the problem is due to how the variable record is being interpreted at runtime as it works fine if I replace that variable with it's value as a hard coded string.

Answer 1

Use JSON.parse(record);

According to the dynamodb-doc documentation it putItem method expects an object rather a string.