How to use batchWriteItem to write more than 25 items into DynamoDB Table using PHP

Question

I am using AWS SDK for PHP 3.x

A single call to BatchWriteItem can write up to 16 MB of data, which can comprise as many as 25 put or delete requests. Individual items to be written can be as large as 400 KB.

  $result = $dynamodbClient->batchWriteItem([
  'RequestItems' => [
    $tableName => [
      [
        'PutRequest' => [
          'Item' => [
            'Id' => ['N' => '1'],
            'AlbumTitle' => [
              'S' => 'Somewhat Famous',
            ],
            'Artist' => [
              'S' => 'No One You Know',
            ],
            'SongTitle' => [
              'S' => 'Call Me Today',
            ],
          ],
        ],
      ],          
    ],
  ],
]);

For single item its working fine. How can I write more than 25 items.

Answer 1

To write more than 25 items, you have to repeatedly call BatchWriteItem, adding items from your collection, 25 at a time.

Something along these lines (pseudo-code):

requests = []; // use an array to stage your put item requests
foreach(item in SourceCollection) {
    addItem(item, requests); // add this item to the array 
    if(count(requests) == 25) { // when you have 25 ready..
       // result = dynamodbClient->batchWriteItem(...)
       requests = []; // clean up the array of put item requests
       // handle the failed items from the result object
    }
}

Make sure to handle failed items from each batchWriteItem result by re-adding them back to the requests

Answer 2

Here is my way for a lambda function:

exports.handler = (event, context, callback) => {
  console.log(`EVENT: ${JSON.stringify(event)}`);

  var AWS = require('aws-sdk');

  AWS.config.update({ region: process.env.REGION })

  var docClient = new AWS.DynamoDB.DocumentClient();

  const {data, table, cb} = JSON.parse(event.body);

  console.log('{data, table, cb}:', {data, table, cb});

  // Build the batches
  var batches = [];
  var current_batch = [];
  var item_count = 0;

  for (var i = 0; i < data.length; i++) {
    // Add the item to the current batch
    item_count++
    current_batch.push({
      PutRequest: {
        Item: data[i],
      },
    })
    // If we've added 25 items, add the current batch to the batches array
    // and reset it
    if (item_count % 25 === 0) {
      batches.push(current_batch)
      current_batch = []
    }
  }

  // Add the last batch if it has records and is not equal to 25
  if (current_batch.length > 0 && current_batch.length !== 25) {
    batches.push(current_batch)
  }

  // Handler for the database operations
  var completed_requests = 0
  var errors = false

  function requestHandler (request) {

    console.log('in the handler: ', request)

    return function (err, data) {
      // Increment the completed requests
      completed_requests++;

      // Set the errors flag
      errors = (errors) ? true : err;

      // Log the error if we got one
      if(err) {
        console.error(JSON.stringify(err, null, 2));
        console.error("Request that caused database error:");
        console.error(JSON.stringify(request, null, 2));
        callback(err);
      }else {
        var response = {
          statusCode: 200,
          headers: {
            'Content-Type': 'application/json',
            'Access-Control-Allow-Methods': 'GET,POST,OPTIONS',
            'Access-Control-Allow-Origin': '*',
            'Access-Control-Allow-Credentials': true
          },
          body: JSON.stringify(data),
          isBase64Encoded: false
        };
        console.log(`success: returned ${data}`);
        callback(null, response);
      }

      // Make the callback if we've completed all the requests
      if(completed_requests === batches.length) {
        cb(errors);
      }
    }
  }

  // Make the requests
  var params;
  for (var j = 0; j < batches.length; j++) {
    // Items go in params.RequestItems.id array
    // Format for the items is {PutRequest: {Item: ITEM_OBJECT}}
    params = '{"RequestItems": {"' + table + '": []}}'
    params = JSON.parse(params)
    params.RequestItems[table] = batches[j]

    console.log('before db.batchWrite: ', params)

    // Perform the batchWrite operation
    docClient.batchWrite(params, requestHandler(params))
  }
};


dealspoondBatchWrite