How to copy/move all objects in Amazon S3 from one prefix to other using the AWS SDK for Node.js

Question

How do I copy all objects from one prefix to other? I have tried all possible ways to copy all objects in one shot from one prefix to other, but the only way that seems to work is by looping over a list of objects and copying them one by one. This is really inefficient. If I have hundreds of files in a folder, will I have to make 100 calls?

var params = {          Bucket: bucket,          CopySource: bucket+'/'+oldDirName+'/filename.txt',          Key: newDirName+'/filename.txt',  }; s3.copyObject(params, function(err, data) {   if (err) {       callback.apply(this, [{           type: "error",           message: "Error while renaming Directory",           data: err       }]);   } else {       callback.apply(this, [{           type: "success",           message: "Directory renamed successfully",           data: data       }]);   } }); 

Answer 1

You will need to make one AWS.S3.listObjects() to list your objects with a specific prefix. But you are correct in that you will need to make one call for every object that you want to copy from one bucket/prefix to the same or another bucket/prefix.

You can also use a utility library like async to manage your requests.

var AWS = require('aws-sdk'); var async = require('async'); var bucketName = 'foo'; var oldPrefix = 'abc/'; var newPrefix = 'xyz/'; var s3 = new AWS.S3({params: {Bucket: bucketName}, region: 'us-west-2'});  var done = function(err, data) {   if (err) console.log(err);   else console.log(data); };  s3.listObjects({Prefix: oldPrefix}, function(err, data) {   if (data.Contents.length) {     async.each(data.Contents, function(file, cb) {       var params = {         Bucket: bucketName,         CopySource: bucketName + '/' + file.Key,         Key: file.Key.replace(oldPrefix, newPrefix)       };       s3.copyObject(params, function(copyErr, copyData){         if (copyErr) {           console.log(copyErr);         }         else {           console.log('Copied: ', params.Key);           cb();         }       });     }, done);   } }); 

Hope this helps!