How to upload image to S3 bucket locally and generating a thumbnail automatically?

Question

Good evening

I have this task. I have to upload an image to the S3 bucket using Node JS and generates a thumbnail on the go and not by using a lambda trigger. Everything should be done on my local machine terminal (or) in the local server(postman). I tried this code.

const fs = require('fs');

const ACESS_ID = 'A**********KV';
const SECRET_ID = 'G***********0';
const BUCKET_NAME = 'node-image-bucket';

// Initializing s3 interface
const s3 = new AWS.S3({
    accessKeyId: ACESS_ID,
    secretAccessKey: SECRET_ID,
});

// File reading function to S3
const uploadFile = (fileName) => {
    // Read content from the file
    const fileContent = fs.readFileSync(fileName);

    // Setting up S3 upload parameters
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'scene2.jpg',
        Body: fileContent
    };

    // Uploading files to the bucket
    s3.upload(params, function(err, data){
        if(err){
            throw err;
        }
        console.log(data);

        console.log(`File uploaded Successfully. ${data.Location}`);
    });
};


uploadFile('./images/bg-hd.jpg');

Above code is working fine with a single image and the problem is every time I upload a file to the S3 bucket I need to change the S3 params key string value

I want to upload multiple images at once and creating a buffer for performance and it should create thumbnails automatically in the same bucket at the different folder.

Could anyone help me, guys! Please Any help Appreciated!!!

Answer 1

You cannot upload multiple files with one s3 operation but you can use the sharp module before uploading https://www.npmjs.com/package/sharp to resize your image before calling the s3 api.

import * as sharp from 'sharp';

async function resize(buffer , width, height) {
  return sharp(buffer).resize(width, height).toBuffer();
}

const thumbnailWidthSize = 200;
const thumbnailWidthHeight = 200;
const thumbnailImage = await resize(fileContent, thumbnailWidthSize, thumbnailWidthHeight)

You can then reuse your current upload function and run it as many times as many image resizes you need with different keys and wrap those calls around promise.all to make the operation fail if any of the upload fails.

await promise.all([ 
  s3upload(image, imageKey), 
  s3upload(thumbnailImage, thumbnailImageKey) 
])

Answer 2

So, there are two parts to your questions -

• Converting the image to thumbnail on the fly while uploading to s3 bucket -

  You can use the thumbd npm module and create a thumbd server.

  Thumbd is an image thumbnailing server built on top of Node.js, SQS, S3, and ImageMagick.

  Prerequistes for the thumbd server - Thumbd requires the following environment variables to be set:

  • AWS_KEY the key for your AWS account (the IAM user must have access to the appropriate SQS and S3 resources).
  • AWS_SECRET the AWS secret key.
  • BUCKET the bucket to download the original images from. The thumbnails will also be placed in this bucket
  • AWS_REGION the AWS Region of the bucket. Defaults to: us-east-1.
  • CONVERT_COMMAND the ImageMagick convert command. Defaults to convert.
  • REQUEST_TIMEOUT how long to wait in milliseconds before aborting a remote request. Defaults to 15000.
  • S3_ACL the acl to set on the uploaded images. Must be one of private, or public-read. Defaults to private.
  • S3_STORAGE_CLASS the storage class for the uploaded images. Must be either STANDARD or REDUCED_REDUNDANCY. Defaults to STANDARD.
  • SQS_QUEUE the queue name to listen for image thumbnailing.
  • When running locally, I set these environment variables in a .env file and execute thumbd using pm2/forever/foreman.

  Setup -

  • apt-get install imagemagick
  • npm install thumbd -g
  • thumbd install
  • thumbd start // Run thumbd as a server

After the thumbd server is up and running, refer the code below to change image to thumbnail while uploading to s3 bucket.

    var aws = require('aws-sdk');
    var url = require("url");
    var awsS3Config = { 
        accessKeyId: ACESS_ID,
        secretAccessKey: SECRET_ID,
        region: 'us-west-2' 
        }


    var BUCKET_NAME = 'node-image-bucket';
    var sourceImageDirectory = "/tmp/"
    var imageUploadDir = "/thumbnails/"
    var imageName = 'image.jpg'
    var uploadImageName = 'image.jpg'

    aws.config.update(awsS3Config);
    var s3 = new aws.S3();

    var Client = require('thumbd').Client,
        client = new Client({
            awsKey: awsS3Config.accessKeyId,
            awsSecret: awsS3Config.secretAccessKey,
            s3Bucket: BUCKET_NAME,
            sqsQueue: 'ThumbnailCreator',
            awsRegion: awsS3Config.region,
            s3Acl: 'public-read'
        });

    export function uploadAndResize(sourceImageDirectory, imageName, imageUploadDir, uploadImageName) {

       return new Promise((resolve, reject)=>{

            client.upload(sourceImageDirectory + imageName, imageUploadDir + uploadImageName, function(err) {

            if (err) {

                reject(err);
            } else {
                client.thumbnail(imageUploadDir + uploadImageName, [{
                    "suffix": "medium",
                    "width": 360,
                    "height": 360,
                    "background": "white",
                    "strategy": "%(command)s %(localPaths[0])s -resize %(width)sX%(height)s^ -gravity north -extent %(width)sX%(height)s  %(convertedPath)s"
                }, {
                    "suffix": "thumb",
                    "width": 100,
                    "height": 100,
                    "background": "white",
                    "strategy": "%(command)s %(localPaths[0])s -resize %(width)sX%(height)s^ -gravity north -extent %(width)sX%(height)s  %(convertedPath)s"
                }], {
                    //notify: 'https://callback.example.com'
                });
                var response = {};

                //https://s3-ap-us-west-2.amazonaws.com/node-image-bucket/1/5825c7d0-127f-4dac-b802-ca24efba2bcd-original.jpeg

                response.url = 'https://s3-' + awsS3Config.region + '.amazonaws.com/' + BUCKET_NAME + '/' + imageUploadDir;
                response.uploadImageName = uploadImageName;
                response.sourceImageName = imageName;

                 resolve(response);

                }

            })
       })

2. Second, you wanted to upload multiple images without changing the string -

   Loop over the below method for all the files in a localpath and you are good to go.

export function uploadFiles(localPath, localFileName, fileUploadDir, uploadFileName) {

  return new Promise((resolve, reject) => {

    fs.readFile(localPath + localFileName, function (err, file) {
      if (err) {

        reject(err);
      }

      var params = {
        ACL: 'public-read',
        Bucket: BUCKET_NAME,
        Key: uploadFileName,
        Body: file
      };

      s3.upload(params, function (err, data) {

        fs.unlink(localPath + localFileName, function (err) {

          if (err) {
            reject(err);
          } else {
            resolve(data)
          }
        });
      });

    });

  })

}