Function killed. Error: memory limit exceeded [Thumbnail example]

Question

I was trying to use the thumbnail code from the google cloud functions example. However, I keep getting the following as a response.

        { code: 1, message: '`convert /tmp/users/ZSE7ZRkFGKZq0Bc3tESEG2uT8922/images/1489813436054.jpg -thumbnail 500x500> -limit memory 32MB /tmp/users/ZSE7ZRkFGKZq0Bc3tESEG2uT8922/images/thumbnail/thumb_1489813436054.jpg` failed with code 1',
  childProcess: 
   ChildProcess {
     domain: null,
     _events: { error: [Function: t], close: [Function] },
     _eventsCount: 2,
     _maxListeners: undefined,
     _closesNeeded: 3,
     _closesGot: 3,
     connected: false,
     signalCode: null,
     exitCode: 1,
     killed: false,
     spawnfile: 'convert',
     _handle: null,
     spawnargs: 
      [ 'convert',
        '/tmp/users/ZSE7ZRkFGKZq0Bc3tESEG2uT8922/images/1489813436054.jpg',
        '-thumbnail',
        '500x500>',
        '-limit',
        'memory 32MB',
        '/tmp/users/ZSE7ZRkFGKZq0Bc3tESEG2uT8922/images/thumbnail/thumb_1489813436054.jpg' ],
     pid: 14,
     stdin: 
      Socket {
        connecting: false,
        _hadError: false,
        _handle: null,
        _parent: null,
        _host: null,
        _readableState: [Object],
        readable: false,
        domain: null,
        _events: [Object],
        _eventsCount: 2,
        _maxListeners: undefined,
        _writableState: [Object],
        writable: false,
        allowHalfOpen: false,
        destroyed: true,
        _bytesDispatched: 0,
        _sockname: null,
        _writev: null,
        _pendingData: null,
        _pendingEncoding: '',
        server: null,
        _server: null,
        write: [Function: writeAfterFIN],
        _idleNext: null,
        _idlePrev: null,
        _idleTimeout: -1 },
     stdout: 
      Socket {
        connecting: false,
        _hadError: false,
        _handle: null,
        _parent: null,
        _host: null,
        _readableState: [Object],
        readable: false,
        domain: null,
        _events: [Object],
        _eventsCount: 3,
        _maxListeners: undefined,
        _writableState: [Object],
        writable: false,
        allowHalfOpen: false,
        destroyed: true,
        _bytesDispatched: 0,
        _sockname: null,
        _writev: null,
        _pendingData: null,
        _pendingEncoding: '',
        server: null,
        _server: null,
        _idleNext: null,
        _idlePrev: null,
        _idleTimeout: -1,
        write: [Function: writeAfterFIN] },
     stderr: 
      Socket {
        connecting: false,
        _hadError: false,
        _handle: null,
        _parent: null,
        _host: null,
        _readableState: [Object],
        readable: false,
        domain: null,
        _events: [Object],
        _eventsCount: 3,
        _maxListeners: undefined,
        _writableState: [Object],
        writable: false,
        allowHalfOpen: false,
        destroyed: true,
        _bytesDispatched: 0,
        _sockname: null,
        _writev: null,
        _pendingData: null,
        _pendingEncoding: '',
        server: null,
        _server: null,
        read: [Function],
        _consuming: true,
        _idleNext: null,
        _idlePrev: null,
        _idleTimeout: -1,
        write: [Function: writeAfterFIN] },
     stdio: [ [Object], [Object], [Object] ] },
  toString: [Function: toString] }

My code...

'use strict';

const functions = require('firebase-functions');
const mkdirp = require('mkdirp-promise');
const gcs = require('@google-cloud/storage')();
const spawn = require('child-process-promise').spawn;
const LOCAL_TMP_FOLDER = '/tmp/';

// Max height and width of the thumbnail in pixels.
const THUMB_MAX_HEIGHT = 500;
const THUMB_MAX_WIDTH = 500;
// Thumbnail prefix added to file names.
const THUMB_PREFIX = 'thumb_';
const uploadDir = 'thumbnail/'; // Not a great var name...

/**
 * When an image is uploaded in the Storage bucket We generate a thumbnail automatically using
 * ImageMagick.
 */
exports.generateThumbnail = functions.storage.object().onChange(event => {
  const filePath = event.data.name;
  const filePathSplit = filePath.split('/');
  const fileName = filePathSplit.pop();
  const fileDir = filePathSplit.join('/') + (filePathSplit.length > 0 ? '/' : '');
  const thumbFilePath = `${fileDir}${uploadDir}${THUMB_PREFIX}${fileName}`;
  const tempLocalDir = `${LOCAL_TMP_FOLDER}${fileDir}`;
  const tempLocalFile = `${tempLocalDir}${fileName}`;
  const tempLocalThumbFile = `${LOCAL_TMP_FOLDER}${thumbFilePath}`;

  // Exit if this is triggered on a file that is not an image.
  if (!event.data.contentType.startsWith('image/')) {
    console.log('This is not an image.');
    return;
  }

  // Exit if the image is already a thumbnail.
  if (fileName.startsWith(THUMB_PREFIX)) {
    console.log('Already a Thumbnail.');
    return;
  }

  // Exit if this is a move or deletion event.
  if (event.data.resourceState === 'not_exists') {
    console.log('This is a deletion event.');
    return;
  }

  // Create the temp directory where the storage file will be downloaded.
  return mkdirp(tempLocalDir).then(() => {
    // Download file from bucket.
    const bucket = gcs.bucket(event.data.bucket);
    return bucket.file(filePath).download({
      destination: tempLocalFile
    }).then(() => {
      console.log('The file has been downloaded to', tempLocalFile);
      // Generate a thumbnail using ImageMagick.
      return spawn('convert', [tempLocalFile, '-thumbnail', `${THUMB_MAX_WIDTH}x${THUMB_MAX_HEIGHT}>`, '-limit', 'memory 32MB', '-debug', 'all', tempLocalThumbFile]).then(() => {
        console.log('Thumbnail created at', tempLocalThumbFile);
        // Uploading the Thumbnail.
        return bucket.upload(tempLocalThumbFile, {
          destination: thumbFilePath
        }).then(() => {
          console.log('Thumbnail uploaded to Storage at', thumbFilePath);
        });
      });
    });
  });
});

I've tried a few different configurations, but nothing seems to work. Kind of at a loss here.

Image:

Answer 1

Apart from increasing your memory limit on the cloud function, you should also delete temporary files before your cloud function finishes. Please watch this recent official Firebase video on the subject: https://www.youtube.com/watch?v=2mjfI0FYP7Y

Files that are written to the temp file location is counted towards the memory usage of your cloud function.

The temporary files are not automatically cleaned up after a function finishes. The files will remain for as long as the server instance is around. So a cloud function that is frequently triggered can cause your server instance to run out of memory regardless of how much memory you allocated in your configuration.

At the top of your file, add a requirement for node filesystem module

const fs = require('fs');

And then in after you have done the file upload, you delete the local file.

return bucket.upload(tempLocalThumbFile, { destination: thumbFilePath })
  .then(() => {
    console.log('Thumbnail uploaded to Storage at', thumbFilePath);
    fs.unlinkSync(tempLocalThumbFile);
  });