Polling with promises using Q

Question

I have a similar situation as described in this blog post: Polling with promises. The author describes how promises were used to poll until a JobID was returned. I would like to convert this using Q.

I would gladly post code as a starting point but I'm not sure what to post. Ideally I am trying to chain promises together. I have been experimenting with Q.delay() but it does not seem to achieve my goal.

var promise = uploadFile();
promise.then(startImageProcessingJob)
.then(saveConvertedImage);

Can you please provide suggestions on how to create a promise that will continue to poll until the data is retrieved (or maximum tries are met).

Here is the author's code that uses bluebird.

var getJobStatusAsync = Promise.promisifyAll(api);

function poll(jobId, retry) {  
  if(!retry) retry = 5;
  if(!retry--) throw new Error('Too many retries');

  return getJobStatusAsync(jobId)
  .then(function(data) {
    if(data.state === 'error') throw new Error(data.error);
    if(data.state === 'finished') return data;
    return Promise.delay(jobId, 10000).then(poll);
  });

Edit:

In response to Traktor53's comment, I am adding the logic I have so far. I was trying to avoid adding extra code that causes bloat to the question.

The objective:

In my Angular application, I want to use ZamZar third party service for converting images to PNG format. My design implementation was to use promises to:

(1) upload the file from the client to the server (Node);

(2) Use the ZamZar API to start the image conversion (obtain the JobID);

(3) using the JobID, poll ZamZar API for status updates until the image is ready for download. Once image is ready I can obtain the fileId and download the file back to Node server.

(4) Once the PNG image is back on my server, I want to return the image to the client browser and place into an HTML canvas (using three.js and fabric.js).

/* Dependencies */
var express = require('express');
var request = require('request');
var formidable = require('formidable');
var randomstring = require("randomstring");
var fs = require('fs');
var Q = require('q');

/*
 * Helper functions in Node
 */
convertFileUtil = function() {

  /**
   * Step 1: upload file from client to node server.
   * formidable is used for file upload management. This means the file is
   * automatically uploaded to a temp directory. We are going to move the
   * uploaded file to our own temp directory for processing.
   * Return Type: A Promise is returned with payload containing the directory
   * path for the image file. This path is referenced in subsequent chained methods.
   */
  var uploadFileFromClientToNodeServer = function(req) {
    var q = Q.defer();
    var form = new formidable.IncomingForm();
    var tmpFolder = 'upload/' + randomstring.generate() + '/';

    //Use formidable to parse the file upload.
    form.parse(req, function(err, fields, files) {
      if (err) {
        console.log(err);
        throw err;
      }

      //When upload is successful, create a temp directory and MOVE file there.
      //Again, file is already uploaded. There is no need to use fs.writeFile* methods.
      mkdirp(tmpFolder, function (err) {
        if (err) {
          q.reject(err);
        } else {

          //File will be saved here.
          var tmpFileSavedLocation = tmpFolder + files.file.name;

          //Call fs.rename to MOVE file from formidable temp directory to our temp directory.
          fs.rename(files.file.path, tmpFileSavedLocation, function (err) {
            if (err) {
              q.reject(err);
            }
            console.log('File saved to directory:', tmpFileSavedLocation);
            q.resolve(tmpFileSavedLocation);
          });
        }
      });
    });

    return q.promise;
  };

  /**
   * Step 2: Post the temp file to zam zar. ZamZar is an API service that converts
   * images to a different file format. For example, when a user uploads an Adobe
   * Illustrator EPS file; the file is sent to zamzar for conversion to a PNG. all
   * image formats are returned as PNG which will be added to the canvas.
   * Return: This promise will return the JobID of our submission. The JobID will be
   * used in subsequent promise to retrieve the converted image.
   */
  var postTempFileToZamZar = function(filePath) {
    console.log('FilePath', filePath);
    var q = Q.defer();
    var formData = {
      target_format: 'png',
      source_file: fs.createReadStream(filePath),
    };
    //console.log('OK', formData);

    //Send file to zamzar for conversion.
    request.post({ url: 'https://sandbox.zamzar.com/v1/jobs/', formData: formData }, function (err, response, body) {
      if (err) {
        console.log('An error occurred', err);
        q.reject(err);
      } else {
        var jsonData = JSON.parse(body);
        console.log('SUCCESS! Conversion job started:', jsonData.id);

        //This object will be returned in promise payload.
        var returnObj = {
          filePath: filePath,
          jobId: jsonData.id,
        };

        console.log('Process complete. Returning: ', returnObj);
        q.resolve(returnObj);

        return q.promise;
      }

    }).auth(zamzarApiKey, '', true);
  };

  /*
   * Step 3: Poll for PNG.
   */
  var pollZamZarForOurPngFile = function(dataObj) {

    console.log('pollZamZarForOurPngFile', dataObj);
  }

  //API
  return {
    uploadFileFromClientToNodeServer: uploadFileFromClientToNodeServer,
    postTempFileToZamZar: postTempFileToZamZar,
    pollZamZarForOurPngFile: pollZamZarForOurPngFile,
  };
};

//Call to convert file.
app.post('/convertFile', function (req, res) {
  var util = convertFileUtil();

  //Get file data.
  var promise = util.uploadFileFromClientToNodeServer(req);
  promise
  .then(util.postTempFileToZamZar)
  .then(util.pollZamZarForOurPngFile);
  .then(function(data) {
    console.log('Done processing');
  });
});

Answer 1

A design idea which may be of interest:

1. write an onFulfill listener (pollZamZarForOurPngFile) for the promise which fulfills with returnObj.
2. This listener returns a Promise object.
3. The returned promise is fulfilled with returnObj (passing it down the chain) if zambar has finished conversion.
4. The returned promise is rejected if zamzar errors or too many retries have occured.

Note this leaves retrieving the file to the next (onFulfilled) listener in the promise chain. I've used Promise for convenience because node.js supports it and its Promise/Aplus compliant. Convert it to Q as you wish. The polling request code is straight off zamzar website and could be from a tutorial example - please check it.

function pollZamZarForOurPngFile( returnObj)
{   var jobID = returnObj.jobId;
    var resolve, reject;
    function unwrap( r, j) { resolve = r, reject = j};
    var promise = new Promise( unwrap);
    var maxRetry = 5;
    var firstDelay = 500;     // 1/2 second
    var retryDelay = 5000;    // 5 second?

    function checkIfFinished()
    {   // refer to https://developers.zamzar.com/docs under node.js tab for documentation

        request.get ('https://sandbox.zamzar.com/v1/jobs/' + jobID,
        function (err, response, body)
        {   if (err)
            {   reject( new Error("checkIfFinished: unable to get job"));
                return;
            }
            if( JSON.parse(body).status == "successful")
            {   resolve( returnObj);    // fulfill return promise with "returnObj" passed in; 
                return;
            }    
            // has not succeeded, need to retry
            if( maxRetry <= 0)
            {    reject( new Error("checkIfFinished: too many retries"));
            }
            else
            {   --maxRetry;
                setTimeout(checkIfFinished, retryDelay);
            }    
        }
    }).auth(apiKey, '', true);
    setTimeout(checkIfFinished, firstDelay);
    return promise;
}

Answer 2

A working version of the poll function using standard Promise/A+ only

function poll(jobId, retry) {
    if(!retry) retry = 5; // default retries = 5
    function delay(timeout) {
        return new Promise(function(fulfill) {
            setTimeout(function() {
                fulfill();
            }, timeout);
        });
    }
    function poller() {
        if(!retry--) throw new Error('Too many retries');

        return getJobStatusAsync(jobId)
        .then(function(data) {
            if (data.state === 'error') throw new Error(data.error);
            if (data.state === 'finished') return data;
            return delay(10000).then(poller);
        });
    }
    return poller();
};

I feel this code can probably be written far "better" ... but it's Saturday, so this is "weekend code", and it at least gives a far better jumping off point for the OP