node.js - Cache Image to Filesystem and Pipe Image to Response

Question

My goal is to construct a simple filesystem caching system for reducing the number of calls we need to make to an API for thumbnail images. The process is to check if the image already exists on the filesystem fs.stat and if not request the image from an API endpoint while at the same time writing the image to the file system. I was hoping I could pipe the request to both the file system and the response at the same time, but I don't believe that is possible so I first stream the response to the file system and then create a stream to pipe the image from the file system to the response object.

It's working well, but I have to believe that is a more efficient/optimized way to do this task in node.js. Any thoughts?

    function (req, res, next) {

        // Check to see if the image exists on the filesystem
        // TODO - stats will provide information on file creation date for cache checking
        fs.stat(pathToFile, function (err, stats) {
            if (err) {

                // If the image does not exist on the file system
                // Pipe the image to a file and then to the response object
                var req = request.get({
                    "uri": "http://www.example.com/image.png",
                    "headers": {
                        "Content-Type": "image/png"
                    }
                });

                // Create a write stream to the file system
                var stream = fs.createWriteStream(pathToFile);
                req.pipe(stream);
                stream.on('finish', function () {
                    fs.createReadStream(pathToFile)
                        .pipe(res);
                })
            }
            else {

                // If the image does exist on the file system, then stream the image to the response object
                fs.createReadStream(pathToFile)
                    .pipe(res);
            }
        })
    }

Answer 1

You can use a ThroughStream to accomplish this without having to wait for the entire file to be written to your file system. This works because the ThroughStream will internally buffer the data being piped into it.

var stream = require('stream')
function (req, res, next) {

    // Check to see if the image exists on the filesystem
    // TODO - stats will provide information on file creation date for cache checking
    fs.stat(pathToFile, function (err, stats) {
        if (err) {

            // If the image does not exist on the file system
            // Pipe the image to a file and the response object
            var req = request.get({
                "uri": "http://www.example.com/image.png",
                "headers": {
                    "Content-Type": "image/png"
                }
            });

            // Create a write stream to the file system
            req.pipe(
              new stream.PassThrough().pipe(
                fs.createWriteStream(pathToFile)
              )
            )

            // pipe to the response at the same time
            req.pipe(res)
        }
        else {

            // If the image does exist on the file system, then stream the image to the response object
            fs.createReadStream(pathToFile)
                .pipe(res);
        }
    })
}