Node.js how to wait on asynchronous call (readdir and stat)

Question

I am working on post method in the server side to retrieve all files inside the requested directory (not recursive), and below is my code.

I am having difficulty sending the response back (res.json(pathContent);) with the updated pathContent without using the setTimeout.

I understand that this is due to the asynchronous behavior of the file system methods used (readdir and stat) and need to use some sort of callback, async, or promise technique.

I tried to use the async.waterfall with the entire body of readdir as one function and the res.json(pathContent) as the other, but it didn't send the updated array to the client side.

I know that there have been thousands of questions regarding this asynchronous operation but could not figure out how to solve my case after reading number of posts.

Any comments would be appreciated. Thanks.

const express = require('express');
const bodyParser = require('body-parser');
const fs = require('fs');
const path = require('path');

var pathName = '';
const pathContent = [];

app.post('/api/files', (req, res) => {
    const newPath = req.body.path;
    fs.readdir(newPath, (err, files) => {
        if (err) {
            res.status(422).json({ message: `${err}` });
            return;
        }
        // set the pathName and empty pathContent
        pathName = newPath;
        pathContent.length = 0;

        // iterate each file
        const absPath = path.resolve(pathName);
        files.forEach(file => {
            // get file info and store in pathContent
            fs.stat(absPath + '/' + file, (err, stats) => {
                if (err) {
                    console.log(`${err}`);
                    return;
                }
                if (stats.isFile()) {
                    pathContent.push({
                        path: pathName,
                        name: file.substring(0, file.lastIndexOf('.')),
                        type: file.substring(file.lastIndexOf('.') + 1).concat(' File'),
                    })
                } else if (stats.isDirectory()) {
                    pathContent.push({
                        path: pathName,
                        name: file,
                        type: 'Directory',
                    });
                }
            });
        });
    });    
    setTimeout(() => { res.json(pathContent); }, 100);
});

Answer 1

The easiest and cleanest way would be use await/async, that way you can make use of promises and the code will almost look like synchronous code.

You therefor need a promisified version of readdir and stat that can be create by the promisify of the utils core lib.

const { promisify } = require('util')

const readdir = promisify(require('fs').readdir)
const stat = promisify(require('fs').stat)

async function getPathContent(newPath) {
  // move pathContent otherwise can have conflicts with concurrent requests
  const pathContent = [];

  let files = await readdir(newPath)

  let pathName = newPath;
  // pathContent.length = 0;  // not needed anymore because pathContent is new for each request

  const absPath = path.resolve(pathName);

  // iterate each file

  // replace forEach with (for ... of) because this makes it easier 
  // to work with "async" 
  // otherwise you would need to use files.map and Promise.all
  for (let file of files) {
    // get file info and store in pathContent
    try {
      let stats = await stat(absPath + '/' + file)
      if (stats.isFile()) {
        pathContent.push({
          path: pathName,
          name: file.substring(0, file.lastIndexOf('.')),
          type: file.substring(file.lastIndexOf('.') + 1).concat(' File'),
        })
      } else if (stats.isDirectory()) {
        pathContent.push({
          path: pathName,
          name: file,
          type: 'Directory',
        });
      }
    } catch (err) {
      console.log(`${err}`);
    }
  }

  return pathContent;
}

app.post('/api/files', (req, res, next) => {
  const newPath = req.body.path;
  getPathContent(newPath).then((pathContent) => {
    res.json(pathContent);
  }, (err) => {
    res.status(422).json({
      message: `${err}`
    });
  })
})

And you should not concatenated paths using + (absPath + '/' + file), use path.join(absPath, file) or path.resolve(absPath, file) instead.

And you never should write your code in a way that the code executed for the request, relays on global variables like var pathName = ''; and const pathContent = [];. This might work in your testing environment, but will for sure lead to problems in production. Where two request work on the variable at the "same time"