How to traverse all files, and support pause and continue

Question

I have created a NodeJS (electron) code for read all the files in a specific directory and subdirectories. I don't want to use too much HD resources, that why I use a delay of 5ms between folders.

Now my question. I want the if my NODE process stop? I want to be able to continue from when it is stopped. How should I do that?

In other words: How to keep index of current state while walking in all files and folder, so I can continue the traversing from when it has stopped.

Thank you

My Code:

var walkAll=function(options){
    var x=0
    walk(options.dir,function(){})
    function walk(dir,callback) {
      var files=fs.readdirSync(dir);
      var stat;
      async.eachSeries(files,function(file,next){
        file=dir +'/' + file
        if (dir.match(/Recycle/)) return next() 
        if (dir.match(/.git/)) return next() 
        if (dir.match(/node_modules/)) return next() 
        fs.lstat(file,function(err,stat){
            if(err) return next()
            if(stat.mode==41398) return next()
            if (stat.isDirectory()) {
                setTimeout(function(file){
                    walk(file,next)
                }.bind(null,file),5)
            }
            else{
                x++
                if(false || x % 1000===0) console.log((new Date().valueOf()-start)/1000,x,file)
                next()
            }
        })
      },function(){
        callback()
      })
    }
}

walkAll({
    dir:'c:/',
    delay:1000
});

Answer 1

Keep a list of sub directories to be visited, and update the list every iteration.

The walk function in the following example takes a previous state, and returns files of next sub directory with next state.

You can save the state before stopping the process, then load the saved state to continue the traversal when restarting.

function walk(state, readdir) {
  let files = [], next = [];
  while (state.length > 0) {
    try {
      const current = state.shift()
      files = readdir(current).map(file => current + '/' + file)
      next = state.concat(files)
      break
    } catch(e) {}
  }
  return [next, files]
}

function main() {
  const {writeFileSync: writeFile, readdirSync: readdir} = require('fs')
  const save = './walk.json'

  let state
  try {
    state = require(save)
  } catch(e) {}
  if (!state || state.length < 1) state = ['.']

  const [nextState, files] = walk(state, readdir)
  console.log(files)
  writeFile(save, JSON.stringify(nextState, null, 2))
}

main()