Node.js - read and download all files in directory from server and save locally

Question

I have a Node Webkit Desktop App and need to download files from the server and save locally for when users are offline. I can download and save a file when I know what the file name is, but how do I read the contents of a directory on the server so I can download each file?

function cacheFiles(filelink, filepath, cb) {
    var path_array =  filelink.split("/");
    var foldername =  path_array[path_array.length - 2]

//create new folder for locally html files
var newdir = filepath + '/' + foldername;
if (fs.existsSync(newdir)){
    alert('file already exists, cannot cache this file.');
} else {
    fs.mkdirSync(newdir);
}
//download and save index.html - THIS WORKS
var indexfile = fs.createWriteStream(newdir+'/index.html');
var request = http.get(filelink, function(response) {
    response.pipe(indexfile);
    indexfile.on('finish', function() {
        indexfile.close(cb);
    });
});
//read contents of data folder - THIS DOESN'T WORK
var datafolder = filelink.replace('index.html','');

fs.readdir( datafolder, function (err, datafiles) { 
    if (!err) {
        console.log(datafiles);
    }else{
        console.log(err) ; 
    }   
});

}

The error I get in my console is:

"ENOENT: no such file or directory, scandir 'C:\Users\my.name\desktopApp\app\http:\www.mysite.co.uk\wp-content\uploads\wifi_corp2\data'"

The above is looking for the files locally and not at the online link I supplied in filelink eg. http://www.mysite.co.uk/wp-content/uploads/wifi_corp2/data

Answer 1

The following code doesn't read a remote file system, it's used for reading files on your local hard drive.

import fs from 'fs'
import path from 'path'

fs.readdir(path.resolve(__dirname, '..', 'public'), 'utf8', (err, files) => {
    files.forEach((file) => console.info(file))
})

Will print out all the file names from one directory up and in a 'public' directory from the script location. You can use fs.readFile to read the contents of each file. If they are JSON, you may read them as utf8 strings and parse them with JSON.parse.

To read files from a remote server, they must be served via express or some other static file server:

import express from 'express'
const app = express()
app.use(express.static('public'))
app.listen(8000)

Then on the client end you could use fetch or request http library to call the express endpoint hosted at port 8000 (in this simple example.

Answer 2

You are mixing up the server code with the desktop app code. Obviously the desktop app can't do a readdir on your server fiiles. Just install a backup or download plugin on Wordpress.