How do I stream JSON from node?

Question

I'm working on a node/express (Sails, technically) based service which will be used to retrieve a large number of items. Several calls will need to return many thousands of items as a JSON serialized array.

Inside node will be a basic control loop to retrieve the items in pages. Each page will be retrieved, have some minor processing performed, and then have its items returned to the client.

At present I'm doing a "store and forward" approach, in which each page's items are concat() to a results array, and then once all items have been retrieved the results are returned.

What I would like to do is more of a yield or streaming approach, in which items are added to the response as soon as they're ready--avoiding the need to build a large in-memory collection and beginning to send usable data as soon as possible.

Answer 1

You can directly write to res object in express with something like this

var data=[/* a large array of json objects*/];//
//let us divide this large array into chunks of smaller array
function chunk(arr, chunkSize) {
    var R = [];
    for (var i = 0; i < arr.length; i += chunkSize)
        R.push(arr.slice(i, i + chunkSize));
    return R;
}
var new_data=chunk(data,10);//[/* array of arrays*/], chunk size is 10
res.writeHead(200, {
        'Content-Type': 'application/json',
        'Transfer-Encoding': 'chunked'
})
res.write("["); //array starting bracket
for (var i = 0; i < new_data.length - 1; i++) {

        res.write(JSON.stringify(new_data[i]) + ',');

}
res.write(JSON.stringify(new_data[new_data.length-i]));
res.write("]"); //array ending bracket
res.end();

And in the client use something like this

// using axios
var url = 'http://localhost:3000';
axios.get(url, { responseType: 'stream' }).then(handleRes);

function handleRes(res) {
    // res.headers available here
    res.data.on('data', data => {
        data = data.toString(); // utf8 by default, change if needed

        if (data === '[' || data === ']') return console.log(data);

        var jsonStr = data.slice(-1) === '}' ? data : data.slice(0, -1);
        console.log(JSON.parse(jsonStr));
    })
}

Answer 2

If you need to generate one large JSON string from all your data, you're pretty much stuck with storing everything in memory and then JSON.stringify-ing it.

An alternative would be to have each item be a separate JSON string, using a newline character as delimiter. That way, as soon as you have processed an item, you can stringify it and pipe it to your response using Node streams, and in the client you can process the data you receive line by line as well. It would go something like this:

// For each page of data you get, loop over the items like you say
for item in dataset // Yes that's Coffeescript

  // Manipulate the item as you need, then make a JSON string out of it
  jsonStr = JSON.stringify(item) + '\n'

  // Pipe the string to your http response
  jsonStr.toStream().pipe(res) // Assuming 'res' is the Express response object

This is not the only way to approach your issue, but especially if you like to use JSON it is one of the easiest implementations. I don't have any experience with Sails though, but I imagine the basic implementation will be the same.

I hope my answer helps you, if not feel free to comment.

Note: the .toStream() method comes from my module streammagic but there are of course other ways to make a stream out of a string.