Incomplete Node HTTP.Response

Question

My application builds a custom array of integers that need to be concatenated in order to create a dynamic request for economic data.

Once my variable options gets created, I push it through an http request and try to parse the response, where I'm getting an error -

SyntaxError: Unexpected end of JSON input
at JSON.parse (<anonymous>)

When I use my code to read the body, the response cuts off early and I don't get the full response in the console.log().

I've tested the request values and placed the results in my search bar and have yielded positive results.

Any Ideas on why the body on the response gets cut short? I need to read all of the data from this request and when parsing I get errors.

Here is the code -

var options = {
host: 'api.eve-central.com',
port: 80,
path: '/api/marketstat/json?typeid=18&typeid=19&typeid=20&typeid=21&typeid=22&typeid=34&typeid=35&typeid=36&typeid=37&typeid=38&typeid=39&typeid=40&typeid=41&typeid=42&typeid=43&typeid=44&typeid=45&typeid=49&typeid=50&typeid=51&typeid=52&typeid=164&typeid=165&typeid=166&typeid=178&typeid=179&typeid=180&typeid=181&typeid=182&typeid=183&typeid=184&typeid=185&typeid=186&typeid=187&typeid=188&typeid=189&typeid=190&typeid=191&typeid=192&typeid=193&typeid=194&typeid=195&typeid=196&typeid=197&typeid=198&typeid=199&typeid=200&typeid=201&typeid=202&typeid=203&typeid=204&typeid=205&typeid=206&typeid=207&typeid=208&typeid=209&typeid=210&typeid=211&typeid=212&typeid=213&typeid=215&typeid=216&typeid=217&typeid=218&typeid=219&typeid=220&typeid=221&typeid=222&typeid=223&typeid=224&typeid=225&typeid=226&typeid=227&typeid=228&typeid=229&typeid=230&typeid=231&typeid=232&typeid=233&typeid=234&typeid=235&typeid=236&typeid=237&typeid=238&typeid=239&typeid=240&typeid=241&typeid=242&typeid=243&typeid=244&typeid=245&typeid=246&typeid=247&typeid=248&typeid=249&typeid=250&typeid=251&typeid=252&typeid=253&typeid=254&usesystem=30000142',
method: 'GET'
}

function requester(options) {
http.request(options, function (res) {
    //console.log('STATUS: ' + res.statusCode);
    //console.log('HEADERS: ' + JSON.stringify(res.headers));
    res.setEncoding('utf8');
    res.on('data', function (chunk) {
        console.log('BODY: ' + chunk);
        var payload = JSON.parse(chunk);
        //console.log('After Parse: ' + payload);
        //Separating the payload into three pieces.
        buy = payload[0]["buy"];
        all = payload[0]["all"];
        sell = payload[0]["sell"];
    });
  }).end();
};

Edit: Removed variables from the code that would interfere with testing.

Answer 1

You need to accumulate the the data that is streamed before performing a parse since what consitutes a data chunk is determined by the io rate.

Instead store all of the chunked data and then parse it

var options = {
  host: 'api.eve-central.com',
  port: 80,
  path: '/api/marketstat/json?typeid=18&typeid=19&typeid=20&typeid=21&typeid=22&typeid=34&typeid=35&typeid=36&typeid=37&typeid=38&typeid=39&typeid=40&typeid=41&typeid=42&typeid=43&typeid=44&typeid=45&typeid=49&typeid=50&typeid=51&typeid=52&typeid=164&typeid=165&typeid=166&typeid=178&typeid=179&typeid=180&typeid=181&typeid=182&typeid=183&typeid=184&typeid=185&typeid=186&typeid=187&typeid=188&typeid=189&typeid=190&typeid=191&typeid=192&typeid=193&typeid=194&typeid=195&typeid=196&typeid=197&typeid=198&typeid=199&typeid=200&typeid=201&typeid=202&typeid=203&typeid=204&typeid=205&typeid=206&typeid=207&typeid=208&typeid=209&typeid=210&typeid=211&typeid=212&typeid=213&typeid=215&typeid=216&typeid=217&typeid=218&typeid=219&typeid=220&typeid=221&typeid=222&typeid=223&typeid=224&typeid=225&typeid=226&typeid=227&typeid=228&typeid=229&typeid=230&typeid=231&typeid=232&typeid=233&typeid=234&typeid=235&typeid=236&typeid=237&typeid=238&typeid=239&typeid=240&typeid=241&typeid=242&typeid=243&typeid=244&typeid=245&typeid=246&typeid=247&typeid=248&typeid=249&typeid=250&typeid=251&typeid=252&typeid=253&typeid=254&usesystem=30000142',
  method: 'GET'
}

function requester(options) {
    http.request(options, function (res) {
        res.setEncoding('utf8');

        // Build response body in a string
        var resBody = '';

        // Listen for data and add
        res.on('data', function (chunk) {
            resBody += chunk
        });

        res.on('end', function () {   
            // Now that the response is done streaming, parse resBody           
            var payload = JSON.parse(resBody);

            //Separating the payload into three pieces.
            buy = payload[0]["buy"];
            all = payload[0]["all"];
            sell = payload[0]["sell"];

            itemStats.hiBuy = buy["max"];
            itemStats.loSel = sell["min"];
            itemStats.iskSpread = itemStats.loSel - itemStats.hiBuy; //Adjusted the spread calculation for station trading.
            itemStats.perSpread = itemStats.iskSpread / itemStats.loSel;
            itemStats.buyVol = buy["volume"];
            itemStats.selVol = sell["volume"];
            itemStats.allVol = all["volume"];
            itemStats.buyPerHR = itemStats.buyVol / 24;
            itemStats.selPerHR = itemStats.selVol / 24;

            console.log("Transferred Values" + JSON.stringify(itemStats));
        });
    });
};