Node.js` "http.get" doesn't return full response body to a browser but just a piece of tag. How to get full response?

Question

I experience a strange behavior with Node.js http.get. I make a request to a url with ajax and want to get the result into my browser. But I only get some piece of a <head> tag content and nothing more, no <body> content. But if I send the result to system console (console.log(chunk)) I get the result as I want to get - the full page. Here are my steps:

// Simple jQuery Ajax GET
$.ajax({
    type: "GET",
    url: "/myapppath",   // send to my app's url
    data: {foo: "bar"},
    success: onLoad,    // the callback, see just bellow
    error: onError,
    dataType: "text"
});

// With this callback function I want to insert the result into <div id="baz">
function onLoad(resp) {
   document.getElementById("baz").innnerHTML = resp;
}

// In /myapppath
http.get("http://stackoverflow.com/", function(result) {
   result.setEncoding('utf8');
   result.on("data", function(chunk) {
      console.log(chunk); // this successfully returns the whole page but into system console
      res.end(chunk);   // this returns just a piece of <head> tag into the browser.
   });
});

So in my <div id="baz"> I only get some piece of <head> tag of http://stackoverflow.com/ request, no <body> tag and its content. That's all I get in <div id="baz"> instead of the whole page:

<!DOCTYPE html>
<html>
<head> 
    <title>Stack Overflow</title>
    <link rel="shortcut icon" href="https://cdn.sstatic.net/stackoverflow/img/favicon.ico">
    <link rel="apple-touch-icon image_src" href="https://cdn.sstatic.net/stackoverflow/img/apple-touch-icon.png">
    <link rel="search" type="application/opensearchdescription+xml" title="Stack Overflow" href="/opensearch.xml">

    <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
    <script type="text/javascript" src="https://cdn.sstatic.net/js/stub.js?v=dd6898efd655"></script>
    <link rel="stylesheet" type="text/css" href="https://cdn.sstatic.net/stackoverflow/all.css?v=8a5907e853ab">

    <link rel="alternate" type="application/atom+xml" title="Feed of recent questions" href="/feeds">
        <script type="text/javascript" defer>
        </script>    
    <script type="text/javascript">          StackExchange.init({"stackAuthUrl":"https://stackauth.com","serverTime":1374771485,"styleCode":true,"enableUserHovercards":true,"site":{"name":"Stack Overflow","description":"Q&A for professional and enthusiast programmers","isNoticesTabEnabled":true,"recaptchaPublicKey":"6LdchgIAAAAAAJwGpIzRQSOFaO0pU6s44Xt8aTwc","enableSocialMediaInSharePopup":true},"user":{"fkey":"b1d105a0cf61e49216c5750a6ad60dec","isAnonymous":true}});
        StackExchange.using.setCacheBreakers({"js/prettify-full.js":"6c261bebf56a","js/moderator.js":"7cf00e91ce39","js/full-anon.js":"c5bf51314708","js/full.js":"02e9182c23d3","js/wmd.js":"2f79c03846d5","js/third-party/jquery.autocomplete.min.js":"e5f01e97f7c3","js/mobile.js":"e8e23ad37820","js/help.js":"6e6623243cf6","js/tageditor.js":"450c9e8426fc","js/tageditornew.js":"b6c68ad4c7dd","js/inline-tag-editing.js":"8e84e8a137f7","js/revisions.js":"7273bb714bba","js/review.js":"2b3ae123e376","js/tagsuggestions.js":"aa48ef6154df","js/post-validation.js":"bb996020492a","js/explore-qlist.js":"1c5bbd79b562","js/events.js":"37756ef3ba47"});
    </script>
    <script type="text/javascript">
        StackExchange.using("gps", function() {
             StackExchange.gps.init(true);
        });
    </script>

        <script type="text/javascript">
            StackExchange.ready(function () {
                $('#nav-tour').click(function () {
                    StackExchange.using("gps", function() {
                        StackExchange.gps.track("aboutpage.click", { aboutclick_location: "headermain" }, true);
                    });
                });
            });
        </script>
</h

But in console.log(chunk) I get the whole page printed in the console as I said above.

What is going on? Why http.get doesn't return full response into browser? What did I miss? What cuts the response?

Jonathan Lonowski · Accepted Answer

console.log(chunk); doesn't actually log the entire page at once, but it does keep logging each chunk as more 'data' arrives.

res.end(), on the other hand, becomes a no-op after the 1st call as it closes the connection, so only the 1st chunk is included.

What you can do is res.write() each chunk of 'data', waiting for 'end' to res.end():

http.get("http://stackoverflow.com/", function (result) {
    result.on('data', function (chunk) {
        res.write(chunk);
    });
    result.on('end', function () {
        res.end();
    });
});

Or, since result is a Stream.Readable (IncomingMessage) and res is presumably a Stream.Writable (guessing a ServerResponse), you should be able to .pipe() them:

http.get('http://stackoverflow.com', function (result) {
    result.pipe(res);
});

Tracker1 · Answer

Your data event is firing multiple times, but you can only "end" your output once... you should probably refactor your handler as follows...

// In /myapppath
http.get("http://stackoverflow.com/", function(result) {
    var responseParts = [];
    result.setEncoding('utf8');
    result.on("data", function(chunk) {
        //add this chunk to the output to send
        responseParts.push(chunk);
    });
    result.on("end", function(){
        //now send your complete response
        res.end(responseParts.join(''));
    });
});

Node.js` "http.get" doesn't return full response body to a browser but just a piece of <head> tag. How to get full response?

Tags:

node.js

httprequest

Green

2 Answers

Jonathan Lonowski

Tracker1

Recent Activity

Donate For Us