Logo Questions Linux Laravel Mysql Ubuntu Git Menu
 

PhantomJS Issue writing to file fs. Can't find variable: fs

I am trying out phantomJS for the first time and i have successfully extracted som data from a site, but when i try to write some content to a file i get the error: ReferenceError: Can't find variable: fs

Here is my script

var page = require('webpage').create();
    var fs = require('fs');

    page.onConsoleMessage = function(msg) {
        console.log(msg);
    };

    page.open("http://www.pinterest.com/search/pins/?q=motorbike", function(status) {
        if (status === "success") {
            page.includeJs("http://code.jquery.com/jquery-latest.js", function() {
                page.evaluate(function() {
                    var imgs = {
                        title: [],
                        href: [],
                        ext: [],
                        src: [],
                        alt: []
                    };
                    $('a.pinImageWrapper').each(function() {
                        imgs.title.push($(this).attr('title'));
                        imgs.href.push($(this).attr('href'));
                        var ext = $(this).children('.pinDomain').html();
                        imgs.ext.push(ext);
                        var img = $(this).children('.fadeContainer').children('img.pinImg');
                        imgs.src.push(img.attr('src'));
                        imgs.alt.push(img.attr('alt'));
                    });
                    if (imgs.title.length >= 1) {
                        for (var i = 0; i < imgs.title.length; i++) {
                            console.log(imgs.title[i]);
                            console.log(imgs.href[i]);
                            console.log(imgs.ext[i]);
                            console.log(imgs.src[i]);
                            console.log(imgs.alt[i]);
                        }
                    } else {
                        console.log('No pins found');
                    }
                    fs.write('foo.txt', 'bar');
                });
                phantom.exit();
            });
        }
    });

What am i missing out here?

Edit: Of the response on this question i learned why i couldnt reach the data inside the evalute, and how i could access it.

            var page = require('webpage').create();
        var fs = require('fs');

        page.onConsoleMessage = function(msg) {
            console.log(msg);
        };

        openPinPage('motorbike');

        function openPinPage(keyword) {
            page.open("http://www.pinterest.com/search/pins/?q=" + keyword, function(status) {
                if (status === "success") {
                    page.includeJs("http://code.jquery.com/jquery-latest.js", function() {
                        getImgsData();
                    });
                }
            });
        }

        function getImgsData() {
            var data = page.evaluate(function() {
                var imgs = {
                    title: [],
                    href: [],
                    ext: [],
                    src: [],
                    alt: []
                };
                $('a.pinImageWrapper').each(function() {
                    imgs.title.push($(this).attr('title'));
                    imgs.href.push($(this).attr('href'));
                    var ext = $(this).children('.pinDomain').html();
                    imgs.ext.push(ext);
                    var img = $(this).children('.fadeContainer').children('img.pinImg');
                    imgs.src.push(img.attr('src'));
                    imgs.alt.push(img.attr('alt'));
                });
                return imgs;
            });
            for (var i = 0; i < data.title.length; i++) {
                console.log(data.title[i]);
            };
            phantom.exit();
        }
like image 664
Dan-Levi Tømta Avatar asked Jul 21 '14 14:07

Dan-Levi Tømta


2 Answers

You can't have phantomjs objects in page.evaluate because that is a webpage. I'll give you a simple example how can you achieve what you are doing.

If you want to write some content of webpage in a file you have to return those contens from page.evaluate. and you will get those values in page.open. Here you have access to fs, so you can write those contents.

I'm showing with a simple example how you can you write some webpage title to a file.

page.open("http://www.pinterest.com/search/pins/?q=motorbike", function(status) {
        if (status === "success") {
            page.includeJs("http://code.jquery.com/jquery-latest.js", function() {

                var title = page.evaluate(function() {
                    return document.title;  // here I don't have access to fs I'll return title of document from here.
                });
                console.log(title) //I got the title now I can write here.
                fs.write('foo.txt', title);
                phantom.exit();
            });
        }
    });
like image 53
Mritunjay Avatar answered Nov 14 '22 23:11

Mritunjay


Straight from the docs for page.evaluate():

Evaluates the given function in the context of the web page. The execution is sandboxed, the web page has no access to the phantom object and it can't probe its own setting.

No further explanation needed.

like image 34
Tomalak Avatar answered Nov 14 '22 21:11

Tomalak