Scraping Netflix

Question

Trying to get my last viewed data out of Netflix programmatically, but having some issues just with the login phase. My current code just causes Netflix to spit back a We were unable to process your request. page:

var request = require('request').defaults({jar: true});
var cheerio = require('cheerio');

var url = "https://www.netflix.com/Login?locale=en-GB&nextpage=https%3A%2F%2Fwww.netflix.com%2FWiViewingActivity";

request(url, function (error, response, body) {
    if (!error && response.statusCode == 200) {
        var $ = cheerio.load(body);
        var authCode = $("#login-form > input").attr("value");
        request.post(url+"?email=myemail%40gmail.com&password=mypassword&RememberMe=on&authURL="+authCode, {
        }, function(err, response, body){
            console.log(body);
        });
    }
})

Any ideas?

Surprisingly, there's next to nothing on Google for Scraping Netflix.

Answer 1

Figured it out, needed to:

1. Send any sort of user-agent string
2. Send the form data using request's form param
3. Send the cookies manually

Here's my final code which gets the latest watched item:

var request = require('request').defaults({jar: true});
var cheerio = require('cheerio');

var url = "https://www.netflix.com/Login";

request(url, function (error, response, body) {
    if (!error && response.statusCode == 200) {
        var $ = cheerio.load(body);
        var authCode = $("#login-form > input").attr("value");
        request.post({url: url, 
            form: {
                "email": "[email protected]",
                "password": "password",
                "authURL": authCode,
                "RememberMe": "on"
            },
            headers:{
                'User-Agent': "NodeScrape"
            }
        }, function(err, response, body){
            var cookies = response.headers['set-cookie'];
            request({url: "https://www.netflix.com/WiViewingActivity", headers: {'Cookie': cookies, 'User-Agent': "NodeScrape"}}, function(error, response, body){
                var $ = cheerio.load(body);
                console.log($(".seriestitle").eq(0).text());
            });
        });
    }
})