Why setCookie() from Puppeteer seems not to work as expected?

Question

The goal I'm trying to achieve is to store the cookies from a website and then to use them in a second moment. Here is the code:

To save cookie:

let cookie = await page.cookies();
cookie = JSON.stringify(cookie);

fs.writeFile("cookie.txt", cookie, function(err, data){
    if (err) {
        console.log(err);
    } else {
        console.log("Successfully Written to File.");
    }
});

To read the cookies within Puppeteer:

await page._client.send("Network.clearBrowserCookies");
await injectCookiesFromFile("cookie.txt", page)
                

async function injectCookiesFromFile(file, page) {
    let cb = async function (_cookies) {
        console.log("Injecting cookies from file: %s", JSON.stringify(_cookies) );
        //await page.setCookie(..._cookies); // method 1
        await page.setCookie(_cookies); // method 2
    };

    fs.readFile(file, async function(err, data) {
        if(err) {
            throw err;
        }

        let cookies = JSON.parse(data);
        console.log(cookies);
     
        //await cb(cookies); // method 1
        for (var i = 0, len = cookies.length; i < len; i++) {
            await cb(cookies[i]); // method 2
        }
    });
}

The operations of reading and saving the cookies on a file seem to work. But there is no effect on the website and it seems that setCookie() method is not properly working. Any idea about it?

Answer 1

Before we begin, it's important to note that reading and writing cookies to a file is not typically necessary when using Puppeteer.

Puppeteer provides a high-level API to control Chrome or Chromium over the DevTools Protocol, which means that Chromium does most of the hard work regarding cookies without all of the manual labor.

Chromium has a built-in class called CookieMonster which handles in-browser storage, management, retrieval, expiration, and eviction of cookies.

In other words, if you are attempting to log into a website using Puppeteer, you can generally do so using the following simple example:

'use strict';

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  await page.goto('https://www.example.com/login');
  
  await page.type('#username', 'johndoe');
  await page.type('#password', 'qwerty1');
  await page.click('#submit');
  
  await page.waitForNavigation();
  
  // You are now logged in ...
  
  await browser.close();
})();

Note: Notice that I did not have to create a cookie file to read and write cookies (this is not cURL).

If you are still set on managing the cookies yourself, it's critical to understand that Puppeteer utilizes asynchronous JavaScript.

This means that in order to write to a file using fs.writeFile, you need to await a Promise that is resolved or rejected in the callback function to ensure that the file is ready before navigation.

const write_file = (file, data) => new Promise((resolve, reject) => {
  fs.writeFile(file, data, 'utf8', error => {
    if (error) {
      console.error(error);
      
      reject(false);
    } else {
      resolve(true);
    }
  });
});

The same is true for fs.readFile():

const read_file = file => new Promise((resolve, reject) => {
  fs.readFile(file, 'utf8', (error, data) => {
    if (error) {
      console.error(error);
      
      reject(false);
    } else {
      resolve(data);
    }
  });
});

Additionally, it's a good idea to wait for a page to completely load before writing cookies to a file, so I would recommend using the waitUntil: 'networkidle0' option in your navigation functions:

await page.goto('https://www.example.com/login', {
  waitUntil: 'networkidle0',
});

Note: This may require you to increase the default timeout option to allow for the page to completely load.

Now that we understand some basic fundamental concepts behind Puppeteer, we can begin to read and write cookies from a file manually.

We can use page.cookies(), page.setCookie(), and cdpSession.send() to manage our cookies (as shown in the example below):

'use strict';

const fs = require('fs');
const puppeteer = require('puppeteer');

const write_file = (file, data) => new Promise((resolve, reject) => {
  fs.writeFile(file, data, 'utf8', error => {
    if (error) {
      console.error(error);
      
      reject(false);
    } else {
      resolve(true);
    }
  });
});

const read_file = file => new Promise((resolve, reject) => {
  fs.readFile(file, 'utf8', (error, data) => {
    if (error) {
      console.error(error);
      
      reject(false);
    } else {
      resolve(data);
    }
  });
});

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  const client = await page.target().createCDPSession();
  
  // Open First Page
  
  await page.goto('https://www.example.com/', {
    waitUntil: 'networkidle0',
  });
  
  // Write All Cookies to File
  
  await write_file('cookies.txt', JSON.stringify(await page.cookies()));
  
  // Open Second Page
  
  await page.goto('https://www.example.com/next-page', {
    waitUntil: 'networkidle0',
  });
  
  // Clear Browser Cookies
  
  await client.send('Network.clearBrowserCookies');
  
  // Read All Cookies from File
  
  await page.setCookie(...JSON.parse(await read_file('cookies.txt') || '[]'));
  
  await browser.close();
})();