Selenium: Scroll till end of the page

Question

Selenium:

I am new to WebDriverJS. I have tried this approach in Java.

Long repaeted = 0l, scrollHeight = 0l, returnHeight = 0l;
while(true){
    if (repaeted == 0) {
        returnHeight = (Long) jse.executeScript("var scroll =document.documentElement.scrollHeight;window.scrollTo(0, scroll); return scroll;");
         System.out.println("Height : "+scrollHeight +"\t Chnage : "+returnHeight+ "\t Repeated : "+repaeted);
         scrollHeight = returnHeight;
     }else {
         returnHeight = (Long) jse.executeScript("var scroll =  document.documentElement.scrollHeight;window.scrollTo(0, scroll); return scroll;");
         System.out.println("Height : "+scrollHeight +"\t Chnage : "+returnHeight+ "\t Repeated : "+repaeted);
         if (scrollHeight.intValue() == returnHeight.intValue()) {
             System.out.println("Break.."+ returnHeight);
             break;
         } else { scrollHeight = returnHeight; }
     }
            repaeted++;
 } 

but I am facing problem in webdriverjs while iterating the loop.

var webdriver = require('..'),
    By = webdriver.By,
    until = webdriver.until;
// make sure chromedriver can be found on your system PATH
var driver = new webdriver.Builder()
    .forBrowser('chrome')
    .withCapabilities(webdriver.Capabilities.chrome())
    .build();


driver.get('https://in.yahoo.com/').then(function(){
        var window = new webdriver.WebDriver.Window(driver);
        window.maximize();
        driver.manage().timeouts().implicitlyWait(1000 * 3);
    })
    .then(function(){
        console.log('Entered');
        var check = 0, count = 0
        for(var i = 0; i< 50; i++){
        //driver.sleep(1000 * 2);
driver.executeScript('var dynamicscroll = document.documentElement.scrollHeight;window.scrollTo(0, dynamicscroll);return dynamicscroll;').then(function(height){
        console.log('Check : '+check+'  Height : '+height +'  Repeated : '+(count++));
        if(check === 0 || check !== height){console.log('continue'); check = height; }
        else { console.log('break'); i = 100; }
            });
        }
        })
    .then(null, function(err) {
      console.error("An error was thrown! By Promise..." + err);
    });

driver.quit();

In my code I have hardcoded for loop to iterate until 50 times and I want to quit/break the loop when the scroll height is reached to end. In this approach, I want to remove hardcode like java-code because I don't know how many times to iterate for other applications whose scroll is kept on increasing dynamically. For example, Facebook application, Yahoo News...

Answer 1

Scrolling to the bottom of a dynamic page can be challenging depending on how it is implemented by the page.

First you'll have to find the container with the scrollbar since it can be different from the one linked to window.scrollTo.

Then scroll the container by increasing scrollTop until the scrollHeight becomes steady with no pending requests. To check if there are pending requests, either evalute jQuery.active if the page has JQuery or hook XMLHttpRequest to monitor the calls on send.

Here is an example using on a generic function to scroll to the bottom of the page a number of times or until the end:

var webdriver = require('selenium-webdriver');

var driver = new webdriver.Builder().forBrowser('chrome').build();

driver.get('https://groups.google.com/forum/#!search/webdriverjs');

 // scroll to the bottom 3 times
driver.executeAsyncScript(scrollBottom, 3)
  .then(n => console.log(`scrolled ${n} time(s)`));

 // scroll to the bottom until the end
driver.executeAsyncScript(scrollBottom)
  .then(n => console.log(`scrolled ${n} time(s)`));

function scrollBottom(){
  var count = arguments[arguments.length - 2] || 0x7fffffff;
  var callback = arguments[arguments.length - 1];

  /* get the scrollable container */
  var elm = document.elementFromPoint(window.innerWidth - 25, window.innerHeight / 2);
  for ( ;elm && (++elm.scrollTop, !elm.scrollTop); elm=elm.parentElement);
  elm = elm || document.documentElement;

  /* hook XMLHttpRequest to monitor Ajax requests */
  if (!('idle' in XMLHttpRequest)) (function(){
    var n = 0, t = Date.now(), send = XMLHttpRequest.prototype.send;
    var dispose = function(){ --n; t = Date.now(); };
    var loadend = function(){ setTimeout(dispose, 1) };
    XMLHttpRequest.idle = function() { return n > 0 ? 0 : Date.now() - t; };
    XMLHttpRequest.prototype.send = function(){
      ++n;
      this.addEventListener('loadend', loadend);
      send.apply(this, arguments);
    };
  })();

  /* scroll until steady scrollHeight or count of scroll and no pending request */
  var i = 0, scrollHeight = -1, scrollTop = -1;
  (function scroll(){
    if ((scrollHeight === elm.scrollHeight || i === count) && XMLHttpRequest.idle() > 60)
      return callback(i);
    scrollTop = elm.scrollTop;
    scrollHeight = elm.scrollHeight;
    if (i < count)
      i += (elm.scrollTop = 0x7fffffff, scrollTop !== elm.scrollTop);
    setTimeout(scroll, 100);
  })();
}

Or by scrolling until the height no longer increases during a specific time (5 seconds here) :

function scrollBottom(){
  var count = arguments[arguments.length - 2] || 0x7fffffff;
  var callback = arguments[arguments.length - 1];
  var timeout = 5000;  /* 5 seconds timeout */
  var i = 0;

  /* get the scrollable container */
  var elm = document.elementFromPoint(window.innerWidth - 25, window.innerHeight / 2);
  for ( ;elm && (++elm.scrollTop, !elm.scrollTop); elm=elm.parentElement);
  elm = elm || document.documentElement;

  /* scroll while the height is increasing or until timeout */
  (function scroll(){
    var endtime = Date.now() + timeout;
    var height = elm.scrollHeight;
    elm.scrollTop = 0x7fffffff;  /* scroll */

    setTimeout(function check(){
      if (Date.now() > endtime)            /* returns if waited more than 5 sec */
        callback(i);
      else if (elm.scrollHeight == height) /* wait again if same height  */
        setTimeout(check, 60);
      else if (++i === count)              /* returns if scrolled the expected count */
        callback(i);
      else                                 /* scroll again */
        setTimeout(scroll, 60);
    }, 250);
  })();
}

Answer 2

From experience, the quickest way to scroll to the end of a page is to look for the footer element and movetoit, usually #footer or .footer or just footer selector will do it. E.g.:

footer = driver.findElement({id: "footer"});
driver.executeScript("arguments[0].scrollIntoView(false);", footer);

In the case of 'endless' streams like Facebook, Twitter, etc. They may block you when you reach a limit so it's okay to combine max iterations with window.scrollTo(0, 300); recursively and wait for some seconds after each scroll.