Failed to pass cookies from Selenium webdriver to httpclient cookie store in clojure

Question

I want to use Selenium to login to a site and then export the cookie to httpclient.

(defn convert-selemium-cookie-to-httpclient-cookie [x]
  (let [sf (SimpleDateFormat. "yyyy-MM-dd")
        fake-date (.parse sf "2018-08-06")]
    (doto
        (BasicClientCookie. (:name x) (:value x))
        (.setDomain (:domain x))
        (.setPath (:path x))
        (.setExpiryDate (if (:expiry x) (:expiry x) fake-date))
        (.setSecure (:secure? x)))))

(defn add-selenium-cookies-to-httpclient-cookie-store [x]
  (let [cs (clj-http.cookies/cookie-store)
        http-cookies (map convert-selemium-cookie-to-httpclient-cookie x)]
    (mapv (fn[x] (.addCookie cs x)) http-cookies)
    cs))

(def driver (new-driver {:browser :firefox}))
(def a (login driver ...)) ;; login
(def c (cookies driver)) ;; get the cookies
(def cs (add-selenium-cookies-to-httpclient-cookie-store c))
(println (client/get "some web site"
                     {:cookie-store cs
                      :client-params {"http.useragent"
                                      "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20"}}))

Although I login successfully in webdriver and that site persists cookies across sessions, after copying the cookies to cookies store, client/get still return the page NOT logged in. I have tried a similar thing in python, and it could successfully transfer cookies from webdriver to requests (http client for python). Not sure where is the problem for java/clojure.

Answer 1

You haven't specify versions of your dependencies. I have tested your code with following:

[org.clojure/clojure "1.8.0"]
[clj-webdriver "0.7.2"]
[org.seleniumhq.selenium/selenium-server "3.0.0-beta4"]

Firefox 48.0.2
Mac OS Sierra 10.12

REPL session at the end of my answer shows that it works (I have used https://httpbin.org/ to test it).

To troubleshoot I would check following:

• print what cookies were returned from driver or check in browser console to see if the response from your login page contains headers to set correct cookies
• print what your http-client cookiestore contains
• enable debug logging for http-client (or enable debug logging in your server) to see what actual request is sent to your login-protected page and if the request contain cookies set by the login page.

In REPL session you can see that cookies returned from selenium contain the cookies set by the first request. They are also present in the http-client cookie store and they are properly returned by the https://httpbin.org/cookies URL indicating that they have been sent with the request by http-client.

(require '[clj-webdriver.taxi :as taxi])
(require '[clj-http.client :as client])
(require '[clj-http.cookies :as cookies])
(require '[clojure.pprint :refer [pprint]])
(import java.text.SimpleDateFormat)
(import org.apache.http.impl.cookie.BasicClientCookie)

(defn convert-selemium-cookie-to-httpclient-cookie [x]
  (let [sf (SimpleDateFormat. "yyyy-MM-dd")
        fake-date (.parse sf "2018-08-06")]
    (doto
      (BasicClientCookie. (:name x) (:value x))
      (.setDomain (:domain x))
      (.setPath (:path x))
      (.setExpiryDate (if (:expiry x) (:expiry x) fake-date))
      (.setSecure (:secure? x)))))

(defn add-selenium-cookies-to-httpclient-cookie-store [x]
  (let [cs (cookies/cookie-store)
        http-cookies (map convert-selemium-cookie-to-httpclient-cookie x)]
    (mapv (fn [x] (.addCookie cs x)) http-cookies) cs))

(def cookie-name (str "c1" (System/currentTimeMillis)))
(def cookie-value (str "v1" (System/currentTimeMillis)))

(pprint cookie-name)
;; "c11475935066134"

(pprint cookie-value)
;; "v11475935066814"

(def driver (taxi/new-driver {:browser :firefox}))

(taxi/to driver (format "https://httpbin.org/cookies/set?%s=%s" cookie-name cookie-value))

(def selenium-cookies (taxi/cookies driver))

(pprint selenium-cookies)
;; #{{:cookie
;;    #object[org.openqa.selenium.Cookie 0x4dc96ce8 "c11475935066134=v11475935066814; path=/; domain=httpbin.org"],
;;    :name "c11475935066134",
;;    :value "v11475935066814",
;;    :path "/",
;;    :expiry nil,
;;    :domain "httpbin.org",
;;    :secure? false}}

(def http-client-cookie-store (add-selenium-cookies-to-httpclient-cookie-store selenium-cookies))

(pprint http-client-cookie-store)
;; #object[org.apache.http.impl.client.BasicCookieStore 0x6dfa86f5 "[[version: 0][name: c11475935066134][value: v11475935066814][domain: httpbin.org][path: /][expiry: Mon Aug 06 00:00:00 CEST 2018]]"]

(def http-client-response
  (client/get
    "https://httpbin.org/cookies"
    {:cookie-store http-client-cookie-store}))

(pprint http-client-response)
;; {:status 200,
;;  :headers
;;  {"Server" "nginx",
;;   "Date" "Sat, 08 Oct 2016 13:58:01 GMT",
;;   "Content-Type" "application/json",
;;   "Content-Length" "64",
;;   "Connection" "close",
;;   "Access-Control-Allow-Origin" "*",
;;   "Access-Control-Allow-Credentials" "true"},
;;  :body
;;  "{\n  \"cookies\": {\n    \"c11475935066134\": \"v11475935066814\"\n  }\n}\n",
;;  :request-time 1001,
;;  :trace-redirects ["https://httpbin.org/cookies"],
;;  :orig-content-encoding nil}