Getting strange connection aborted errors with python requests

Question

import json
import string
import socket
import requests
from bs4 import BeautifulSoup

# Default header to be used first.
headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36"}

# Create a session using requests to log in.
with requests.Session() as s:
    # Grab new headers and cookies from login page
    t = s.get("http://minewind.com/forums/ucp.php?mode=login", headers=headers)

    sid = t.cookies['phpbb3_qpac2_sid'] # Store sid to be used in POST data.


    # POST data to be sent
    payload = {"login": "Login",
               "password": "*********",
               "redirect": "./ucp.php?mode=login",
               "redirect": "index.php",
               "sid": sid,
               "username": "myusername"
               }
    # Send POST data to the login page, including proper headers.
    s1 = s.post("http://minewind.com/forums/ucp.php?mode=login", data=payload, headers=t.headers)

    print (t.headers)

    # Check to see if we are really logged in, WHICH WE ARENT!!!! ;_;
    s2 = s.get("http://minewind.com/forums/index.php", headers=t.headers)

    # Pretty up the code and grab links.
    perty = BeautifulSoup(s2.content)
    perty.prettify()


    for links in perty.find_all('a'):
        print (links.get('href'))

I finally configured the POST data properly to my knowledge, but now i'm getting some weird connection errors, any ideas? ERRORS:

Traceback (most recent call last):
  File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 331, in _make_request
    httplib_response = conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 516, in urlopen
    body=body, headers=headers)
  File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 333, in _make_request
    httplib_response = conn.getresponse()
  File "C:\Python33\lib\http\client.py", line 1143, in getresponse
    response.begin()
  File "C:\Python33\lib\http\client.py", line 354, in begin
    version, status, reason = self._read_status()
  File "C:\Python33\lib\http\client.py", line 316, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "C:\Python33\lib\socket.py", line 297, in readinto
    return self._sock.recv_into(b)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Python33\lib\site-packages\requests\adapters.py", line 362, in send
    timeout=timeout
  File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 559, in urlopen
    _pool=self, _stacktrace=stacktrace)
  File "C:\Python33\lib\site-packages\requests\packages\urllib3\util\retry.py", line 245, in increment
    raise six.reraise(type(error), error, _stacktrace)
  File "C:\Python33\lib\site-packages\requests\packages\urllib3\packages\six.py", line 309, in reraise
    raise value.with_traceback(tb)
  File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 516, in urlopen
    body=body, headers=headers)
  File "C:\Python33\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 333, in _make_request
    httplib_response = conn.getresponse()
  File "C:\Python33\lib\http\client.py", line 1143, in getresponse
    response.begin()
  File "C:\Python33\lib\http\client.py", line 354, in begin
    version, status, reason = self._read_status()
  File "C:\Python33\lib\http\client.py", line 316, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "C:\Python33\lib\socket.py", line 297, in readinto
    return self._sock.recv_into(b)
requests.packages.urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing con
nection was forcibly closed by the remote host', None, 10054))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Anthony\site.py", line 28, in <module>
    s2 = s.get("http://minewind.com/forums/index.php", headers=t.headers)
  File "C:\Python33\lib\site-packages\requests\sessions.py", line 469, in get
    return self.request('GET', url, **kwargs)
  File "C:\Python33\lib\site-packages\requests\sessions.py", line 457, in request
    resp = self.send(prep, **send_kwargs)
  File "C:\Python33\lib\site-packages\requests\sessions.py", line 569, in send
    r = adapter.send(request, **kwargs)
  File "C:\Python33\lib\site-packages\requests\adapters.py", line 407, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was for
cibly closed by the remote host', None, 10054))

Any thoughts on why i'm getting these 'connection aborted' errors?

Answer 1

You can just use the user-agent headers for this instance, I complicating it by grabbing the login page headers which wasn't necessary. Also you don't need to know the sid cookie beforehand like I thought you did. You can just include it with the POST data as empty. Just make sure you are inspecting which form data is being passed with firebug or similar utilities stated above.

import requests
from bs4 import BeautifulSoup
import sys

headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36"}

with requests.Session() as s:

    payload = {"login": "Login",
               "password": "mypassword",
               "redirect": "./ucp.php?mode=login",
               "redirect": "index.php",
               "sid": "",
               "username": "myusername"}

    url = "http://minewind.com/forums/index.php"

    s1 = s.post("http://minewind.com/forums/ucp.php?mode=login", data=payload, headers=headers)
    s2 = s.get(url, headers=headers)