Menu
I have a Django project in which I'm starting to write Selenium tests. The first one looking like this:
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from core.models import User
from example import settings
BACH_EMAIL = "[email protected]"
PASSWORD = "password"
class TestImportCRMData(StaticLiveServerTestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.webdriver = webdriver.Chrome()
cls.webdriver.implicitly_wait(10)
@classmethod
def tearDownClass(cls):
cls.webdriver.close()
cls.webdriver.quit()
super().tearDownClass()
def setUp(self):
self.admin = User.objects.create_superuser(email=BACH_EMAIL, password=PASSWORD)
def test_admin_tool(self):
self.webdriver.get(f"http://{settings.ADMIN_HOST}:{self.server_thread.port}/admin")
self.webdriver.find_element_by_id("id_username").send_keys(BACH_EMAIL)
self.webdriver.find_element_by_id("id_password").send_keys(PASSWORD)
self.webdriver.find_element_by_id("id_password").send_keys(Keys.RETURN)
self.webdriver.find_element_by_link_text("Users").click()
When I run it, the test pass but still ends with this error:
Traceback (most recent call last):
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 83, in _execute
return self.cursor.execute(sql)
psycopg2.OperationalError: database "test_example" is being accessed by other users
DETAIL: There is 1 other session using the database.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_manage.py", line 168, in <module>
utility.execute()
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_manage.py", line 142, in execute
_create_command().run_from_argv(self.argv)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\core\management\commands\test.py", line 26, in run_from_argv
super().run_from_argv(argv)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\core\management\base.py", line 316, in run_from_argv
self.execute(*args, **cmd_options)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\core\management\base.py", line 353, in execute
output = self.handle(*args, **options)
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_manage.py", line 104, in handle
failures = TestRunner(test_labels, **options)
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_runner.py", line 255, in run_tests
extra_tests=extra_tests, **options)
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_runner.py", line 156, in run_tests
return super(DjangoTeamcityTestRunner, self).run_tests(test_labels, extra_tests, **kwargs)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\test\runner.py", line 607, in run_tests
self.teardown_databases(old_config)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\test\runner.py", line 580, in teardown_databases
keepdb=self.keepdb,
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\test\utils.py", line 297, in teardown_databases
connection.creation.destroy_test_db(old_name, verbosity, keepdb)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\base\creation.py", line 257, in destroy_test_db
self._destroy_test_db(test_database_name, verbosity)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\base\creation.py", line 274, in _destroy_test_db
% self.connection.ops.quote_name(test_database_name))
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 85, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\utils.py", line 89, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 83, in _execute
return self.cursor.execute(sql)
django.db.utils.OperationalError: database "test_example" is being accessed by other users
DETAIL: There is 1 other session using the database.
The problem of course is that the next run of the tests, the database still exists, so, the tests don't run without confirming deletion of the database.
If I comment out the last line:
self.webdriver.find_element_by_link_text("Users").click()
then I don't get this error. I guess just because the database connection is not established. Sometimes it's 1 other session, sometimes it's up to 4. In one of the cases of 4 sessions, these were the running sessions:
select * from pg_stat_activity where datname = 'test_example';
100123 test_example 29892 16393 pupeno "" ::1 61967 2018-11-15 17:28:19.552431 2018-11-15 17:28:19.562398 2018-11-15 17:28:19.564623 idle SELECT "core_user"."id", "core_user"."password", "core_user"."last_login", "core_user"."is_superuser", "core_user"."email", "core_user"."is_staff", "core_user"."is_active", "core_user"."date_joined" FROM "core_user" WHERE "core_user"."id" = 1
100123 test_example 33028 16393 pupeno "" ::1 61930 2018-11-15 17:28:18.792466 2018-11-15 17:28:18.843383 2018-11-15 17:28:18.851828 idle SELECT "django_admin_log"."id", "django_admin_log"."action_time", "django_admin_log"."user_id", "django_admin_log"."content_type_id", "django_admin_log"."object_id", "django_admin_log"."object_repr", "django_admin_log"."action_flag", "django_admin_log"."change_message", "core_user"."id", "core_user"."password", "core_user"."last_login", "core_user"."is_superuser", "core_user"."email", "core_user"."is_staff", "core_user"."is_active", "core_user"."date_joined", "django_content_type"."id", "django_content_type"."app_label", "django_content_type"."model" FROM "django_admin_log" INNER JOIN "core_user" ON ("django_admin_log"."user_id" = "core_user"."id") LEFT OUTER JOIN "django_content_type" ON ("django_admin_log"."content_type_id" = "django_content_type"."id") WHERE "django_admin_log"."user_id" = 1 ORDER BY "django_admin_log"."action_time" DESC LIMIT 10
100123 test_example 14128 16393 pupeno "" ::1 61988 2018-11-15 17:28:19.767225 2018-11-15 17:28:19.776150 2018-11-15 17:28:19.776479 idle SELECT "core_firm"."id", "core_firm"."name", "core_firm"."host_name" FROM "core_firm" WHERE "core_firm"."id" = 1
100123 test_example 9604 16393 pupeno "" ::1 61960 2018-11-15 17:28:19.469197 2018-11-15 17:28:19.478775 2018-11-15 17:28:19.478788 idle COMMIT
I've been trying to find the minimum reproducible example of this problem, but so far I haven't succeeded.
Any ideas what could be causing this or how to find out more about what the issue could be?
741
This error message...
django.db.utils.OperationalError: database "test_example" is being accessed by other users
DETAIL: There is 1 other session using the database.
...implies that there is an existing session using the database and the new session can't access the database.
Some more information regarding the Django version, Database type and version along with Selenium, ChromeDriver and Chrome versions would have helped us to debug this issue in a better way.
However, you need to take care of a couple of things from Selenium perspective as follows:
As you are initiating a new session on the next run of the tests you need to remove the line of code cls.webdriver.close() as the next line of code cls.webdriver.quit() will be enough to terminate the existing session. As per the best practices you should invoke the quit() method within the tearDown() {}. Invoking quit() DELETEs the current browsing session through sending "quit" command with {"flags":["eForceQuit"]} and finally sends the GET request on /shutdown EndPoint. Here is an example below :
1503397488598 webdriver::server DEBUG -> DELETE /session/8e457516-3335-4d3b-9140-53fb52aa8b74
1503397488607 geckodriver::marionette TRACE -> 37:[0,4,"quit",{"flags":["eForceQuit"]}]
1503397488821 webdriver::server DEBUG -> GET /shutdown
So on invoking quit() method the Web Browser session and the WebDriver instance gets killed completely. Hence you don't have to incorporate any additional steps which will be an overhead.
You can find a detailed discussion in Selenium : How to stop geckodriver process impacting PC memory, without calling driver.quit()?
So you need to:
Remove the implicitly_wait(10):
cls.webdriver.implicitly_wait(10)
Induce WebDriverWait while interacting with elements:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "id_username"))).send_keys(BACH_EMAIL)
self.webdriver.find_element_by_id("id_password").send_keys(PASSWORD)
self.webdriver.find_element_by_id("id_password").send_keys(Keys.RETURN)
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.LINK_TEXT, "Users"))).click()
Now, as per the discussion Persistent connections not closed by LiveServerTestCase, preventing dropping test databases this issue was observed, reported, discussed within Djangov1.6 and fixed. The main issue was:
Whenever a PostgreSQL connection is marked as persistent (
CONN_MAX_AGE = None) and a LiveServerTestCase is executed, the connection from the server thread is never closed, leading to inability to drop the test database.
This is exactly the reason why you see:
select * from pg_stat_activity where datname = 'test_example';
100123 test_example 29892 16393 pupeno "" ::1 61967 2018-11-15 17:28:19.552431 2018-11-15 17:28:19.562398 2018-11-15 17:28:19.564623 idle SELECT "core_user"."id", "core_user"."password", "core_user"."last_login", "core_user"."is_superuser", "core_user"."email", "core_user"."is_staff", "core_user"."is_active", "core_user"."date_joined" FROM "core_user" WHERE "core_user"."id" = 1
100123 test_example 33028 16393 pupeno "" ::1 61930 2018-11-15 17:28:18.792466 2018-11-15 17:28:18.843383 2018-11-15 17:28:18.851828 idle SELECT "django_admin_log"."id", "django_admin_log"."action_time", "django_admin_log"."user_id", "django_admin_log"."content_type_id", "django_admin_log"."object_id", "django_admin_log"."object_repr", "django_admin_log"."action_flag", "django_admin_log"."change_message", "core_user"."id", "core_user"."password", "core_user"."last_login", "core_user"."is_superuser", "core_user"."email", "core_user"."is_staff", "core_user"."is_active", "core_user"."date_joined", "django_content_type"."id", "django_content_type"."app_label", "django_content_type"."model" FROM "django_admin_log" INNER JOIN "core_user" ON ("django_admin_log"."user_id" = "core_user"."id") LEFT OUTER JOIN "django_content_type" ON ("django_admin_log"."content_type_id" = "django_content_type"."id") WHERE "django_admin_log"."user_id" = 1 ORDER BY "django_admin_log"."action_time" DESC LIMIT 10
100123 test_example 14128 16393 pupeno "" ::1 61988 2018-11-15 17:28:19.767225 2018-11-15 17:28:19.776150 2018-11-15 17:28:19.776479 idle SELECT "core_firm"."id", "core_firm"."name", "core_firm"."host_name" FROM "core_firm" WHERE "core_firm"."id" = 1
100123 test_example 9604 16393 pupeno "" ::1 61960 2018-11-15 17:28:19.469197 2018-11-15 17:28:19.478775 2018-11-15 17:28:19.478788 idle COMMIT
It was further observed that, even with CONN_MAX_AGE=None, after LiveServerTestCase.tearDownClass(), querying PostgreSQL's pg_stat_activity shows a lingering connection in state idle (which was the connection created by the previous test in your case). So it was pretty evident that the idle connections are not closed when the thread terminates and the needle of supection was at:
LiveServerThread(threading.Thread) which control the threads for running a live http server while the tests are running:
class LiveServerThread(threading.Thread):
def __init__(self, host, static_handler, connections_override=None):
self.host = host
self.port = None
self.is_ready = threading.Event()
self.error = None
self.static_handler = static_handler
self.connections_override = connections_override
super(LiveServerThread, self).__init__()
def run(self):
"""
Sets up the live server and databases, and then loops over handling
http requests.
"""
if self.connections_override:
# Override this thread's database connections with the ones
# provided by the main thread.
for alias, conn in self.connections_override.items():
connections[alias] = conn
try:
# Create the handler for serving static and media files
handler = self.static_handler(_MediaFilesHandler(WSGIHandler()))
self.httpd = self._create_server(0)
self.port = self.httpd.server_address[1]
self.httpd.set_app(handler)
self.is_ready.set()
self.httpd.serve_forever()
except Exception as e:
self.error = e
self.is_ready.set()
def _create_server(self, port):
return WSGIServer((self.host, port), QuietWSGIRequestHandler, allow_reuse_address=False)
def terminate(self):
if hasattr(self, 'httpd'):
# Stop the WSGI server
self.httpd.shutdown()
self.httpd.server_close()
LiveServerTestCase(TransactionTestCase) which basically does the same as TransactionTestCase but also launches a live http server in a separate thread so that the tests may use another testing framework to be used by Selenium, instead of the built-in dummy client:
class LiveServerTestCase(TransactionTestCase):
host = 'localhost'
static_handler = _StaticFilesHandler
@classproperty
def live_server_url(cls):
return 'http://%s:%s' % (cls.host, cls.server_thread.port)
@classmethod
def setUpClass(cls):
super(LiveServerTestCase, cls).setUpClass()
connections_override = {}
for conn in connections.all():
# If using in-memory sqlite databases, pass the connections to
# the server thread.
if conn.vendor == 'sqlite' and conn.is_in_memory_db(conn.settings_dict['NAME']):
# Explicitly enable thread-shareability for this connection
conn.allow_thread_sharing = True
connections_override[conn.alias] = conn
cls._live_server_modified_settings = modify_settings(
ALLOWED_HOSTS={'append': cls.host},
)
cls._live_server_modified_settings.enable()
cls.server_thread = cls._create_server_thread(connections_override)
cls.server_thread.daemon = True
cls.server_thread.start()
# Wait for the live server to be ready
cls.server_thread.is_ready.wait()
if cls.server_thread.error:
# Clean up behind ourselves, since tearDownClass won't get called in
# case of errors.
cls._tearDownClassInternal()
raise cls.server_thread.error
@classmethod
def _create_server_thread(cls, connections_override):
return LiveServerThread(
cls.host,
cls.static_handler,
connections_override=connections_override,
)
@classmethod
def _tearDownClassInternal(cls):
# There may not be a 'server_thread' attribute if setUpClass() for some
# reasons has raised an exception.
if hasattr(cls, 'server_thread'):
# Terminate the live server's thread
cls.server_thread.terminate()
cls.server_thread.join()
# Restore sqlite in-memory database connections' non-shareability
for conn in connections.all():
if conn.vendor == 'sqlite' and conn.is_in_memory_db(conn.settings_dict['NAME']):
conn.allow_thread_sharing = False
@classmethod
def tearDownClass(cls):
cls._tearDownClassInternal()
cls._live_server_modified_settings.disable()
super(LiveServerTestCase, cls).tearDownClass()
The solution was to close only the non-overriden connections and was incorporated from this pull request / commit. The changes were:
In django/test/testcases.py add:
finally:
connections.close_all()
Add a new file tests/servers/test_liveserverthread.py:
from django.db import DEFAULT_DB_ALIAS, connections
from django.test import LiveServerTestCase, TestCase
class LiveServerThreadTest(TestCase):
def run_live_server_thread(self, connections_override=None):
thread = LiveServerTestCase._create_server_thread(connections_override)
thread.daemon = True
thread.start()
thread.is_ready.wait()
thread.terminate()
def test_closes_connections(self):
conn = connections[DEFAULT_DB_ALIAS]
if conn.vendor == 'sqlite' and conn.is_in_memory_db():
self.skipTest("the sqlite backend's close() method is a no-op when using an in-memory database")
# Pass a connection to the thread to check they are being closed.
connections_override = {DEFAULT_DB_ALIAS: conn}
saved_sharing = conn.allow_thread_sharing
try:
conn.allow_thread_sharing = True
self.assertTrue(conn.is_usable())
self.run_live_server_thread(connections_override)
self.assertFalse(conn.is_usable())
finally:
conn.allow_thread_sharing = saved_sharing
In tests/servers/tests.py remove:
finally:
TestCase.tearDownClass()
In tests/servers/tests.py add:
finally:
if hasattr(TestCase, 'server_thread'):
TestCase.server_thread.terminate()
Steps:
You can find a similar discussion in django.db.utils.IntegrityError: FOREIGN KEY constraint failed while executing LiveServerTestCases through Selenium and Python Django
181
There has been an issue reported long back for the same (bug #22424).
One thing you need to make sure is that CONN_MAX_AGE is set to 0 and not None
Also, you can use something like below in your teardown
from django.db import connections
from django.db.utils import OperationalError
@classmethod
def tearDownClass(cls):
# Workaround for https://code.djangoproject.com/ticket/22414
# Persistent connections not closed by LiveServerTestCase, preventing dropping test databases
# https://github.com/cjerdonek/django/commit/b07fbca02688a0f8eb159f0dde132e7498aa40cc
def close_sessions(conn):
close_sessions_query = """
SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE
datname = current_database() AND
pid <> pg_backend_pid();
"""
with conn.cursor() as cursor:
try:
cursor.execute(close_sessions_query)
except OperationalError:
# We get kicked out after closing.
pass
for alias in connections:
connections[alias].close()
close_sessions(connections[alias])
print "Forcefully closed database connections."
Above code is from below URL
https://github.com/cga-harvard/Hypermap-Registry/blob/cd4efad61f18194ddab2c662aa431aa21dec03f4/hypermap/tests/test_csw.py
30
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With
