Menu
I'm trying to run a test that was previously working but has suddenly stopped running but now i seem to get an error on all my tests e.g.
from httmock import HTTMock
from unittest import TestCase
from unittest.mock import patch, call, mock_open, MagicMock, Mock, ANY
import os.path
import os
from src.operators import InjestDictDescriptionOperator
from airflow.hooks.base_hook import BaseHook
from airflow.hooks.postgres_hook import PostgresHook
from airflow.hooks.S3_hook import S3Hook
class TestInjestDictDescriptionOperator(TestCase):
def setUp(self):
# hook patches
self.open_file_mock = patch('builtins.open').start()
self.os_path_isdir = patch.object(os.path, 'isdir').start()
self.os_makedirs = patch.object(os, 'makedirs').start()
self.open_file_write_mock = self.open_file_mock.return_value.__enter__.return_value.write
# prepare the target
self.target = InjestDictDescriptionOperator(
task_id='InjestDictDescriptionOperatorTest',
sql=None,
postgres_conn_id='test',
aws_conn_id='s3-conn-1',
s3_bucket_name=‘data’,
output_path='output/path/1')
def tearDown(self):
patch.stopall()
def testTmpFolderCreationIfItDoesntExist(self):
self.os_path_isdir.return_value = False
self.target.execute(None)
self.os_makedirs.assert_called_with('/tmp/')
def testTmpFolderNotCreatedIfItExists(self):
self.os_path_isdir.return_value = True
self.target.execute(None)
self.os_makedirs.assert_not_called()
def testTmpFileCreation(self):
self.target.execute(None)
self.open_file_mock.assert_called_with(
'/tmp/modelling/temp.txt',
'w+',
encoding='utf-8')
def testTmpFileDataDump(self):
self.target.execute(None)
self.open_file_write_mock.assert_has_calls(
[ call(f"{doc['name']}\n") for doc in self.dummy_data ]
, any_order=False)
The traceback details of the problem are
ests/operators/modelling/language/test_injest_dict_description_operator.py:9: in <module>
from src.operators.modelling.language import InjestDictDescriptionOperator
src/operators/modelling/language/__init__.py:1: in <module>
from .injest_onboarded_commands_operator import Operator as InjestOnboardedCommandsOperator
src/operators/modelling/language/injest_onboarded_commands_operator.py:9: in <module>
from airflow.models import BaseOperator
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/__init__.py:50: in <module>
from airflow.models import DAG # noqa: E402
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/__init__.py:21: in <module>
from airflow.models.baseoperator import BaseOperator, BaseOperatorLink # noqa: F401
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/baseoperator.py:43: in <module>
from airflow.models.dag import DAG
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dag.py:52: in <module>
from airflow.models.dagbag import DagBag
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:50: in <module>
class DagBag(BaseDagBag, LoggingMixin):
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:80: in DagBag
DAGBAG_IMPORT_TIMEOUT = conf.getint('core', 'DAGBAG_IMPORT_TIMEOUT')
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/configuration.py:414: in getint
return int(self.get(section, key, **kwargs))
E ValueError: invalid literal for int() with base 10: '30.0'collection failure
tests/operators/modelling/language/test_injest_dict_description_operator.py:9: in <module>
from src.operators.modelling.language import InjestDictDescriptionOperator
src/operators/modelling/language/__init__.py:1: in <module>
from .injest_onboarded_commands_operator import Operator as InjestOnboardedCommandsOperator
src/operators/modelling/language/injest_onboarded_commands_operator.py:9: in <module>
from airflow.models import BaseOperator
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/__init__.py:50: in <module>
from airflow.models import DAG # noqa: E402
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/__init__.py:21: in <module>
from airflow.models.baseoperator import BaseOperator, BaseOperatorLink # noqa: F401
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/baseoperator.py:43: in <module>
from airflow.models.dag import DAG
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dag.py:52: in <module>
from airflow.models.dagbag import DagBag
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:50: in <module>
class DagBag(BaseDagBag, LoggingMixin):
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:80: in DagBag
DAGBAG_IMPORT_TIMEOUT = conf.getint('core', 'DAGBAG_IMPORT_TIMEOUT')
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/configuration.py:414: in getint
return int(self.get(section, key, **kwargs))
E ValueError: invalid literal for int() with base 10: '30.0'
The only thing that changed was apache-airflow. I upgraded to the latest version 2.0 but then realised i would need to refactor parts of my code so downgraded to a later version.
254
The Python ValueError: invalid literal for int() with base 10 error is raised when you try to convert a string value that is not formatted as an integer. To solve this problem, you can use the float() method to convert a floating-point number in a string to an integer.
ValueError: invalid literal for int() with base 10 occurs when you convert the string or decimal or characters values not formatted as an integer. To solve the error, you can use the float() method to convert entered decimal input and then use the int() method to convert your number to an integer.
Happened to me after I installed Airflow 2 on accident and then downgraded to 1.10.12. Solution was to remove ~/airflow after the downgrade and let it re-create since this is just my test machine.
156
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With
