ValueError: 以 10 为基数的 int() 的文字无效：运行 unittest 时为“30.0”

Question

我正在尝试运行一个以前可以运行但突然停止运行的测试，但现在我似乎在所有测试中都出现错误，例如

from httmock import HTTMock
from unittest import TestCase
from unittest.mock import patch, call, mock_open, MagicMock, Mock, ANY

import os.path
import os


from src.operators import InjestDictDescriptionOperator
from airflow.hooks.base_hook import BaseHook
from airflow.hooks.postgres_hook import PostgresHook
from airflow.hooks.S3_hook import S3Hook

class TestInjestDictDescriptionOperator(TestCase):
    def setUp(self):
        # hook patches
        self.open_file_mock = patch('builtins.open').start()
        self.os_path_isdir = patch.object(os.path, 'isdir').start()
        self.os_makedirs = patch.object(os, 'makedirs').start()
        self.open_file_write_mock = self.open_file_mock.return_value.__enter__.return_value.write


        # prepare the target
        self.target = InjestDictDescriptionOperator(
            task_id='InjestDictDescriptionOperatorTest',
            sql=None,
            postgres_conn_id='test',
            aws_conn_id='s3-conn-1',
            s3_bucket_name=‘data’,
            output_path='output/path/1')

    def tearDown(self):
        patch.stopall()

    def testTmpFolderCreationIfItDoesntExist(self):
        self.os_path_isdir.return_value = False
        self.target.execute(None)
        self.os_makedirs.assert_called_with('/tmp/')

    def testTmpFolderNotCreatedIfItExists(self):
        self.os_path_isdir.return_value = True
        self.target.execute(None)
        self.os_makedirs.assert_not_called()

    def testTmpFileCreation(self):
        self.target.execute(None)
        self.open_file_mock.assert_called_with(
            '/tmp/modelling/temp.txt',
            'w+',
            encoding='utf-8')

    def testTmpFileDataDump(self):
        self.target.execute(None)
        self.open_file_write_mock.assert_has_calls(
            [ call(f"{doc['name']}\n") for doc in self.dummy_data ]
            , any_order=False)

问题的回溯细节是

ests/operators/modelling/language/test_injest_dict_description_operator.py:9: in <module>
    from src.operators.modelling.language import InjestDictDescriptionOperator
src/operators/modelling/language/__init__.py:1: in <module>
    from .injest_onboarded_commands_operator import Operator as InjestOnboardedCommandsOperator
src/operators/modelling/language/injest_onboarded_commands_operator.py:9: in <module>
    from airflow.models import BaseOperator
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/__init__.py:50: in <module>
    from airflow.models import DAG  # noqa: E402
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/__init__.py:21: in <module>
    from airflow.models.baseoperator import BaseOperator, BaseOperatorLink  # noqa: F401
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/baseoperator.py:43: in <module>
    from airflow.models.dag import DAG
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dag.py:52: in <module>
    from airflow.models.dagbag import DagBag
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:50: in <module>
    class DagBag(BaseDagBag, LoggingMixin):
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:80: in DagBag
    DAGBAG_IMPORT_TIMEOUT = conf.getint('core', 'DAGBAG_IMPORT_TIMEOUT')
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/configuration.py:414: in getint
    return int(self.get(section, key, **kwargs))
E   ValueError: invalid literal for int() with base 10: '30.0'collection failure
tests/operators/modelling/language/test_injest_dict_description_operator.py:9: in <module>
    from src.operators.modelling.language import InjestDictDescriptionOperator
src/operators/modelling/language/__init__.py:1: in <module>
    from .injest_onboarded_commands_operator import Operator as InjestOnboardedCommandsOperator
src/operators/modelling/language/injest_onboarded_commands_operator.py:9: in <module>
    from airflow.models import BaseOperator
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/__init__.py:50: in <module>
    from airflow.models import DAG  # noqa: E402
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/__init__.py:21: in <module>
    from airflow.models.baseoperator import BaseOperator, BaseOperatorLink  # noqa: F401
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/baseoperator.py:43: in <module>
    from airflow.models.dag import DAG
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dag.py:52: in <module>
    from airflow.models.dagbag import DagBag
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:50: in <module>
    class DagBag(BaseDagBag, LoggingMixin):
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:80: in DagBag
    DAGBAG_IMPORT_TIMEOUT = conf.getint('core', 'DAGBAG_IMPORT_TIMEOUT')
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/configuration.py:414: in getint
    return int(self.get(section, key, **kwargs))
E   ValueError: invalid literal for int() with base 10: '30.0'

唯一改变的是apache-airflow。我升级到最新版本 2.0，但后来意识到我需要重构部分代码，因此降级到更高版本。

Answer 1

在我意外安装 Airflow 2 然后降级到 1.10.12 后发生在我身上。解决方案是在降级后删除 ~/airflow 并让它重新创建，因为这只是我的测试机器。

Answer 2

DAGBAG_IMPORT_TIMEOUT 已在配置文件中升级为浮动，对于 2.0，对于 1.10.14，它需要是浮动的。

完全删除airflow，包括cfg文件并重新安装该版本

Answer 3

更新airflow.cfg文件。搜索变量dagbag_import_timeout并将其值类型更新为整数而不是浮点数，从dagbag_import_timeout = 30.0到dagbag_import_timeout = 30。这应该可以正常工作。