AssertionError: /home/circleci/.pyenv/versions/3.8.17/lib/python3.8/distutils/core.py with python 3.8.17

In our repository,

=================================== FAILURES ===================================
____________________ test[ mlflow/tracking/fluent.py:1834 ] ____________________

_ = ' mlflow/tracking/fluent.py:1834 '

    @pytest.mark.parametrize('_', [' mlflow/tracking/fluent.py:1834 '])
    def test(_):
        import mlflow
    
        mlflow.autolog(log_models=False, exclusive=True)
    
        import sklearn
    
>       mlflow.sklearn.autolog(log_models=True)

_          = ' mlflow/tracking/fluent.py:1834 '
mlflow     = <module 'mlflow' from '/home/circleci/project/mlflow/__init__.py'>
sklearn    = <module 'sklearn' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/__init__.py'>

.examples/test_mlflow_tracking_fluent_autolog_21.py:13: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../mlflow/utils/autologging_utils/__init__.py:414: in autolog
    return _autolog(*args, **kwargs)
        _autolog   = <function autolog at 0x7f6a382b13a0>
        args       = ()
        config_to_store = {'disable': False,
 'disable_for_unsupported_versions': False,
 'exclusive': False,
 'log_datasets': True,
 'log_input_examples': False,
 'log_model_signatures': True,
 'log_models': True,
 'log_post_training_metrics': True,
 'max_tuning_runs': 5,
 'pos_label': None,
 'registered_model_name': None,
 'serialization_format': 'cloudpickle',
 'silent': False}
        default_params = {'disable': False,
 'disable_for_unsupported_versions': False,
 'exclusive': False,
 'log_datasets': True,
 'log_input_examples': False,
 'log_model_signatures': True,
 'log_models': True,
 'log_post_training_metrics': True,
 'max_tuning_runs': 5,
 'pos_label': None,
 'registered_model_name': None,
 'serialization_format': 'cloudpickle',
 'silent': False}
        is_silent_mode = False
        kwargs     = {'log_models': True}
        name       = 'sklearn'
        param_spec = mappingproxy(OrderedDict([('log_input_examples',
                           <Parameter "log_input_examples=False">),
                          ('log_model_signatures',
                           <Parameter "log_model_signatures=True">),
                          ('log_models', <Parameter "log_models=True">),
                          ('log_datasets', <Parameter "log_datasets=True">),
                          ('disable', <Parameter "disable=False">),
                          ('exclusive', <Parameter "exclusive=False">),
                          ('disable_for_unsupported_versions',
                           <Parameter "disable_for_unsupported_versions=False">),
                          ('silent', <Parameter "silent=False">),
                          ('max_tuning_runs', <Parameter "max_tuning_runs=5">),
                          ('log_post_training_metrics',
                           <Parameter "log_post_training_metrics=True">),
                          ('serialization_format',
                           <Parameter "serialization_format='cloudpickle'">),
                          ('registered_model_name',
                           <Parameter "registered_model_name=None">),
                          ('pos_label', <Parameter "pos_label=None">)]))
../mlflow/sklearn/__init__.py:1239: in autolog
    _autolog(
        disable    = False
        disable_for_unsupported_versions = False
        exclusive  = False
        log_datasets = True
        log_input_examples = False
        log_model_signatures = True
        log_models = True
        log_post_training_metrics = True
        max_tuning_runs = 5
        pos_label  = None
        registered_model_name = None
        serialization_format = 'cloudpickle'
        silent     = False
../mlflow/sklearn/__init__.py:1799: in _autolog
    estimators_to_patch = _gen_estimators_to_patch()
        _TRAINING_PREFIX = 'training_'
        _all_estimators = <function _all_estimators at 0x7f6aeb2d7280>
        _apply_sklearn_descriptor_unbound_method_call_fix = <function _autolog.<locals>._apply_sklearn_descriptor_unbound_method_call_fix at 0x7f6aeb2d7820>
        _create_child_runs_for_parameter_search = <function _create_child_runs_for_parameter_search at 0x7f6aeb2d70d0>
        _gen_lightgbm_sklearn_estimators_to_patch = <function _gen_lightgbm_sklearn_estimators_to_patch at 0x7f6aeb2de670>
        _gen_xgboost_sklearn_estimators_to_patch = <function _gen_xgboost_sklearn_estimators_to_patch at 0x7f6aeb2de5e0>
        _get_X_y_and_sample_weight = <function _get_X_y_and_sample_weight at 0x7f6aeb2de790>
        _get_estimator_info_tags = <function _get_estimator_info_tags at 0x7f6aeb2de700>
        _get_meta_estimators_for_autologging = <function _get_meta_estimators_for_autologging at 0x7f6aeb2dee50>
        _is_parameter_search_estimator = <function _is_parameter_search_estimator at 0x7f6aeb2deee0>
        _log_estimator_content = <function _log_estimator_content at 0x7f6aeb2dedc0>
        _log_parameter_search_results_as_artifact = <function _log_parameter_search_results_as_artifact at 0x7f6aeb2def70>
        _log_posttraining_metadata = <function _autolog.<locals>._log_posttraining_metadata at 0x7f6aeb2d7700>
        _log_pretraining_metadata = <function _autolog.<locals>._log_pretraining_metadata at 0x7f6aeb2d7790>
        context_registry = <module 'mlflow.tracking.context.registry' from '/home/circleci/project/mlflow/tracking/context/registry.py'>
        disable    = False
        disable_for_unsupported_versions = False
        exclusive  = False
        fit_mlflow = <function _autolog.<locals>.fit_mlflow at 0x7f6b00b150d0>
        fit_mlflow_xgboost_and_lightgbm = <function _autolog.<locals>.fit_mlflow_xgboost_and_lightgbm at 0x7f6b009f35e0>
        flavor_name = 'sklearn'
        infer_signature = <function infer_signature at 0x7f6aa6e75a60>
        log_datasets = True
        log_input_examples = False
        log_model_signatures = True
        log_models = True
        log_post_training_metrics = True
        max_tuning_runs = 5
        patched_fit = <function _autolog.<locals>.patched_fit at 0x7f6aeb2d7670>
        patched_metric_api = <function _autolog.<locals>.patched_metric_api at 0x7f6aeb2d7550>
        patched_model_score = <function _autolog.<locals>.patched_model_score at 0x7f6aeb2d73a0>
        patched_predict = <function _autolog.<locals>.patched_predict at 0x7f6aeb2d75e0>
        pd         = <module 'pandas' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/pandas/__init__.py'>
        pos_label  = None
        serialization_format = 'cloudpickle'
        silent     = False
        sklearn    = <module 'sklearn' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/__init__.py'>
../mlflow/sklearn/__init__.py:97: in _gen_estimators_to_patch
    _, estimators_to_patch = zip(*_all_estimators())
        _all_estimators = <function _all_estimators at 0x7f6aeb2d7280>
        _get_meta_estimators_for_autologging = <function _get_meta_estimators_for_autologging at 0x7f6aeb2dee50>
../mlflow/sklearn/utils.py:856: in _all_estimators
    return all_estimators()
        all_estimators = <function all_estimators at 0x7f6b005e9430>
../../.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/utils/discovery.py:63: in all_estimators
    for _, module_name, _ in pkgutil.walk_packages(path=[root], prefix="sklearn."):
        BaseEstimator = <class 'sklearn.base.BaseEstimator'>
        ClassifierMixin = <class 'sklearn.base.ClassifierMixin'>
        ClusterMixin = <class 'sklearn.base.ClusterMixin'>
        IS_PYPY    = False
        RegressorMixin = <class 'sklearn.base.RegressorMixin'>
        TransformerMixin = <class 'sklearn.base.TransformerMixin'>
        _          = True
        all_classes = []
        ignore_warnings = <function ignore_warnings at 0x7f6aeb2d7c10>
        is_abstract = <function all_estimators.<locals>.is_abstract at 0x7f6aeb185af0>
        module_name = 'sklearn._build_utils'
        module_parts = ['sklearn', '_build_utils']
        root       = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn'
        type_filter = None
../../.pyenv/versions/3.8.17/lib/python3.8/pkgutil.py:92: in walk_packages
    __import__(info.name)
        info       = ModuleInfo(module_finder=FileFinder('/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn'), name='sklearn._build_utils', ispkg=True)
        onerror    = None
        path       = ['/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/__check_build']
        prefix     = 'sklearn.'
        seen       = <function walk_packages.<locals>.seen at 0x7f6aeb185d30>
../../.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils/__init__.py:12: in <module>
    from .pre_build_helpers import basic_check_build
        __builtins__ = <builtins>
        __cached__ = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils/__pycache__/__init__.cpython-38.pyc'
        __doc__    = '\nUtilities useful during the build.\n'
        __file__   = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils/__init__.py'
        __loader__ = <_frozen_importlib_external.SourceFileLoader object at 0x7f6aeb16dcd0>
        __name__   = 'sklearn._build_utils'
        __package__ = 'sklearn._build_utils'
        __path__   = ['/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils']
        __spec__   = ModuleSpec(name='sklearn._build_utils', loader=<_frozen_importlib_external.SourceFileLoader object at 0x7f6aeb16dcd0>, origin='/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils/__init__.py', submodule_search_locations=['/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils'])
        contextlib = <module 'contextlib' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/contextlib.py'>
        os         = <module 'os' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/os.py'>
        sklearn    = <module 'sklearn' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/__init__.py'>
../../.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils/pre_build_helpers.py:10: in <module>
    from setuptools.command.build_ext import customize_compiler, new_compiler
        __builtins__ = <builtins>
        __cached__ = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils/__pycache__/pre_build_helpers.cpython-38.pyc'
        __doc__    = 'Helpers to check build environment before actual build of scikit-learn'
        __file__   = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils/pre_build_helpers.py'
        __loader__ = <_frozen_importlib_external.SourceFileLoader object at 0x7f6aeb16df10>
        __name__   = 'sklearn._build_utils.pre_build_helpers'
        __package__ = 'sklearn._build_utils'
        __spec__   = ModuleSpec(name='sklearn._build_utils.pre_build_helpers', loader=<_frozen_importlib_external.SourceFileLoader object at 0x7f6aeb16df10>, origin='/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/sklearn/_build_utils/pre_build_helpers.py')
        glob       = <module 'glob' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/glob.py'>
        os         = <module 'os' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/os.py'>
        subprocess = <module 'subprocess' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/subprocess.py'>
        sys        = <module 'sys' (built-in)>
        tempfile   = <module 'tempfile' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/tempfile.py'>
        textwrap   = <module 'textwrap' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/textwrap.py'>
../../.pyenv/versions/3.8.17/lib/python3.8/site-packages/setuptools/__init__.py:7: in <module>
    import _distutils_hack.override  # noqa: F401
        __builtins__ = <builtins>
        __cached__ = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/setuptools/__pycache__/__init__.cpython-38.pyc'
        __doc__    = "Extensions to the 'distutils' for large or complex distributions"
        __file__   = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/setuptools/__init__.py'
        __loader__ = <_frozen_importlib_external.SourceFileLoader object at 0x7f6aeb16d9a0>
        __name__   = 'setuptools'
        __package__ = 'setuptools'
        __path__   = ['/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/setuptools']
        __spec__   = ModuleSpec(name='setuptools', loader=<_frozen_importlib_external.SourceFileLoader object at 0x7f6aeb16d9a0>, origin='/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/setuptools/__init__.py', submodule_search_locations=['/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/setuptools'])
        functools  = <module 'functools' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/functools.py'>
        os         = <module 'os' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/os.py'>
        re         = <module 're' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/re.py'>
../../.pyenv/versions/3.8.17/lib/python3.8/site-packages/_distutils_hack/override.py:1: in <module>
    __import__('_distutils_hack').do_override()
        __builtins__ = <builtins>
        __cached__ = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/_distutils_hack/__pycache__/override.cpython-38.pyc'
        __doc__    = None
        __file__   = '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/_distutils_hack/override.py'
        __loader__ = <_frozen_importlib_external.SourceFileLoader object at 0x7f6aeb1aa6a0>
        __name__   = '_distutils_hack.override'
        __package__ = '_distutils_hack'
        __spec__   = ModuleSpec(name='_distutils_hack.override', loader=<_frozen_importlib_external.SourceFileLoader object at 0x7f6aeb1aa6a0>, origin='/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/site-packages/_distutils_hack/override.py')
../../.pyenv/versions/3.8.17/lib/python3.8/site-packages/_distutils_hack/__init__.py:77: in do_override
    ensure_local_distutils()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    def ensure_local_distutils():
        import importlib
    
        clear_distutils()
    
        # With the DistutilsMetaFinder in place,
        # perform an import to cause distutils to be
        # loaded from setuptools._distutils. Ref #2906.
        with shim():
            importlib.import_module('distutils')
    
        # check that submodules load as expected
        core = importlib.import_module('distutils.core')
πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡πŸ‘‡
>       assert '_distutils' in core.__file__, core.__file__
E       AssertionError: /home/circleci/.pyenv/versions/3.8.17/lib/python3.8/distutils/core.py

core       = <module 'distutils.core' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/distutils/core.py'>
importlib  = <module 'importlib' from '/home/circleci/.pyenv/versions/3.8.17/lib/python3.8/importlib/__init__.py'>

../../.pyenv/versions/3.8.17/lib/python3.8/site-packages/_distutils_hack/__init__.py:64: AssertionError

Since I can’t edit the issue description, I’m writing here. In our repository (mlflow/mlflow), one of our circleci jobs started failing after the python version switched to 3.8.17 from 3.8.16.