From 69299fccef1a15275f2b0b28ca15e439c2ec2f07 Mon Sep 17 00:00:00 2001 From: CrazyBolillo Date: Wed, 3 Apr 2024 21:32:51 -0600 Subject: [PATCH 1/3] Improve performance by caching find_spec Certain checkers upstream on pylint like import-error heavily use find_spec. This method is IO intensive as it looks for files across several search paths to return a ModuleSpec. Since imports across files may repeat themselves it makes sense to cache this method in order to speed up the linting process. Closes pylint-dev/pylint#9310. --- astroid/interpreter/_import/spec.py | 12 +++++++++++- astroid/manager.py | 2 ++ tests/test_manager.py | 2 ++ tests/test_modutils.py | 2 ++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index 93096e54e6..e1c5ed0155 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -16,6 +16,7 @@ import warnings import zipimport from collections.abc import Iterator, Sequence +from functools import lru_cache from pathlib import Path from typing import Any, Literal, NamedTuple, Protocol @@ -440,10 +441,15 @@ def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSp :return: A module spec, which describes how the module was found and where. """ + return _find_spec(tuple(modpath), tuple(path) if path else None) + + +@lru_cache(maxsize=1024) +def _find_spec(modpath: tuple, path: tuple) -> ModuleSpec: _path = path or sys.path # Need a copy for not mutating the argument. - modpath = modpath[:] + modpath = list(modpath) submodule_path = None module_parts = modpath[:] @@ -468,3 +474,7 @@ def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSp spec = spec._replace(submodule_search_locations=submodule_path) return spec + + +def clear_spec_cache() -> None: + _find_spec.cache_clear() diff --git a/astroid/manager.py b/astroid/manager.py index a7a51f19c5..195ac66459 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -442,10 +442,12 @@ def clear_cache(self) -> None: # pylint: disable=import-outside-toplevel from astroid.brain.helpers import register_all_brains from astroid.inference_tip import clear_inference_tip_cache + from astroid.interpreter._import.spec import clear_spec_cache from astroid.interpreter.objectmodel import ObjectModel from astroid.nodes._base_nodes import LookupMixIn from astroid.nodes.scoped_nodes import ClassDef + clear_spec_cache() clear_inference_tip_cache() _invalidate_cache() # inference context cache diff --git a/tests/test_manager.py b/tests/test_manager.py index 7861927930..160fa944bb 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -23,6 +23,7 @@ AttributeInferenceError, ) from astroid.interpreter._import import util +from astroid.interpreter._import.spec import clear_spec_cache from astroid.modutils import EXT_LIB_DIRS, module_in_path from astroid.nodes import Const from astroid.nodes.scoped_nodes import ClassDef, Module @@ -41,6 +42,7 @@ class AstroidManagerTest( ): def setUp(self) -> None: super().setUp() + clear_spec_cache() self.manager = test_utils.brainless_manager() def test_ast_from_file(self) -> None: diff --git a/tests/test_modutils.py b/tests/test_modutils.py index 929c58992c..be7095e2dd 100644 --- a/tests/test_modutils.py +++ b/tests/test_modutils.py @@ -22,6 +22,7 @@ from astroid import modutils from astroid.const import PY310_PLUS from astroid.interpreter._import import spec +from astroid.interpreter._import.spec import clear_spec_cache from . import resources @@ -41,6 +42,7 @@ class ModuleFileTest(unittest.TestCase): package = "mypypa" def tearDown(self) -> None: + clear_spec_cache() for k in list(sys.path_importer_cache): if "MyPyPa" in k: del sys.path_importer_cache[k] From 98a0380c98896a963350d98055f88b01da75cbdb Mon Sep 17 00:00:00 2001 From: CrazyBolillo Date: Sun, 28 Apr 2024 20:51:10 -0600 Subject: [PATCH 2/3] Remove AstroidCacheSetupMixin This class predates efforts to have a central interface to control global state (including caches) and it is no longer needed. --- astroid/interpreter/_import/spec.py | 4 ---- astroid/manager.py | 4 ++-- tests/resources.py | 26 -------------------------- tests/test_manager.py | 10 ++++------ tests/test_modutils.py | 3 +-- tests/test_regrtest.py | 3 ++- 6 files changed, 9 insertions(+), 41 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index e1c5ed0155..77351c2381 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -474,7 +474,3 @@ def _find_spec(modpath: tuple, path: tuple) -> ModuleSpec: spec = spec._replace(submodule_search_locations=submodule_path) return spec - - -def clear_spec_cache() -> None: - _find_spec.cache_clear() diff --git a/astroid/manager.py b/astroid/manager.py index 195ac66459..fc30bf9e2f 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -442,12 +442,11 @@ def clear_cache(self) -> None: # pylint: disable=import-outside-toplevel from astroid.brain.helpers import register_all_brains from astroid.inference_tip import clear_inference_tip_cache - from astroid.interpreter._import.spec import clear_spec_cache + from astroid.interpreter._import.spec import _find_spec from astroid.interpreter.objectmodel import ObjectModel from astroid.nodes._base_nodes import LookupMixIn from astroid.nodes.scoped_nodes import ClassDef - clear_spec_cache() clear_inference_tip_cache() _invalidate_cache() # inference context cache @@ -461,6 +460,7 @@ def clear_cache(self) -> None: util.is_namespace, ObjectModel.attributes, ClassDef._metaclass_lookup_attribute, + _find_spec, ): lru_cache.cache_clear() # type: ignore[attr-defined] diff --git a/tests/resources.py b/tests/resources.py index 455dc6fb69..853fd796f1 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -9,7 +9,6 @@ from pathlib import Path from astroid import builder -from astroid.manager import AstroidManager from astroid.nodes.scoped_nodes import Module DATA_DIR = Path("testdata") / "python3" @@ -34,28 +33,3 @@ def tearDown(self) -> None: for key in list(sys.path_importer_cache): if key.startswith(datadir): del sys.path_importer_cache[key] - - -class AstroidCacheSetupMixin: - """Mixin for handling test isolation issues with the astroid cache. - - When clearing the astroid cache, some tests fail due to - cache inconsistencies, where some objects had a different - builtins object referenced. - This saves the builtins module and TransformVisitor and - replaces them after the tests finish. - The builtins module is special, since some of the - transforms for a couple of its objects (str, bytes etc) - are executed only once, so astroid_bootstrapping will be - useless for retrieving the original builtins module. - """ - - @classmethod - def setup_class(cls): - cls._builtins = AstroidManager().astroid_cache.get("builtins") - cls._transforms = AstroidManager.brain["_transform"] - - @classmethod - def teardown_class(cls): - AstroidManager().astroid_cache["builtins"] = cls._builtins - AstroidManager.brain["_transform"] = cls._transforms diff --git a/tests/test_manager.py b/tests/test_manager.py index 160fa944bb..c91ec0a4cf 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -23,7 +23,6 @@ AttributeInferenceError, ) from astroid.interpreter._import import util -from astroid.interpreter._import.spec import clear_spec_cache from astroid.modutils import EXT_LIB_DIRS, module_in_path from astroid.nodes import Const from astroid.nodes.scoped_nodes import ClassDef, Module @@ -37,13 +36,11 @@ def _get_file_from_object(obj) -> str: return obj.__file__ -class AstroidManagerTest( - resources.SysPathSetup, resources.AstroidCacheSetupMixin, unittest.TestCase -): +class AstroidManagerTest(resources.SysPathSetup, unittest.TestCase): def setUp(self) -> None: super().setUp() - clear_spec_cache() self.manager = test_utils.brainless_manager() + self.manager.clear_cache() def test_ast_from_file(self) -> None: filepath = unittest.__file__ @@ -393,9 +390,10 @@ def test_denied_modules_raise(self) -> None: self.manager.ast_from_module_name("math") -class IsolatedAstroidManagerTest(resources.AstroidCacheSetupMixin, unittest.TestCase): +class IsolatedAstroidManagerTest(unittest.TestCase): def test_no_user_warning(self): mgr = manager.AstroidManager() + self.addCleanup(mgr.clear_cache) with warnings.catch_warnings(): warnings.filterwarnings("error", category=UserWarning) mgr.ast_from_module_name("setuptools") diff --git a/tests/test_modutils.py b/tests/test_modutils.py index be7095e2dd..7921757621 100644 --- a/tests/test_modutils.py +++ b/tests/test_modutils.py @@ -22,7 +22,6 @@ from astroid import modutils from astroid.const import PY310_PLUS from astroid.interpreter._import import spec -from astroid.interpreter._import.spec import clear_spec_cache from . import resources @@ -42,7 +41,7 @@ class ModuleFileTest(unittest.TestCase): package = "mypypa" def tearDown(self) -> None: - clear_spec_cache() + astroid.MANAGER.clear_cache() for k in list(sys.path_importer_cache): if "MyPyPa" in k: del sys.path_importer_cache[k] diff --git a/tests/test_regrtest.py b/tests/test_regrtest.py index 67ccca630f..45f241f8cf 100644 --- a/tests/test_regrtest.py +++ b/tests/test_regrtest.py @@ -26,10 +26,11 @@ HAS_NUMPY = True -class NonRegressionTests(resources.AstroidCacheSetupMixin, unittest.TestCase): +class NonRegressionTests(unittest.TestCase): def setUp(self) -> None: sys.path.insert(0, resources.find("data")) MANAGER.always_load_extensions = True + self.addCleanup(MANAGER.clear_cache) def tearDown(self) -> None: MANAGER.always_load_extensions = False From 82c839ef26abb503978e2177f9fcae7ca51a0134 Mon Sep 17 00:00:00 2001 From: CrazyBolillo Date: Sat, 4 May 2024 15:10:29 -0600 Subject: [PATCH 3/3] Fix mypy warnings and update typing --- astroid/interpreter/_import/spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index 77351c2381..f4398f4642 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -15,7 +15,7 @@ import types import warnings import zipimport -from collections.abc import Iterator, Sequence +from collections.abc import Iterable, Iterator, Sequence from functools import lru_cache from pathlib import Path from typing import Any, Literal, NamedTuple, Protocol @@ -424,7 +424,7 @@ def _find_spec_with_path( raise ImportError(f"No module named {'.'.join(module_parts)}") -def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSpec: +def find_spec(modpath: Iterable[str], path: Iterable[str] | None = None) -> ModuleSpec: """Find a spec for the given module. :type modpath: list or tuple @@ -445,11 +445,11 @@ def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSp @lru_cache(maxsize=1024) -def _find_spec(modpath: tuple, path: tuple) -> ModuleSpec: +def _find_spec(module_path: tuple, path: tuple) -> ModuleSpec: _path = path or sys.path # Need a copy for not mutating the argument. - modpath = list(modpath) + modpath = list(module_path) submodule_path = None module_parts = modpath[:]