diff --git a/docs/source/metadata.rst b/docs/source/metadata.rst index 29c821a9..c7e9ae30 100644 --- a/docs/source/metadata.rst +++ b/docs/source/metadata.rst @@ -75,8 +75,9 @@ Metadata Providers :class:`~libcst.metadata.WhitespaceInclusivePositionProvider`, :class:`~libcst.metadata.ExpressionContextProvider`, :class:`~libcst.metadata.ScopeProvider`, -:class:`~libcst.metadata.QualifiedNameProvider`, and -:class:`~libcst.metadata.ParentNodeProvider` +:class:`~libcst.metadata.QualifiedNameProvider`, +:class:`~libcst.metadata.ParentNodeProvider`, and +:class:`~libcst.metadata.TypeInferenceProvider` are currently provided. Each metadata provider may has its own custom data structure. Position Metadata @@ -183,3 +184,21 @@ We provide :class:`~libcst.metadata.ParentNodeProvider` for those use cases. .. autoclass:: libcst.metadata.ParentNodeProvider :no-undoc-members: + +Type Inference Metadata +----------------------- +`Type inference `_ is to automatically infer +data types of expression for deeper understanding source code. +In Python, type checkers like `Mypy `_ or +`Pyre `_ analyze `type annotations `_ +and infer types for expressions. +:class:`~libcst.metadata.TypeInferenceProvider` is provided by `Pyre Query API `_ +which requires `setup watchman `_ for incremental typechecking. +:class:`~libcst.metadata.FullRepoManger` is built for manage the inter process communication to Pyre. + +.. autoclass:: libcst.metadata.TypeInferenceProvider + :no-undoc-members: + +.. autoclass:: libcst.metadata.FullRepoManager + :no-undoc-members: + :special-members: __init__ diff --git a/libcst/metadata/__init__.py b/libcst/metadata/__init__.py index a8a5952f..294ec713 100644 --- a/libcst/metadata/__init__.py +++ b/libcst/metadata/__init__.py @@ -16,6 +16,7 @@ from libcst.metadata.expression_context_provider import ( ExpressionContext, ExpressionContextProvider, ) +from libcst.metadata.full_repo_manager import FullRepoManager from libcst.metadata.parent_node_provider import ParentNodeProvider from libcst.metadata.position_provider import ( BasicPositionProvider, @@ -79,6 +80,7 @@ __all__ = [ "Assignments", "Accesses", "TypeInferenceProvider", + "FullRepoManager", # Experimental APIs: "ExperimentalReentrantCodegenProvider", "CodegenPartial", diff --git a/libcst/metadata/full_repo_manager.py b/libcst/metadata/full_repo_manager.py index 47efb519..2d4ce883 100644 --- a/libcst/metadata/full_repo_manager.py +++ b/libcst/metadata/full_repo_manager.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import ( TYPE_CHECKING, Callable, + Collection, Dict, List, Mapping, @@ -39,10 +40,21 @@ class FullRepoManager: def __init__( self, repo_root_dir: str, - paths: List[str], - providers: List["ProviderT"], + paths: Collection[str], + providers: Collection["ProviderT"], timeout: int = 5, ) -> None: + """ + Given project root directory with pyre and watchman setup, :class:`~libcst.metadata.FullRepoManager` + handles the inter process communication to read the required full repository cache data for + metadata provider like :class:`~libcst.metadata.TypeInferenceProvider`. + + :param paths: a collection of paths to access full repository data. + :param providers: a collection of metadata provider classes require accessing full repository + data, currently supports :class:`~libcst.metadata.TypeInferenceProvider`. + :param timeout: number of seconds. Raises `TimeoutExpired `_ + when timeout. + """ self.root_path: Path = Path(repo_root_dir) self._cache: Dict["ProviderT", Mapping[str, object]] = {} self._timeout = timeout @@ -80,15 +92,42 @@ class FullRepoManager: cache[provider] = handler(self._paths) self._cache = cache - def get_metadata_wrapper_for_path(self, path: str) -> MetadataWrapper: + def get_cache_for_path(self, path: str) -> Mapping["ProviderT", object]: + """ + Retrieve cache for a source file. The file needs to appear in the ``paths`` parameter when + constructing :class:`~libcst.metadata.FullRepoManager`. + + .. code-block:: python + + manager = FullRepoManager(".", {"a.py", "b.py"}, {TypeInferenceProvider}) + MetadataWrapper(module, cache=manager.get_cache_for_path("a.py")) + """ + if path not in self._paths: + raise Exception( + "The path needs to be in paths parameter when constructing FullRepoManager for efficient batch processing." + ) self._resolve_cache() - module = cst.parse_module((self.root_path / path).read_text()) - cache = { + return { provider: data for provider, files in self._cache.items() for _path, data in files.items() if _path == path } + + def get_metadata_wrapper_for_path(self, path: str) -> MetadataWrapper: + """ + Create a :class:`~libcst.metadata.MetadataWrapper` given a source file path. + The path needs to be a path relative to project root directory. + The source code is read and parsed as :class:`~libcst.Module` for + :class:`~libcst.metadata.MetadataWrapper`. + + .. code-block:: python + + manager = FullRepoManager(".", {"a.py", "b.py"}, {TypeInferenceProvider}) + wrapper = manager.get_metadata_wrapper_for_path("a.py") + """ + module = cst.parse_module((self.root_path / path).read_text()) + cache = self.get_cache_for_path(path) return MetadataWrapper(module, True, cache) diff --git a/libcst/metadata/tests/test_full_repo_manager.py b/libcst/metadata/tests/test_full_repo_manager.py index 9686cd96..eb5294d5 100644 --- a/libcst/metadata/tests/test_full_repo_manager.py +++ b/libcst/metadata/tests/test_full_repo_manager.py @@ -22,7 +22,7 @@ class FullRepoManagerTest(UnitTest): def test_get_metadata_wrapper_with_empty_cache(self, mocked_handler: Mock) -> None: path = "tests/pyre/simple_class.py" mocked_handler.return_value = {path: {"types": []}} - manager = FullRepoManager(REPO_ROOT_DIR, [], [TypeInferenceProvider]) + manager = FullRepoManager(REPO_ROOT_DIR, [path], [TypeInferenceProvider]) wrapper = manager.get_metadata_wrapper_for_path(path) self.assertEqual(wrapper.resolve(TypeInferenceProvider), {}) @@ -35,6 +35,19 @@ class FullRepoManagerTest(UnitTest): mocked_handler.return_value = { path: json.loads((Path(REPO_ROOT_DIR) / f"{path_prefix}.json").read_text()) } - manager = FullRepoManager(REPO_ROOT_DIR, [], [TypeInferenceProvider]) + manager = FullRepoManager(REPO_ROOT_DIR, [path], [TypeInferenceProvider]) wrapper = manager.get_metadata_wrapper_for_path(path) _test_simple_class_helper(self, wrapper) + + @patch.object(FullRepoManager, "_handle_pyre_cache") + def test_get_metadata_wrapper_with_invalid_path(self, mocked_handler: Mock) -> None: + path = "tests/pyre/simple_class.py" + mocked_handler.return_value = {path: {"types": []}} + manager = FullRepoManager( + REPO_ROOT_DIR, ["invalid_path.py"], [TypeInferenceProvider] + ) + with self.assertRaisesRegex( + Exception, + "The path needs to be in paths parameter when constructing FullRepoManager for efficient batch processing.", + ): + manager.get_metadata_wrapper_for_path(path) diff --git a/libcst/metadata/type_inference_provider.py b/libcst/metadata/type_inference_provider.py index eb9d77f7..19cb7581 100644 --- a/libcst/metadata/type_inference_provider.py +++ b/libcst/metadata/type_inference_provider.py @@ -11,8 +11,8 @@ from mypy_extensions import TypedDict import libcst as cst from libcst._position import CodePosition, CodeRange -from libcst.metadata import PositionProvider from libcst.metadata.base_provider import BatchableMetadataProvider +from libcst.metadata.position_provider import PositionProvider class Position(TypedDict): @@ -52,6 +52,19 @@ class PyreData(TypedDict): class TypeInferenceProvider(BatchableMetadataProvider[str]): + """ + Access inferred type annotation through `Pyre Query API `_. + It requires `setup watchman `_ + and start pyre server by running ``pyre`` command. + The inferred type is a string of `type annotation `_. + E.g. ``typing.List[libcst._nodes.expression.Name]`` + is the inferred type of name ``n`` in expression ``n = [cst.Name("")]``. + All name references use the fully qualified name regardless how the names are imported. + (e.g. ``import libcst; libcst.Name`` and ``import libcst as cst; cst.Name`` refer to the same name.) + Pyre infers the type of :class:`~libcst.Name`, :class:`~libcst.Attribute` and :class:`~libcst.Call` nodes. + The inter process communication to Pyre server is managed by :class:`~libcst.metadata.FullRepoManager`. + """ + METADATA_DEPENDENCIES = (PositionProvider,) is_cache_required = True