diff --git a/docs/source/index.rst b/docs/source/index.rst index 32dba04a..ea2f5625 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,6 +25,7 @@ LibCST Parsing and Visitors Metadata + Scope Analysis Matchers diff --git a/docs/source/metadata.rst b/docs/source/metadata.rst index 98587bce..d8b6b8e2 100644 --- a/docs/source/metadata.rst +++ b/docs/source/metadata.rst @@ -12,7 +12,7 @@ LibCST ships with a metadata interface that defines a standardized way to associate nodes in a CST with arbitrary metadata while maintaining the immutability of the tree. The metadata interface is designed to be declarative and type safe. Here's a quick example of using the metadata interface to get line and column -numbers of nodes through the :class:`~libcst.metadta.SyntacticPositionProvider`: +numbers of nodes through the :class:`~libcst.metadata.SyntacticPositionProvider`: .. _libcst-metadata-position-example: .. code-block:: python @@ -24,7 +24,7 @@ numbers of nodes through the :class:`~libcst.metadta.SyntacticPositionProvider`: pos = self.get_metadata(cst.SyntacticPositionProvider, node).start print(f"{node.value} found at line {pos.line}, column {pos.column}") - wrapper = cst.metadta.MetadataWrapper(cst.parse_module("x = 1")) + wrapper = cst.metadata.MetadataWrapper(cst.parse_module("x = 1")) result = wrapper.visit(NamePrinter()) # should print "x found at line 1, column 0" More examples of using the metadata interface can be found on the @@ -33,15 +33,15 @@ More examples of using the metadata interface can be found on the Accessing Metadata ------------------ -To work with metadata you need to wrap a module with a :class:`~libcst.metadta.MetadataWrapper`. -The wrapper provides a :func:`~libcst.metadta.MetadataWrapper.resolve` function and a -:func:`~libcst.metadta.MetadataWrapper.resolve_many` function to generate metadata. +To work with metadata you need to wrap a module with a :class:`~libcst.metadata.MetadataWrapper`. +The wrapper provides a :func:`~libcst.metadata.MetadataWrapper.resolve` function and a +:func:`~libcst.metadata.MetadataWrapper.resolve_many` function to generate metadata. -.. autoclass:: libcst.MetadataWrapper +.. autoclass:: libcst.metadata.MetadataWrapper If you're working with visitors, which extend :class:`~libcst.MetadataDependent`, metadata dependencies will be automatically computed when visited by a -:class:`~libcst.metadta.MetadataWrapper` and are accessible through +:class:`~libcst.metadata.MetadataWrapper` and are accessible through :func:`~libcst.MetadataDependent.get_metadata` .. autoclass:: libcst.MetadataDependent @@ -51,15 +51,15 @@ Providing Metadata Metadata is generated through provider classes that can be declared as a dependency by a subclass of :class:`~libcst.MetadataDependent`. These providers are then -resolved automatically using methods provided by :class:`~libcst.metadta.MetadataWrapper`. +resolved automatically using methods provided by :class:`~libcst.metadata.MetadataWrapper`. In most cases, you should extend :class:`~libcst.BatchableMetadataProvider` when writing a provider, unless you have a particular reason to not to use a batchable visitor. Only extend from :class:`~libcst.BaseMetadataProvider` if your provider does not use the visitor pattern for computing metadata for a tree. .. autoclass:: libcst.BaseMetadataProvider -.. autoclass:: libcst.BatchableMetadataProvider -.. autoclass:: libcst.VisitorMetadataProvider +.. autoclass:: libcst.metadata.BatchableMetadataProvider +.. autoclass:: libcst.metadata.VisitorMetadataProvider .. _libcst-metadata-position: diff --git a/docs/source/scope_tutorial.ipynb b/docs/source/scope_tutorial.ipynb new file mode 100644 index 00000000..5b011e37 --- /dev/null +++ b/docs/source/scope_tutorial.ipynb @@ -0,0 +1,215 @@ +{ + "cells": [ + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + "==============\n", + "Scope Analysis\n", + "==============\n", + "Scope analysis keeps track of assignments and accesses which could be useful for code automatic refactoring. If you're not familiar with Scope analysis, see :doc:`Metadata ` for more detail about Scope metadata. This tutorial demonstrates some use cases of Scope analysis. \n", + "Given source codes, Scope analysis parses all variable :class:`~libcst.metadata.Assignment` (or a :class:`~libcst.metadata.BuiltinAssignment` if it's a builtin) and :class:`~libcst.metadata.Access` to store in :class:`~libcst.metadata.Scope` containers.\n", + "\n", + "Given the following example source code contains a couple of unused imports (``f``, ``i``, ``m`` and ``n``) and undefined variable references (``func_undefined`` and ``var_undefined``). Scope analysis helps us identifying those unused imports and undefined variables to automatically provide warnings to developers to prevent bugs while they're developing.\n", + "With a parsed :class:`~libcst.Module`, we construct a :class:`~libcst.metadata.MetadataWrapper` object and it provides a :func:`~libcst.metadata.MetadataWrapper.resolve` function to resolve metadata given a metadata provider.\n", + ":class:`~libcst.metadata.ScopeProvider` is used here for analysing scope and there are three types of scopes (:class:`~libcst.metadata.GlobalScope`, :class:`~libcst.metadata.FunctionScope` and :class:`~libcst.metadata.ClassScope`) in this example.\n", + "\n", + ".. note::\n", + " The scope analysis only handles local variable name access and cannot handle simple string type annotation forward references. See :class:`~libcst.metadata.Access`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "nbsphinx": "hidden" + }, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append(\"../../\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import libcst as cst\n", + "\n", + "source = \"\"\"\\\n", + "import a, b, c as d, e as f # expect to keep: a, c as d\n", + "from g import h, i, j as k, l as m # expect to keep: h, j as k\n", + "from n import o # expect to be removed entirely\n", + "\n", + "a()\n", + "\n", + "def fun():\n", + " d()\n", + "\n", + "class Cls:\n", + " att = h.something\n", + " \n", + " def __new__(self) -> \"Cls\":\n", + " var = k.method()\n", + " func_undefined(var_undefined)\n", + "\"\"\"\n", + "wrapper = cst.metadata.MetadataWrapper(cst.parse_module(source))\n", + "scopes = set(wrapper.resolve(cst.metadata.ScopeProvider).values())\n", + "for scope in scopes:\n", + " print(scope)" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + "Warn on unused imports and undefined references\n", + "===============================================\n", + "To find all unused imports, we iterate through :attr:`~libcst.metadata.Scope.assignments` and an assignment is unused when its :attr:`~libcst.metadata.BaseAssignment.references` is empty. To find all undefined references, we iterate through :attr:`~libcst.metadata.Scope.accesses` (we focus on :class:`~libcst.Import`/:class:`~libcst.ImportFrom` assignments) and an access is undefined reference when its :attr:`~libcst.metadata.Access.referents` is empty. When reporting the warning to developer, we'll want to report the line number and column offset along with the suggestion to make it more clear. We can get position information from :class:`~libcst.metadata.SyntacticPositionProvider` and print the warnings as follows.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "from typing import Dict, Union, Set\n", + "\n", + "unused_imports: Dict[Union[cst.Import, cst.ImportFrom], Set[str]] = defaultdict(set)\n", + "undefined_references: Dict[cst.CSTNode, Set[str]] = defaultdict(set)\n", + "ranges = wrapper.resolve(cst.metadata.SyntacticPositionProvider)\n", + "for scope in scopes:\n", + " for assignment in scope.assignments:\n", + " node = assignment.node\n", + " if isinstance(assignment, cst.metadata.Assignment) and isinstance(\n", + " node, (cst.Import, cst.ImportFrom)\n", + " ):\n", + " if len(assignment.references) == 0:\n", + " unused_imports[node].add(assignment.name)\n", + " location = ranges[node].start\n", + " print(\n", + " f\"Warning on line {location.line:2d}, column {location.column:2d}: Imported name `{assignment.name}` is unused.\"\n", + " )\n", + "\n", + " for access in scope.accesses:\n", + " if len(access.referents) == 0:\n", + " node = access.node\n", + " location = ranges[node].start\n", + " print(\n", + " f\"Warning on line {location.line:2d}, column {location.column:2d}: Name reference `{node.value}` is not defined.\"\n", + " )\n" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + "Automatically Remove Unused Import\n", + "==================================\n", + "Unused import is a commmon code suggestion provided by lint tool like `flake8 F401 `_ ``imported but unused``.\n", + "Even though reporing unused import is already useful, with LibCST we can provide automatic fix to remove unused import. That can make the suggestion more actionable and save developer's time.\n", + "\n", + "An import statement may import multiple names, we want to remove those unused names from the import statement. If all the names in the import statement are not used, we remove the entire import.\n", + "To remove the unused name, we implement ``RemoveUnusedImportTransformer`` by subclassing :class:`~libcst.CSTTransformer`. We overwrite ``leave_Import`` and ``leave_ImportFrom`` to modify the import statements.\n", + "When we find the import node in lookup table, we iterate through all ``names`` and keep used names in ``names_to_keep``.\n", + "If ``names_to_keep`` is empty, all names are unused and we remove the entire import node.\n", + "Otherwise, we update the import node and just removing partial names." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class RemoveUnusedImportTransformer(cst.CSTTransformer):\n", + " def __init__(\n", + " self, unused_imports: Dict[Union[cst.Import, cst.ImportFrom], Set[str]]\n", + " ) -> None:\n", + " self.unused_imports = unused_imports\n", + "\n", + " def leave_import_alike(\n", + " self,\n", + " original_node: Union[cst.Import, cst.ImportFrom],\n", + " updated_node: Union[cst.Import, cst.ImportFrom],\n", + " ) -> Union[cst.Import, cst.ImportFrom, cst.RemovalSentinel]:\n", + " if original_node not in self.unused_imports:\n", + " return updated_node\n", + " names_to_keep = []\n", + " for name in updated_node.names:\n", + " asname = name.asname\n", + " if asname is not None:\n", + " name_value = asname.name.value\n", + " else:\n", + " name_value = name.name.value\n", + " if name_value not in self.unused_imports[original_node]:\n", + " names_to_keep.append(name.with_changes(comma=cst.MaybeSentinel.DEFAULT))\n", + " if len(names_to_keep) == 0:\n", + " return cst.RemoveFromParent()\n", + " else:\n", + " return updated_node.with_changes(names=names_to_keep)\n", + "\n", + " def leave_Import(\n", + " self, original_node: cst.Import, updated_node: cst.Import\n", + " ) -> cst.Import:\n", + " return self.leave_import_alike(original_node, updated_node)\n", + "\n", + " def leave_ImportFrom(\n", + " self, original_node: cst.ImportFrom, updated_node: cst.ImportFrom\n", + " ) -> cst.ImportFrom:\n", + " return self.leave_import_alike(original_node, updated_node)\n" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + "After the transform, we use ``.code`` to generate fixed code and all unused names are fixed as expected!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fixed_module = wrapper.module.visit(RemoveUnusedImportTransformer(unused_imports))\n", + "print(fixed_module.code)" + ] + } + ], + "metadata": { + "celltoolbar": "Raw Cell Format", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/libcst/metadata/scope_provider.py b/libcst/metadata/scope_provider.py index 10d7b09b..b9349b91 100644 --- a/libcst/metadata/scope_provider.py +++ b/libcst/metadata/scope_provider.py @@ -38,7 +38,7 @@ class Access: """ An Access records an access of an assignment. - .. warning:: + .. note:: This scope analysis only analyze access via a :class:`~libcst.Name` or a :class:`~libcst.Name` node embedded in other node like :class:`~libcst.Call` or :class:`~libcst.Attribute`. It doesn't support type anontation using :class:`~libcst.SimpleString` literal for forward @@ -102,8 +102,10 @@ class BaseAssignment(abc.ABC): @property def accesses(self) -> Tuple[Access, ...]: """Return all accesses of the assignment. - Deprecated: This will be removed soon. Please use - :attr:`~libcst.metadata.BaseAssignment.references` instead! + + .. warning:: + Deprecated: This will be removed soon. Please use + :attr:`~libcst.metadata.BaseAssignment.references` instead! """ # we don't want to publicly expose the mutable version of this warnings.warn( @@ -296,7 +298,7 @@ class Scope(abc.ABC): Use ``name in scope`` to check whether a name is viewable in the scope. Use ``scope[name]`` to retrieve all viewable assignments in the scope. - .. warning:: + .. note:: This scope analysis module only analyzes local variable names and it doesn't handle attribute names; for example, given a.b.c = 1, local variable name ``a`` is recorded as an assignment instead of ``c`` or ``a.b.c``. To analyze the assignment/access of