From 9bfc4faea92c121980cf4ca256f7e384d5a82894 Mon Sep 17 00:00:00 2001
From: Jennifer Taylor <jetaylor@fb.com>
Date: Wed, 11 Dec 2019 16:40:14 -0800
Subject: [PATCH] Add documentation for libcst.codemod and friends.

---
 docs/source/codemods.rst                   | 142 +++++++++++++++
 docs/source/codemods_tutorial.rst          | 197 +++++++++++++++++++++
 docs/source/index.rst                      |   2 +
 docs/source/scope_tutorial.ipynb           |   2 +
 docs/source/tutorial.ipynb                 |   6 +-
 libcst/codemod/_cli.py                     |  73 ++++++--
 libcst/codemod/_codemod.py                 |  33 +++-
 libcst/codemod/_command.py                 |  74 +++++---
 libcst/codemod/_context.py                 |  41 +++--
 libcst/codemod/_runner.py                  |  69 ++++++--
 libcst/codemod/_visitor.py                 |  37 +++-
 libcst/codemod/visitors/_add_imports.py    |  51 ++++++
 libcst/codemod/visitors/_gather_imports.py |  38 ++++
 libcst/helpers/module.py                   |   7 +-
 14 files changed, 688 insertions(+), 84 deletions(-)
 create mode 100644 docs/source/codemods.rst
 create mode 100644 docs/source/codemods_tutorial.rst

diff --git a/docs/source/codemods.rst b/docs/source/codemods.rst
new file mode 100644
index 00000000..c52533e8
--- /dev/null
+++ b/docs/source/codemods.rst
@@ -0,0 +1,142 @@
+========
+Codemods
+========
+
+LibCST defines a codemod as an automated refactor that can be applied to a codebase
+of arbitrary size. Codemods are provided as a framework for writing higher-order
+transforms that consist of other, simpler transforms. It includes provisions for
+quickly creating a command-line interface to execute a codemod.
+
+.. _libcst-codemod-base:
+
+------------
+Codemod Base
+------------
+
+All codemods derive from a common base, :class:`~libcst.codemod.Codemod`. This class
+includes a context, automatic metadata resolution and multi-pass transform support.
+Codemods are intended to be executed using the :func:`~libcst.codemod.transform_module`
+interface.
+
+.. autoclass:: libcst.codemod.Codemod
+.. autoclass:: libcst.codemod.CodemodContext
+
+As a convenience, LibCST-compatible visitors are provided which extend the feature-set
+of :class:`~libcst.codemod.Codemod` to LibCST visitors and transforms. Remember that
+:class:`~libcst.codemod.ContextAwareTransformer` is still a
+:class:`~libcst.codemod.Codemod`, so you should still execute it using
+:func:`~libcst.codemod.transform_module`.
+
+.. autoclass:: libcst.codemod.ContextAwareTransformer
+  :exclude-members: transform_module_impl
+.. autoclass:: libcst.codemod.ContextAwareVisitor
+
+It is often necessary to bail out of a codemod mid-operation when you realize that
+you do not want to operate on a module. This can be for any reason such as realizing
+the module includes some operation that you do not support. If you wish to skip a
+module, you can raise the :class:`~libcst.codemod.SkipFile` exception. For codemods
+executed using the :func:`~libcst.codemod.transform_module` interface, all warnings
+emitted up to the exception being thrown will be preserved in the result.
+
+.. autoclass:: libcst.codemod.SkipFile
+
+-------------------
+Execution Interface
+-------------------
+
+As documented in the Codemod Base section above, codemods are meant to be
+programmatically executed using :func:`~libcst.codemod.transform_module`. Executing
+in this manner handles all of the featureset of codemods, including metadata calculation
+and exception handling.
+
+.. autofunction:: libcst.codemod.transform_module
+.. autoclass:: libcst.codemod.TransformResult
+.. autoclass:: libcst.codemod.TransformSuccess
+.. autoclass:: libcst.codemod.TransformFailure
+.. autoclass:: libcst.codemod.TransformSkip
+.. autoclass:: libcst.codemod.SkipReason
+.. autoclass:: libcst.codemod.TransformExit
+
+--------------------
+Command-Line Support
+--------------------
+
+LibCST includes additional support to facilitate faster development of codemods which
+are to be run at the command-line. This is achieved through the
+:class:`~libcst.codemod.CodemodCommand` class and the ``codemod`` utility which lives
+inside ``libcst.tool``. The :class:`~libcst.codemod.CodemodCommand` class provides a
+codemod description and an interface to add arguments to the command-line. This is
+translated to a custom help message and command-line options that a user can provide
+when running a codemod at the command-line.
+
+For a brief overview of supported universal options, run the ``codemod`` utility like so::
+
+    python3 -m libcst.tool codemod --help
+
+The utility provides support for gathering up and parallelizing codemods across a
+series of files or directories, auto-formatting changed code according to a configured
+formatter, generating a unified diff of changes instead of applying them to files,
+taking code from stdin and codemodding it before returning to stdout, and printing
+progress and warnings to stderr during execution of a codemod.
+
+Help is auto-customized if a codemod class is provided, including any added options
+and the codemod description. For an example, run the ``codemod`` utility like so::
+
+    python3 -m libcst.tool codemod noop.NOOPCommand --help
+
+A second utility, ``list``, can list all available codemods given your configuration.
+Run it like so::
+
+    python3 -m libcst.tool list
+
+Finally, to set up a directory for codemodding using these tools, including additional
+directories where codemods can be found, use the ``initialize`` utility. To see help
+for how to use this, run the ``initialize`` utility like so::
+
+    python3 -m libcst.tool initialize --help
+
+The above tools operate against any codemod which subclasses from
+:class:`~libcst.codemod.CodemodCommand`. Remember that :class:`~libcst.codemod.CodemodCommand`
+is a subclass of :class:`~libcst.codemod.Codemod`, so all of the features documented
+in the :ref:`libcst-codemod-base` section are available in addition to command-line
+support. Any command-line enabled codemod can also be programmatically instantiated
+and invoked using the above-documented :func:`~libcst.codemod.transform_module`
+interface.
+
+.. autoclass:: libcst.codemod.CodemodCommand
+  :exclude-members: transform_module
+
+Additionally, a few convenience classes have been provided which take the boilerplate
+out of common types of codemods:
+
+.. autoclass:: libcst.codemod.VisitorBasedCodemodCommand
+.. autoclass:: libcst.codemod.MagicArgsCodemodCommand
+  :exclude-members: transform_module_impl
+
+--------------------
+Command-Line Toolkit
+--------------------
+
+Several helpers for constructing a command-line interface are provided. These are used
+in the ``codemod`` utility to provide LibCST's de-facto command-line interface but they
+are also available to be used directly in the case that circumstances demand a custom
+command-line tool.
+
+.. autofunction:: libcst.codemod.gather_files
+.. autofunction:: libcst.codemod.exec_transform_with_prettyprint
+.. autofunction:: libcst.codemod.parallel_exec_transform_with_prettyprint
+.. autoclass:: libcst.codemod.ParallelTransformResult
+.. autofunction:: libcst.codemod.diff_code
+
+---------------------
+Library of Transforms
+---------------------
+
+LibCST additionally includes a library of transforms to reduce the need for boilerplate
+inside codemods. As of now, the list includes the following helpers.
+
+.. autoclass:: libcst.codemod.visitors.GatherImportsVisitor
+  :exclude-members: visit_Import, visit_ImportFrom
+.. autoclass:: libcst.codemod.visitors.AddImportsVisitor
+  :exclude-members: CONTEXT_KEY, visit_Module, leave_ImportFrom, leave_Module
+.. autofunction:: libcst.helpers.module.insert_header_comments
diff --git a/docs/source/codemods_tutorial.rst b/docs/source/codemods_tutorial.rst
new file mode 100644
index 00000000..02e640ad
--- /dev/null
+++ b/docs/source/codemods_tutorial.rst
@@ -0,0 +1,197 @@
+=====================
+Working With Codemods
+=====================
+
+Codemods are an abstraction on top of LibCST for performing large-scale changes
+to an entire codebase. See :doc:`Codemods <codemods>` for the complete
+documentation.
+
+-------------------------------
+Setting up and Running Codemods
+-------------------------------
+
+Let's say you were interested in converting legacy ``.format()`` calls to shiny new
+Python 3.6 f-strings. LibCST ships with a command-line interface known as
+``libcst.tool``. This includes a few provisions for working with codemods at the
+command-line. It also includes a library of pre-defined codemods, one of which is
+a transform that can convert most ``.format()`` calls to f-strings. So, let's use this
+to give Python 3.6 f-strings a try.
+
+
+You might be lucky enough that the defaults for LibCST perfectly match your coding
+style, but chances are you want to customize LibCST to your repository. Initialize
+your repository by running the following command in the root of your repository and
+then edit the produced ``.libcst.codemod.yaml`` file::
+
+    python3 -m libcst.tool initialize .
+
+The file includes provisions for customizing any generated code marker, calling an
+external code formatter such as `black <https://pypi.org/project/black/>`_, blackisting
+patterns of files you never wish to touch and a list of modules that contain valid
+codemods that can be executed. If you want to write and run codemods specific to your
+repository or organization, you can add an in-repo module location to the list of
+modules and LibCST will discover codemods in all locations.
+
+Now that your repository is initialized, let's have a quick look at what's currently
+available for running. Run the following command from the root of your repository::
+
+    python3 -m libcst.tool list
+
+You'll see several codemods available to you, one of which is
+``convert_format_to_fstring.ConvertFormatStringCommand``. The description to the right
+of this codemod indicates that it converts ``.format()`` calls to f-strings, so let's
+give it a whirl! Execute the codemod from the root of your repository like so::
+
+    python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand .
+
+If you want to try it out on only one file or a specific subdirectory, you can replace
+the ``.`` in the above command with a relative directory, file, list of directories or
+list of files. While LibCST is walking through your repository and codemodding files
+you will see a progress indicator. If there's anything the codemod can't do or any
+unexpected syntax errors, you will also see them on your console as it progresses.
+
+If everything works out, you'll notice that your ``.format()`` calls have been
+converted to f-strings!
+
+-----------------
+Writing a Codemod
+-----------------
+
+Codemods use the same principles as the rest of LibCST. They take LibCST's core,
+metadata and matchers and package them up as a simple command-line interface. So,
+anything you can do with LibCST in isolation you can also do with a codemod.
+
+Let's say you need to clean up some legacy code which used magic values instead
+of constants. You've already got a constants module called ``utils.constants``
+and you want to assume that every reference to a raw string matching a particular
+constant should be converted to that constant. For the simplest version of this
+codemod, you'll need a command-line tool that takes as arguments the string to
+replace and the constant to replace it with. You'll also need to ensure that
+modified modules import the constant itself.
+
+So, you can write something similar to the following::
+
+    import argparse
+    from ast import literal_eval
+    from typing import Union
+
+    import libcst as cst
+    from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand
+    from libcst.codemod.visitors import AddImportsVisitor
+
+
+    class ConvertConstantCommand(VisitorBasedCodemodCommand):
+
+        # Add a description so that future codemodders can see what this does.
+        DESCRIPTION: str = "Converts raw strings to constant accesses."
+
+        @staticmethod
+        def add_args(arg_parser: argparse.ArgumentParser) -> None:
+            # Add command-line args that a codemodd user can specify for running this
+            # codemod.
+            arg_parser.add_argument(
+                "--string",
+                dest="string",
+                metavar="STRING",
+                help="String contents that we should look for.",
+                type=str,
+                required=True,
+            )
+            arg_parser.add_argument(
+                "--constant",
+                dest="constant",
+                metavar="CONSTANT",
+                help="Constant identifier we should replace strings with.",
+                type=str,
+                required=True,
+            )
+
+        def __init__(self, context: CodemodContext, string: str, constant: str) -> None:
+            # Initialize the base class with context, and save our args. Remember, the
+            # "dest" for each argument we added above must match a parameter name in
+            # this init.
+            super().__init__(context)
+            self.string = string
+            self.constant = constant
+
+        def leave_SimpleString(
+            self, original_node: cst.SimpleString, updated_node: cst.SimpleString
+        ) -> Union[cst.SimpleString, cst.Name]:
+            if literal_eval(updated_node.value) == self.string:
+                # Check to see if the string matches what we want to replace. If so,
+                # then we do the replacement. We also know at this point that we need
+                # to import the constant itself.
+                AddImportsVisitor.add_needed_import(
+                    self.context, "utils.constants", self.constant,
+                )
+                return cst.Name(self.constant)
+            # This isn't a string we're concerned with, so leave it unchanged.
+            return updated_node
+
+This codemod is pretty simple. It defines a command-line description, sets up to parse
+a few required command-line args, initializes its own member variables with the
+command-line args that were parsed for it by ``libcst.tool codemod`` and finally
+replaces any string which matches our string command-line argument with a constant.
+It also takes care of adding the import required for the constant to be defined properly.
+
+Cool! Let's look at the command-line help for this codemod. Let's assume you saved it
+as ``constant_folding.py`` inside ``libcst.codemod.commands``. You can get help for the
+codemod by running the following command::
+
+    python3 -m libcst.tool codemod constant_folding.ConvertConstantCommand --help
+
+Notice that along with the default arguments, the ``--string`` and ``--constant``
+arguments are present in the help, and the command-line description has been updated
+with the codemod's description string. You'' notice that the codemod also shows up
+on ``libcst.tool list``.
+
+----------------
+Testing Codemods
+----------------
+
+Instead of iterating on a codemod by running it repeatedly on a codebase and seeing
+what happens, we can write a series of unit tests that assert on desired
+transformations. Given the above constant folding codemod that we wrote, we can test
+it with some code similar to the following::
+
+    from libcst.codemod import CodemodTest
+    from libcst.codemod.commands.constant_folding import ConvertConstantCommand
+
+
+    class TestConvertConstantCommand(CodemodTest):
+
+        # The codemod that will be instantiated for us in assertCodemod.
+        TRANSFORM = ConvertConstantCommand
+
+        def test_noop(self) -> None:
+            before = """
+                foo = "bar"
+            """
+            after = """
+                foo = "bar"
+            """
+
+            # Verify that if we don't have a valid string match, we don't make
+            # any substitutions.
+            self.assertCodemod(before, after, string="baz", constant="BAZ")
+
+        def test_substitution(self) -> None:
+            before = """
+                foo = "bar"
+            """
+            after = """
+                from utils.constants import BAR
+
+                foo = BAR
+            """
+
+            # Verify that if we do have a valid string match, we make a substitution
+            # as well as import the constant.
+            self.assertCodemod(before, after, string="bar", constant="BAR")
+
+If we save this as ``test_constant_folding.py`` inside ``libcst.codemod.commands.tests``
+then we can execute the tests with the following line::
+
+    python3 -m unittest libcst.codemod.commands.tests.test_constant_folding
+
+That's all there is to it!
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 02850e35..e4645a4a 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -27,6 +27,7 @@ LibCST
    Metadata <metadata_tutorial>
    Scope Analysis <scope_tutorial>
    Matchers <matchers_tutorial>
+   Codemodding <codemods_tutorial>
    Best Practices <best_practices>
 
 
@@ -39,6 +40,7 @@ LibCST
    visitors
    metadata
    matchers
+   codemods
    experimental
 
 
diff --git a/docs/source/scope_tutorial.ipynb b/docs/source/scope_tutorial.ipynb
index 062752ae..e650859e 100644
--- a/docs/source/scope_tutorial.ipynb
+++ b/docs/source/scope_tutorial.ipynb
@@ -6,6 +6,8 @@
     "raw_mimetype": "text/restructuredtext"
    },
    "source": [
+    " .. _libcst-scope-tutorial:\n",
+    "\n",
     "==============\n",
     "Scope Analysis\n",
     "==============\n",
diff --git a/docs/source/tutorial.ipynb b/docs/source/tutorial.ipynb
index 25d7c15a..1fe57070 100644
--- a/docs/source/tutorial.ipynb
+++ b/docs/source/tutorial.ipynb
@@ -6,9 +6,9 @@
     "raw_mimetype": "text/restructuredtext"
    },
    "source": [
-    "========\n",
-    "Tutorial\n",
-    "========\n",
+    "====================\n",
+    "Parsing and Visiting\n",
+    "====================\n",
     "\n",
     "LibCST provides helpers to parse source code string as concrete syntax tree. In order to perform static analysis to identify patterns in the tree or modify the tree programmatically, we can use visitor pattern to traverse the tree. In this tutorial, we demonstrate a common three-step-workflow to build an automated refactoring (codemod) application:\n",
     "\n",
diff --git a/libcst/codemod/_cli.py b/libcst/codemod/_cli.py
index d188d0e8..bb535087 100644
--- a/libcst/codemod/_cli.py
+++ b/libcst/codemod/_cli.py
@@ -76,7 +76,9 @@ def gather_files(
 ) -> List[str]:
     """
     Given a list of files or directories (can be intermingled), return a list of
-    all python files that exist at those locations.
+    all python files that exist at those locations. If ``include_stubs`` is ``True``,
+    this will include ``.py`` and ``.pyi`` stub files. If it is ``False``, only
+    ``.py`` files will be included in the returned list.
     """
     ret: List[str] = []
     for fd in files_or_dirs:
@@ -94,6 +96,16 @@ def gather_files(
 def diff_code(
     oldcode: str, newcode: str, context: int, *, filename: Optional[str] = None
 ) -> str:
+    """
+    Given two strings representing a module before and after a codemod, produce
+    a unified diff of the changes with ``context`` lines of context. Optionally,
+    assign the ``filename`` to the change, and if it is not available, assume
+    that the change was performed on stdin/stdout. If no change is detected,
+    return an empty string instead of returning an empty unified diff. This is
+    comparable to revision control software which only shows differences for
+    files that have changed.
+    """
+
     if oldcode == newcode:
         return ""
 
@@ -123,8 +135,15 @@ def exec_transform_with_prettyprint(
     formatter_args: Sequence[str] = (),
 ) -> Optional[str]:
     """
-    Given an instantiated transform, and a code string, transform that code string
-    by executing the transform, and then print any generated warnings to the screen.
+    Given an instantiated codemod and a string representing a module, transform that
+    code by executing the transform, optionally invoking the formatter and finally
+    printing any generated warnings to stderr. If the code includes the generated
+    marker at any spot and ``include_generated`` is not set to ``True``, the code
+    will not be modified. If ``format_code`` is set to ``False`` or the instantiated
+    codemod does not modify the code, the code will not be formatted.
+
+    In all cases a module will be returned. Whether it is changed depends on the
+    input parameters as well as the codemod itself.
     """
 
     if not include_generated and generated_code_marker in code:
@@ -408,13 +427,23 @@ def _print_parallel_result(
 
 @dataclass(frozen=True)
 class ParallelTransformResult:
-    # Number of files that we successfully transformed
+    """
+    The result of running
+    :func:`~libcst.codemod.parallel_exec_transform_with_prettyprint` against
+    a series of files. This is a simple summary, with counts for number of
+    successfully codemodded files, number of files that we failed to codemod,
+    number of warnings generated when running the codemod across the files, and
+    the number of files that we skipped when running the codemod.
+    """
+
+    #: Number of files that we successfully transformed.
     successes: int
-    # Number of files that we failed to transform
+    #: Number of files that we failed to transform.
     failures: int
-    # Number of warnings generated when running transform across files
+    #: Number of warnings generated when running transform across files.
     warnings: int
-    # Number of files skipped because they were blacklisted or generated
+    #: Number of files skipped because they were blacklisted, generated
+    #: or the codemod requested to skip.
     skips: int
 
 
@@ -435,11 +464,31 @@ def parallel_exec_transform_with_prettyprint(  # noqa: C901
     blacklist_patterns: Sequence[str] = (),
 ) -> ParallelTransformResult:
     """
-    Given a list of files, and an instantiated transform we should apply to them,
-    fork and apply in parallel to all of the files. "jobs" controls the maximum
-    number of in-flight transforms, and needs to be at least 1. To make this API
-    simpler, we take an instantiated transform. This means we're implicitly relying
-    on fork behavior on *NIX systems, and this will not work on Windows.
+    Given a list of files and an instantiated codemod we should apply to them,
+    fork and apply the codemod in parallel to all of the files, including any
+    configured formatter. The ``jobs`` parameter controls the maximum number of
+    in-flight transforms, and needs to be at least 1. If not included, the number
+    of jobs will automatically be set to the number of CPU cores. If ``unified_diff``
+    is set to a number, changes to files will be printed to stdout with
+    ``unified_diff`` lines of context. If it is set to ``None`` or left out, files
+    themselves will be updated with changes and formatting.
+
+    A progress indicator as well as any generated warnings will be printed to stderr.
+    To supress the interactive progress indicator, set ``hide_progress`` to ``True``.
+    Files that include the generated code marker will be skipped unless the
+    ``include_generated`` parameter is set to ``True``. Similarly, files that match
+    a supplied blacklist of regex patterns will be skipped. Warnings for skipping
+    both blacklisted and generated files will be printed to stderr along with
+    warnings generated by the codemod unless ``hide_blacklisted`` and
+    ``hide_generated`` are set to ``True``. Files that were successfully codemodded
+    will not be printed to stderr unless ``show_successes`` is set to ``True``.
+
+    To make this API possible, we take an instantiated transform. This is due to
+    the fact that lambdas are not pickleable and pickling functions is undefined.
+    This means we're implicitly relying on fork behavior on UNIX-like systems, and
+    this function will not work on Windows systems. To create a command-line utility
+    that runs on Windows, please instead see
+    :func:`~libcst.codemod.exec_transform_with_prettyprint`.
     """
 
     # Ensure that we have no duplicates, otherwise we might get race conditions
diff --git a/libcst/codemod/_codemod.py b/libcst/codemod/_codemod.py
index 98732f52..1489f834 100644
--- a/libcst/codemod/_codemod.py
+++ b/libcst/codemod/_codemod.py
@@ -16,9 +16,16 @@ from libcst.codemod._context import CodemodContext
 class Codemod(MetadataDependent, ABC):
     """
     Abstract base class that all codemods must subclass from. Classes wishing
-    to perform arbitrary mutations on a tree should subclass from this. Classes
-    wishing to perform visitor-based mutation should instead subclass from
-    ContextAwareTransformer.
+    to perform arbitrary, non-visitor-based mutations on a tree should subclass
+    from this class directly. Classes wishing to perform visitor-based mutation
+    should instead subclass from :class:`~libcst.codemod.ContextAwareTransformer`.
+
+    Note that a :class:`~libcst.codemod.Codemod` is a subclass of
+    :class:`~libcst.MetadataDependent`, meaning that you can declare metadata
+    dependencies with the :attr:`~libcst.MetadataDependent.METADATA_DEPENDENCIES`
+    class property and while you are executing a transform you can call
+    :meth:`~libcst.MetadataDependent.get_metadata` to retrieve
+    the resolved metadata.
     """
 
     def __init__(self, context: CodemodContext) -> None:
@@ -27,20 +34,26 @@ class Codemod(MetadataDependent, ABC):
 
     def should_allow_multiple_passes(self) -> bool:
         """
-        Override this and return True to allow your transform to be called
+        Override this and return ``True`` to allow your transform to be called
         repeatedly until the tree doesn't change between passes. By default,
         this is off, and should suffice for most transforms.
         """
         return False
 
     def warn(self, warning: str) -> None:
+        """
+        Emit a warning that is displayed to the user who has invoked this codemod.
+        """
         self.context.warnings.append(warning)
 
     @property
     def module(self) -> Module:
         """
         Reference to the currently-traversed module. Note that this is only available
-        during a transform itself.
+        during the execution of a codemod. The module reference is particularly
+        handy if you want to use :meth:`libcst.Module.code_for_node` or
+        :attr:`libcst.Module.config_for_parsing` and don't wish to track a reference
+        to the top-level module manually.
         """
         module = self.context.module
         if module is None:
@@ -53,8 +66,9 @@ class Codemod(MetadataDependent, ABC):
     @abstractmethod
     def transform_module_impl(self, tree: Module) -> Module:
         """
-        Override this with your transform. You should take in the tree,
-        optionally mutate it and then return it.
+        Override this with your transform. You should take in the tree, optionally
+        mutate it and then return the mutated version. The module reference and all
+        calculated metadata are available for the lifetime of this function.
         """
         ...
 
@@ -74,8 +88,9 @@ class Codemod(MetadataDependent, ABC):
     def transform_module(self, tree: Module) -> Module:
         """
         Transform entrypoint which handles multi-pass logic and metadata calculation
-        for you. This is the method that you should call if you wish to
-        invoke a codemod directly.
+        for you. This is the method that you should call if you wish to invoke a
+        codemod directly. This is the method that is called by
+        :func:`~libcst.codemod.transform_module`.
         """
 
         if not self.should_allow_multiple_passes():
diff --git a/libcst/codemod/_command.py b/libcst/codemod/_command.py
index ebaaeff7..3188447c 100644
--- a/libcst/codemod/_command.py
+++ b/libcst/codemod/_command.py
@@ -21,31 +21,33 @@ _Codemod = TypeVar("_Codemod", bound=Codemod)
 
 class CodemodCommand(Codemod, ABC):
     """
-    A command is a type of transform that is normally instantiated and
-    run from the command-line. It behaves like any other Codemod in
-    that it can be used anywhere that any other Codemod can be used.
-    However, it can also be used with 'run_command' to make a transform
-    into a CLI tool. It also includes facilities for automatically running
-    certain common transforms after executing your transform_module_impl.
-    The following list of transforms are supported at this time:
+    A :class:`~libcst.codemod.Codemod` which can be invoked on the command-line
+    using the ``libcst.tool codemod`` utility. It behaves like any other codemod
+    in that it can be instantiated and run identically to a
+    :class:`~libcst.codemod.Codemod`. However, it provides support for providing
+    help text and command-line arguments to ``libcst.tool codemod`` as well as
+    facilities for automatically running certain common transforms after executing
+    your :meth:`~libcst.codemod.Codemod.transform_module_impl`.
 
-     - AddImportsVisitor (adds needed imports to a file).
+    The following list of transforms are automatically run at this time:
 
+     - :class:`~libcst.codemod.visitors.AddImportsVisitor` (adds needed imports to a file).
     """
 
-    # An overrideable description attribute so that codemods can provide
-    # a short summary of what they do.
+    #: An overrideable description attribute so that codemods can provide
+    #: a short summary of what they do. This description will show up in
+    #: command-line help as well as when listing available codemods.
     DESCRIPTION: str = "No description."
 
     @staticmethod
     def add_args(arg_parser: argparse.ArgumentParser) -> None:
         """
         Override this to add arguments to the CLI argument parser. These args
-        will show up when the user invokes the 'run_command' script with
-        --help. They will also be presented to your class's __init__ method.
-        So, if you define a command with an argument 'foo', you should also
+        will show up when the user invokes ``libcst.tool codemod`` with
+        ``--help``. They will also be presented to your class's ``__init__``
+        method. So, if you define a command with an argument 'foo', you should also
         have a corresponding 'foo' positional or keyword argument in your
-        class's __init__ method.
+        class's ``__init__`` method.
         """
 
         pass
@@ -54,6 +56,14 @@ class CodemodCommand(Codemod, ABC):
         inst = transform(self.context)
         return inst.transform_module(tree)
 
+    @abstractmethod
+    def transform_module_impl(self, tree: Module) -> Module:
+        """
+        Override this with your transform. You should take in the tree, optionally
+        mutate it and then return the mutated version. The module reference and all                                                                calculated metadata are available for the lifetime of this function.
+        """
+        ...
+
     def transform_module(self, tree: Module) -> Module:
         # Overrides (but then calls) Codemod's transform_module to provide
         # a spot where additional supported transforms can be attached and run.
@@ -81,8 +91,10 @@ class CodemodCommand(Codemod, ABC):
 class VisitorBasedCodemodCommand(ContextAwareTransformer, CodemodCommand, ABC):
     """
     A command that acts identically to a visitor-based transform, but also has
-    the support of add_args and running supported helper transforms after
-    execution. See CodemodCommand and ContextAwareTransformer for documentation.
+    the support of :meth:`~libcst.codemod.CodemodCommand.add_args` and running
+    supported helper transforms after execution. See
+    :class:`~libcst.codemod.CodemodCommand` and
+    :class:`~libcst.codemod.ContextAwareTransformer` for additional documentation.
     """
 
     pass
@@ -91,14 +103,16 @@ class VisitorBasedCodemodCommand(ContextAwareTransformer, CodemodCommand, ABC):
 class MagicArgsCodemodCommand(CodemodCommand, ABC):
     """
     A "magic" args command, which auto-magically looks up the transforms that
-    are yielded from get_transforms and instantiates them using values out
-    of the context. Visitors yielded in get_transforms must have constructor
-    arguments that match a key in the context scratch. The easiest way to
-    guarantee that is to use add_args to add a command arg that will be parsed
-    for each of the args. However, if you wish to chain transforms, adding
-    to the scratch in one transform will make the value available in the
-    constructor in subsequent transforms as well as the scratch for subsequent
-    transforms.
+    are yielded from :meth:`~libcst.codemod.MagicArgsCodemodCommand.get_transforms`
+    and instantiates them using values out of the context. Visitors yielded in
+    :meth:`~libcst.codemod.MagicArgsCodemodCommand.get_transforms` must have
+    constructor arguments that match a key in the context
+    :attr:`~libcst.codemod.CodemodContext.scratch`. The easiest way to
+    guarantee that is to use :meth:`~libcst.codemod.CodemodCommand.add_args`
+    to add a command arg that will be parsed for each of the args. However, if
+    you wish to chain transforms, adding to the scratch in one transform will make
+    the value available to the constructor in subsequent transforms as well as the
+    scratch for subsequent transforms.
     """
 
     def __init__(self, context: CodemodContext, **kwargs: Dict[str, object]) -> None:
@@ -107,6 +121,18 @@ class MagicArgsCodemodCommand(CodemodCommand, ABC):
 
     @abstractmethod
     def get_transforms(self) -> Generator[Type[Codemod], None, None]:
+        """
+        A generator which yields one or more subclasses of
+        :class:`~libcst.codemod.Codemod`. In the general case, you will usually
+        yield a series of classes, but it is possible to programmatically decide
+        which classes to yield depending on the contents of the context
+        :attr:`~libcst.codemod.CodemodContext.scratch`.
+
+        Note that you should yield classes, not instances of classes, as the
+        point of :class:`~libcst.codemod.MagicArgsCodemodCommand` is to
+        instantiate them for you with the contents of
+        :attr:`~libcst.codemod.CodemodContext.scratch`.
+        """
         ...
 
     def _instantiate(self, transform: Type[Codemod]) -> Codemod:
diff --git a/libcst/codemod/_context.py b/libcst/codemod/_context.py
index 7ff220c8..d061ad43 100644
--- a/libcst/codemod/_context.py
+++ b/libcst/codemod/_context.py
@@ -14,32 +14,43 @@ import libcst as cst
 class CodemodContext:
     """
     A context holding all information that is shared amongst all transforms
-    in a single codemod invocation. When chaining multiple transforms together,
-    the context holds the state that needs to be passed between transforms.
-    The context is responsible for keeping track of metadata wrappers and the
-    filename of the file that is being modified (if available).
+    and visitors in a single codemod invocation. When chaining multiple
+    transforms together, the context holds the state that needs to be passed
+    between transforms. The context is responsible for keeping track of
+    metadata wrappers and the filename of the file that is being modified
+    (if available).
     """
 
-    # List of warnings gathered up while running a codemod. Add to this list
-    # by calling the `warn()` method on a transform.
+    #: List of warnings gathered while running a codemod. Add to this list
+    #: by calling :meth:`~libcst.codemod.Codemod.warn` method from a class
+    #: that subclasses from :class:`~libcst.codemod.Codemod`,
+    #: :class:`~libcst.codemod.ContextAwareTransformer` or
+    #: :class:`~libcst.codemod.ContextAwareVisitor`.
     warnings: List[str] = field(default_factory=list)
-    # Scratch dictionary available for codemods which spread across multiple
-    # transforms. Codemods are free to add to this at will.
+
+    #: Scratch dictionary available for codemods which are spread across multiple
+    #: transforms. Codemods are free to add to this at will.
     scratch: Dict[str, Any] = field(default_factory=dict)
-    # The current filename if a codemod is being executed against a file that
-    # lives on disk.
+
+    #: The current filename if a codemod is being executed against a file that
+    #: lives on disk. Populated by
+    #: :func:`libcst.codemod.parallel_exec_transform_with_prettyprint` when
+    #: running codemods from the command line.
     filename: Optional[str] = None
-    # The current top level metadata wrapper for the module being modified.
-    # To access computed metadata, use `self.get_metadata` to retrieve values
-    # when inside an actively running transform.
+
+    #: The current top level metadata wrapper for the module being modified.
+    #: To access computed metadata when inside an actively running codemod, use
+    #: the :meth:`~libcst.MetadataDependent.get_metadata` method on
+    #: :class:`~libcst.codemod.Codemod`.
     wrapper: Optional[cst.MetadataWrapper] = None
 
     @property
     def module(self) -> Optional[cst.Module]:
         """
         The current top level module being modified. As a convenience, you can
-        use `self.module` to refer to this when inside an actively running
-        transform.
+        use the :attr:`~libcst.codemod.Codemod.module` property on
+        :class:`~libcst.codemod.Codemod` to refer to this when inside an actively
+        running codemod.
         """
 
         wrapper = self.wrapper
diff --git a/libcst/codemod/_runner.py b/libcst/codemod/_runner.py
index da4b7bbd..1ad2a0c5 100644
--- a/libcst/codemod/_runner.py
+++ b/libcst/codemod/_runner.py
@@ -18,66 +18,93 @@ from libcst import parse_module
 from libcst.codemod._codemod import Codemod
 
 
+# All datastructures defined in this class are pickleable so that they can be used
+# as a return value with the multiprocessing module.
+
+
 @dataclass(frozen=True)
 class TransformSuccess:
     """
+    A :class:`~libcst.codemod.TransformResult` used when the codemod was successful.
     Stores all the information we might need to display to the user upon success, as
     well as the transformed file contents.
-
-    This datastructure is pickleable so that it can be used as a return value with
-    the multiprocessing module.
     """
 
+    #: All warning messages that were generated during the codemod.
     warning_messages: Sequence[str]
+
+    #: The updated code, post-codemod.
     code: str
 
 
 @dataclass(frozen=True)
 class TransformFailure:
     """
+    A :class:`~libcst.codemod.TransformResult` used when the codemod failed.
     Stores all the information we might need to display to the user upon a failure.
-
-    This datastructure is pickleable so that it can be used as a return value with
-    the multiprocessing module.
     """
 
+    #: All warning messages that were generated before the codemod crashed.
     warning_messages: Sequence[str]
+
+    #: The exception that was raised during the codemod.
     error: Exception
+
+    #: The traceback string that was recorded at the time of exception.
     traceback_str: str
 
 
 @dataclass(frozen=True)
 class TransformExit:
     """
-    If the script is interrupted (e.g. KeyboardInterrupt) we don't have to
-    print anything special.
+    A :class:`~libcst.codemod.TransformResult` used when the codemod was interrupted
+    by the user (e.g. KeyboardInterrupt).
     """
 
+    #: An empty list of warnings, included so that all
+    #: :class:`~libcst.codemod.TransformResult` have a ``warning_messages`` attribute.
     warning_messages: Sequence[str] = ()
 
 
 class SkipReason(Enum):
+    """
+    An enumeration of all valid reasons for a codemod to skip.
+    """
+
+    #: The module was skipped because we detected that it was generated code, and
+    #: we were configured to skip generated files.
     GENERATED = "generated"
+
+    #: The module was skipped because we detected that it was blacklisted, and we
+    #: were configured to skip blacklisted files.
     BLACKLISTED = "blacklisted"
+
+    #: The module was skipped because the codemod requested us to skip using the
+    #: :class:`~libcst.codemod.SkipFile` exception.
     OTHER = "other"
 
 
 @dataclass(frozen=True)
 class TransformSkip:
     """
-    This file was skipped.
-
-    This could be because it's a generated file, or due to filename
-    blacklist, or because the transform raised SkipFile.
+    A :class:`~libcst.codemod.TransformResult` used when the codemod requested to
+    be skipped. This could be because it's a generated file, or due to filename
+    blacklist, or because the transform raised :class:`~libcst.codemod.SkipFile`.
     """
 
+    #: The reason that we skipped codemodding this module.
     skip_reason: SkipReason
+
+    #: The description populated from the :class:`~libcst.codemod.SkipFile` exception.
     skip_description: str
+
+    #: All warning messages that were generated before the codemod decided to skip.
     warning_messages: Sequence[str] = ()
 
 
 class SkipFile(Exception):
-    """Raise this exception to skip codemodding the current file.
+    """
+    Raise this exception to skip codemodding the current file.
 
     The exception message should be the reason for skipping.
     """
@@ -90,9 +117,19 @@ TransformResult = Union[
 
 def transform_module(transformer: Codemod, code: str) -> TransformResult:
     """
-    Given a module in a string and a Codemod to transform the module with,
-    execute the codemod on the code and return a TransformResult. This will
-    never raise an exception, instead will return a TransformFailure.
+    Given a module as represented by a string and a :class:`~libcst.codemod.Codemod`
+    that we wish to run, execute the codemod on the code and return a
+    :class:`~libcst.codemod.TransformResult`. This should never raise an exception.
+    On success, this returns a :class:`~libcst.codemod.TransformSuccess` containing
+    any generated warnings as well as the transformed code. If the codemod is
+    interrupted with a Ctrl+C, this returns a :class:`~libcst.codemod.TransformExit`.
+    If the codemod elected to skip by throwing a :class:`~libcst.codemod.SkipFile`
+    exception, this will return a :class:`~libcst.codemod.TransformSkip` containing
+    the reason for skipping as well as any warnings that were generated before
+    the codemod decided to skip. If the codemod throws an unexpected exception,
+    this will return a :class:`~libcst.codemod.TransformFailure` containing the
+    exception that occured as well as any warnings that were generated before the
+    codemod crashed.
     """
     try:
         input_tree = parse_module(code)
diff --git a/libcst/codemod/_visitor.py b/libcst/codemod/_visitor.py
index a61bce74..f66a0645 100644
--- a/libcst/codemod/_visitor.py
+++ b/libcst/codemod/_visitor.py
@@ -19,7 +19,10 @@ class ContextAwareTransformer(Codemod, MatcherDecoratableTransformer):
     A transformer which visits using LibCST. Allows visitor-based mutation of a tree.
     Classes wishing to do arbitrary non-visitor-based mutation on a tree should
     instead subclass from :class:`Codemod` and implement
-    :meth:`~Codemod.transform_module_impl`.
+    :meth:`~Codemod.transform_module_impl`. This is a subclass of
+    :class:`~libcst.matchers.MatcherDecoratableTransformer` so all features of matchers
+    as well as :class:`~libcst.CSTTransformer` are available to subclasses of this
+    class.
     """
 
     def __init__(self, context: CodemodContext) -> None:
@@ -32,10 +35,30 @@ class ContextAwareTransformer(Codemod, MatcherDecoratableTransformer):
 
 class ContextAwareVisitor(MatcherDecoratableVisitor, MetadataDependent):
     """
-    A collector which visits using LibCST. Allows visitor-based collecting of info
+    A visitor which visits using LibCST. Allows visitor-based collecting of info
     on a tree. All codemods which wish to implement an information collector should
-    subclass from this instead of directly from :class:`MatcherDecoratableVisitor`
-    or :class:`CSTVisitor` since this provides access to the current codemod context.
+    subclass from this instead of directly from
+    :class:`~libcst.matchers.MatcherDecoratableVisitor` or :class:`~libcst.CSTVisitor`
+    since this provides access to the current codemod context. As a result, this
+    class allows access to metadata which was calculated in a parent
+    :class:`~libcst.codemod.Codemod` through the
+    :meth:`~libcst.MetadataDependent.get_metadata` method.
+
+    Note that you cannot directly run a :class:`~libcst.codemod.ContextAwareVisitor`
+    using :func:`~libcst.codemod.transform_module` because visitors by definition
+    do not transform trees. However, you can instantiate a
+    :class:`~libcst.codemod.ContextAwareVisitor` inside a codemod and pass it to the
+    :class:`~libcst.CSTNode.visit` method on any node in order to run information
+    gathering with metadata and context support.
+
+    Remember that a :class:`~libcst.codemod.ContextAwareVisitor` is a subclass of
+    :class:`~libcst.MetadataDependent`, meaning that you still need to declare
+    your metadata dependencies with
+    :attr:`~libcst.MetadataDependent.METADATA_DEPENDENCIES` before you can retrieve
+    metadata using :meth:`~libcst.MetadataDependent.get_metadata`, even if the parent
+    codemod has listed its own metadata dependencies. Note also that the dependencies
+    listed on this class must be a strict subset of the dependencies listed in the
+    parent codemod.
     """
 
     def __init__(self, context: CodemodContext) -> None:
@@ -65,6 +88,12 @@ class ContextAwareVisitor(MatcherDecoratableVisitor, MetadataDependent):
                 dep: wrapper._metadata[dep] for dep in dependencies
             }
 
+    def warn(self, warning: str) -> None:
+        """
+        Emit a warning that is displayed to the user who has invoked this codemod.
+        """
+        self.context.warnings.append(warning)
+
     @property
     def module(self) -> cst.Module:
         """
diff --git a/libcst/codemod/visitors/_add_imports.py b/libcst/codemod/visitors/_add_imports.py
index 069efc24..93fe4777 100644
--- a/libcst/codemod/visitors/_add_imports.py
+++ b/libcst/codemod/visitors/_add_imports.py
@@ -14,6 +14,46 @@ from libcst.codemod.visitors._gather_imports import GatherImportsVisitor
 
 
 class AddImportsVisitor(ContextAwareTransformer):
+    """
+    Ensures that given imports exist in a module. Given a
+    :class:`~libcst.codemod.CodemodContext` and a sequence of tuples specifying
+    a module to import from as a string and optionally an object to import from
+    that module, ensures that that import exists. It will modify existing imports
+    as necessary if the module in question is already being imported from.
+
+    This is one of the transforms that is available automatically to you when
+    running a codemod. To use it in this manner, import
+    :class:`~libcst.codemod.visitors.AddImportsVisitor` and then call the static
+    :meth:`~libcst.codemod.visitors.AddImportsVisitor.add_needed_import` method,
+    giving it the current context (found as ``self.context`` for all subclasses of
+    :class:`~libcst.codemod.Codemod`), the module you wish to import from and
+    optionally an object you wish to import from that module.
+
+    For example::
+
+        AddImportsVisitor.add_needed_import(self.context, "typing", "Optional")
+
+    This will produce the following code in a module, assuming there was no
+    typing import already::
+
+        from typing import Optional
+
+    As another example::
+
+        AddImportsVisitor.add_needed_import(self.context, "typing")
+
+    This will produce the following code in a module, assuming there was no
+    import already::
+
+        import typing
+
+    Note that this is a subclass of :class:`~libcst.CSTTransformer` so it is
+    possible to instantiate it and pass it to a :class:`~libcst.Module`
+    :meth:`~libcst.CSTNode.visit` method. However, it is far easier to use
+    the automatic transform feature of :class:`~libcst.codemod.CodemodCommand`
+    and schedule an import to be added by calling
+    :meth:`~libcst.codemod.visitors.AddImportsVisitor.add_needed_import`
+    """
 
     CONTEXT_KEY = "AddImportsVisitor"
 
@@ -30,6 +70,17 @@ class AddImportsVisitor(ContextAwareTransformer):
     def add_needed_import(
         context: CodemodContext, module: str, obj: Optional[str] = None
     ) -> None:
+        """
+        Schedule an import to be added in a future invocation of this class by
+        updating the ``context`` to include the ``module`` and optionally ``obj``
+        to be imported. When subclassing from
+        :class:`~libcst.codemod.CodemodCommand`, this will be performed for you
+        after your transform finishes executing. If you are subclassing from a
+        :class:`~libcst.codemod.Codemod` instead, you will need to call the
+        :meth:`~libcst.CSTNode.visit` method on the module under modification
+        with an instance of this class after performing your transform.
+        """
+
         if module == "__future__" and obj is None:
             raise Exception("Cannot import __future__ directly!")
         imports = AddImportsVisitor._get_imports_from_context(context)
diff --git a/libcst/codemod/visitors/_gather_imports.py b/libcst/codemod/visitors/_gather_imports.py
index 6007cfa7..6892fc9e 100644
--- a/libcst/codemod/visitors/_gather_imports.py
+++ b/libcst/codemod/visitors/_gather_imports.py
@@ -12,6 +12,44 @@ from libcst.codemod._visitor import ContextAwareVisitor
 
 
 class GatherImportsVisitor(ContextAwareVisitor):
+    """
+    Gathers all imports in a module and stores them as attributes on the instance.
+    Intended to be instantiated and passed to a :class:`~libcst.Module`
+    :meth:`~libcst.CSTNode.visit` method in order to gather up information about
+    imports on a module. Note that this is not a substitute for scope analysis or
+    qualified name support. Please see :ref:`libcst-scope-tutorial` for a more
+    robust way of determining the qualified name and definition for an arbitrary
+    node.
+
+    After visiting a module the following attributes will be populated:
+
+     module_imports
+      A sequence of strings representing modules that were imported directly, such as
+      in the case of ``import typing``. Each module directly imported but not aliased
+      will be included here.
+     object_mapping
+      A mapping of strings to sequences of strings representing modules where we
+      imported objects from, such as in the case of ``from typing import Optional``.
+      Each from import that was not aliased will be included here, where the keys of
+      the mapping are the module we are importing from, and the value is a
+      sequence of objects we are importing from the module.
+     module_aliases
+      A mapping of strings representing modules that were imported and aliased,
+      such as in the case of ``import typing as t``. Each module imported this
+      way will be represented as a key in this mapping, and the value will be
+      the local alias of the module.
+     alias_mapping
+      A mapping of strings to sequences of tuples representing modules where we
+      imported objects from and aliased using ``as`` syntax, such as in the case
+      of ``from typing import Optional as opt``. Each from import that was aliased
+      will be included here, where the keys of the mapping are the module we are
+      importing from, and the value is a tuple representing the original object
+      name and the alias.
+     all_imports
+      A collection of all :class:`~libcst.Import` and :class:`~libcst.ImportFrom`
+      statements that were encountered in the module.
+    """
+
     def __init__(self, context: CodemodContext) -> None:
         super().__init__(context)
         # Track the available imports in this transform
diff --git a/libcst/helpers/module.py b/libcst/helpers/module.py
index 35e5b15a..044797f9 100644
--- a/libcst/helpers/module.py
+++ b/libcst/helpers/module.py
@@ -11,7 +11,12 @@ import libcst
 
 
 def insert_header_comments(node: libcst.Module, comments: List[str]) -> libcst.Module:
-    """Insert comments after last non-empty line in header."""
+    """
+    Insert comments after last non-empty line in header. Use this to insert one or more
+    comments after any copyright preamble in a :class:`~libcst.Module`. Each comment in
+    the list of ``comments`` must start with a ``#`` and will be placed on its own line
+    in the appropriate location.
+    """
     # Split the lines up into a contiguous comment-containing section and
     # the empty whitespace section that follows
     last_comment_index = -1