chore: remove macos-13 from ci (#1433)

remove macos-13 from ci

.github/workflows/build.yml

@@ -10,39 +10,35 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # macos-13 is an intel runner, macos-14 is apple silicon
-        os: [macos-13, macos-14, ubuntu-latest, windows-latest]
+        os:
+          [
+            macos-latest,
+            ubuntu-latest,
+            ubuntu-24.04-arm,
+            windows-latest,
+            windows-11-arm,
+          ]
     env:
       SCCACHE_VERSION: 0.2.13
-      CIBW_BEFORE_ALL_LINUX: "curl https://sh.rustup.rs -sSf | env -u CARGO_HOME sh -s -- --default-toolchain stable --profile minimal -y"
-      CIBW_BEFORE_BUILD_LINUX: "rm -rf native/target; ln -s /host/${{github.workspace}}/native/target native/target; [ -d /host/${{github.workspace}}/native/target ] ||  mkdir /host/${{github.workspace}}/native/target"
-      CIBW_ENVIRONMENT_LINUX: 'PATH="$PATH:$HOME/.cargo/bin" LIBCST_NO_LOCAL_SCHEME=$LIBCST_NO_LOCAL_SCHEME CARGO_HOME=/host/home/runner/.cargo'
-      CIBW_BEFORE_ALL_MACOS: "rustup target add aarch64-apple-darwin x86_64-apple-darwin"
-      CIBW_BEFORE_ALL_WINDOWS: "rustup target add x86_64-pc-windows-msvc i686-pc-windows-msvc"
-      CIBW_ENVIRONMENT: 'PATH="$PATH:$HOME/.cargo/bin" LIBCST_NO_LOCAL_SCHEME=$LIBCST_NO_LOCAL_SCHEME'
-      CIBW_SKIP: "cp27-* cp34-* cp35-* pp* *-win32 *-win_arm64 *-musllinux_*"
-      CIBW_ARCHS_LINUX: auto aarch64
-      CIBW_BUILD_VERBOSITY: 1
+      GITHUB_WORKSPACE: "${{github.workspace}}"
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v5
+          persist-credentials: false
+      - uses: actions/setup-python@v6
         with:
-          cache: pip
-          cache-dependency-path: "pyproject.toml"
           python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
       - name: Disable scmtools local scheme
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
         run: >-
           echo LIBCST_NO_LOCAL_SCHEME=1 >> $GITHUB_ENV
-      - name: Set up QEMU
-        if: runner.os == 'Linux'
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: all
+      - name: Enable building wheels for pre-release CPython versions
+        if: github.event_name != 'release'
+        run: echo CIBW_ENABLE=cpython-prerelease >> $GITHUB_ENV
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.17.0
+        uses: pypa/cibuildwheel@v3.2.1
       - uses: actions/upload-artifact@v4
         with:
           path: wheelhouse/*.whl

.github/workflows/ci.yml

@@ -6,6 +6,8 @@ on:
       - main
   pull_request:
 
+permissions: {}
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -13,30 +15,32 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13"
+          - "3.13t"
+          - "3.14"
+          - "3.14t"
     steps:
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.7.13"
+          python-version: ${{ matrix.python-version }}
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v5
-        with:
-          cache: pip
-          cache-dependency-path: "pyproject.toml"
-          python-version: ${{ matrix.python-version }}
-      - name: Install hatch
-        run: |
-          pip install -U hatch
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+          persist-credentials: false
+      - uses: dtolnay/rust-toolchain@stable
       - name: Build LibCST
-        run: hatch -vv env create
-      - name: Tests
-        run: hatch run test
-      - name: Pure Parser Tests
-        env:
-          LIBCST_PARSER_TYPE: pure
-        run: hatch run test
+        run: uv sync --locked --dev
+      - name: Native Parser Tests
+        run: uv run poe test
+      - name: Coverage
+        run: uv run coverage report
 
   # Run linters
   lint:
@@ -45,15 +49,14 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v5
+          persist-credentials: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
         with:
-          cache: pip
-          cache-dependency-path: "pyproject.toml"
+          version: "0.7.13"
           python-version: "3.10"
-      - name: Install hatch
-        run: pip install -U hatch
-      - run: hatch run lint
-      - run: hatch run fixtures
+      - run: uv run poe lint
+      - run: uv run poe fixtures
 
   # Run pyre typechecker
   typecheck:
@@ -62,43 +65,13 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v5
+          persist-credentials: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
         with:
-          cache: pip
-          cache-dependency-path: "pyproject.toml"
+          version: "0.7.13"
           python-version: "3.10"
-      - name: Install hatch
-        run: pip install -U hatch
-      - run: hatch run typecheck
-
-  # Upload test coverage
-  coverage:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v5
-        with:
-          cache: pip
-          cache-dependency-path: "pyproject.toml"
-          python-version: "3.10"
-      - name: Install hatch
-        run: pip install -U hatch
-      - name: Generate Coverage
-        run: |
-          hatch run coverage run setup.py test
-          hatch run coverage xml -i
-      - uses: codecov/codecov-action@v3
-        with:
-          files: coverage.xml
-          fail_ci_if_error: true
-          verbose: true
-      - name: Archive Coverage
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage
-          path: coverage.xml
+      - run: uv run poe typecheck
 
   # Build the docs
   docs:
@@ -107,15 +80,14 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v5
+          persist-credentials: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
         with:
-          cache: pip
-          cache-dependency-path: "pyproject.toml"
+          version: "0.7.13"
           python-version: "3.10"
-      - name: Install hatch
-        run: pip install -U hatch
-      - uses: ts-graphviz/setup-graphviz@v1
-      - run: hatch run docs
+      - uses: ts-graphviz/setup-graphviz@v2
+      - run: uv run --group docs poe docs
       - name: Archive Docs
         uses: actions/upload-artifact@v4
         with:
@@ -130,46 +102,41 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.13t"]
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt, clippy
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python-version }}
       - name: test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --manifest-path=native/Cargo.toml --release
+        run: cargo test --manifest-path=native/Cargo.toml --release
       - name: test without python
         if: matrix.os == 'ubuntu-latest'
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --manifest-path=native/Cargo.toml --release --no-default-features
+        run: cargo test --manifest-path=native/Cargo.toml --release --no-default-features
       - name: clippy
-        uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --manifest-path=native/Cargo.toml --all-features
+        run: cargo clippy --manifest-path=native/Cargo.toml --all-targets --all-features
       - name: compile-benchmarks
-        uses: actions-rs/cargo@v1
-        with:
-          command: bench
-          args: --manifest-path=native/Cargo.toml --no-run
+        run: cargo bench --manifest-path=native/Cargo.toml --no-run
 
   rustfmt:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt
       - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all --manifest-path=native/Cargo.toml -- --check
+      - name: format
+        run: cargo fmt --all --manifest-path=native/Cargo.toml -- --check
+  build:
+    # only trigger here for pull requests - regular pushes are handled in pypi_upload
+    if: ${{ github.event_name == 'pull_request' }}
+    uses: Instagram/LibCST/.github/workflows/build.yml@main

.github/workflows/pypi_upload.yml

@@ -16,44 +16,45 @@ jobs:
     name: Upload wheels to pypi
     runs-on: ubuntu-latest
     needs: build
+    permissions:
+      id-token: write
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          persist-credentials: false
       - name: Download binary wheels
         id: download
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           pattern: wheels-*
           path: wheelhouse
           merge-multiple: true
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
-          cache: pip
-          cache-dependency-path: "pyproject.toml"
           python-version: "3.10"
-      - name: Install hatch
-        run: pip install -U hatch
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.7.13"
+          enable-cache: false
       - name: Build a source tarball
         env:
           LIBCST_NO_LOCAL_SCHEME: 1
+          OUTDIR: ${{ steps.download.outputs.download-path }}
         run: >-
-          hatch run python -m
+          uv run python -m
           build
           --sdist
-          --outdir ${{ steps.download.outputs.download-path }}
+          --outdir "$OUTDIR"
       - name: Publish distribution 📦 to Test PyPI
         if: github.event_name == 'push'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          user: __token__
-          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           repository-url: https://test.pypi.org/legacy/
           packages-dir: ${{ steps.download.outputs.download-path }}
       - name: Publish distribution 📦 to PyPI
         if: github.event_name == 'release'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
           packages-dir: ${{ steps.download.outputs.download-path }}

.github/workflows/zizmor.yml

@@ -0,0 +1,35 @@
+name: GitHub Actions Security Analysis with zizmor 🌈
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["**"]
+
+jobs:
+  zizmor:
+    name: zizmor latest via PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Run zizmor 🌈
+        run: uvx zizmor --format sarif . > results.sarif 
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v4
+        with:
+          sarif_file: results.sarif
+          category: zizmor

.pyre_configuration

@@ -2,6 +2,9 @@
     "exclude": [
         ".*\/native\/.*"
     ],
+    "ignore_all_errors": [
+        ".venv"
+    ],
     "source_directories": [
         "."
     ],

CHANGELOG.md

@@ -1,3 +1,209 @@
+# 1.8.6 - 2025-11-03
+
+## What's Changed
+* Update pyproject.toml for 3.14t by @itamaro in https://github.com/Instagram/LibCST/pull/1417
+* Update PyO3 to 0.26 by @cjwatson in https://github.com/Instagram/LibCST/pull/1413
+* Make CodemodCommand's supported_transforms order deterministic by @frvnkliu in https://github.com/Instagram/LibCST/pull/1424
+
+## New Contributors
+* @cjwatson made their first contribution in https://github.com/Instagram/LibCST/pull/1413
+* @frvnkliu made their first contribution in https://github.com/Instagram/LibCST/pull/1424
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.8.5...v1.8.6
+
+# 1.8.5 - 2025-09-25
+
+## What's Changed
+* fixed: circular import error by @drinkmorewaterr in https://github.com/Instagram/LibCST/pull/1406
+
+
+# 1.8.4 - 2025-09-09
+
+## What's Changed
+* fixed: generate Attribute nodes when applying type annotations by @tungol in https://github.com/Instagram/LibCST/pull/1396
+* added: Support parsing of t-strings #1374  by @drinkmorewaterr in https://github.com/Instagram/LibCST/pull/1398
+* added: add support for PEP758 by @drinkmorewaterr in https://github.com/Instagram/LibCST/pull/1401
+
+## New Contributors
+* @tungol made their first contribution in https://github.com/Instagram/LibCST/pull/1396
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.8.2...v1.8.4
+
+# 1.8.3 - 2025-08-29
+## What's Changed
+* removed: remove entry points to pure parser by @drinkmorewaterr in https://github.com/Instagram/LibCST/pull/1375
+* fixed: fixes match statements to work with PositionProvider by @imsut in https://github.com/Instagram/LibCST/pull/1389
+
+
+## New Contributors
+* @hunterhogan made their first contribution in https://github.com/Instagram/LibCST/pull/1378
+* @thomas-serre-sonarsource made their first contribution in https://github.com/Instagram/LibCST/pull/1379
+* @imsut made their first contribution in https://github.com/Instagram/LibCST/pull/1389
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.8.2...v1.8.3
+
+# 1.8.2 - 2025-06-13
+
+# Fixed
+* fix(dependency): add back typing-extensions for 3.9 by @Lee-W in https://github.com/Instagram/LibCST/pull/1358
+
+## New Contributors
+* @Lee-W made their first contribution in https://github.com/Instagram/LibCST/pull/1358
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.8.1...v1.8.2
+
+# 1.8.1 - 2025-06-10
+
+## Added
+* add helper to convert nodes to matchers by @zsol in https://github.com/Instagram/LibCST/pull/1351
+
+## Updated
+* Avoid raising bare Exception by @zaicruvoir1rominet in https://github.com/Instagram/LibCST/pull/1168
+* Upgrade PyYAML-ft version and use new module name by @lysnikolaou in https://github.com/Instagram/LibCST/pull/1353
+
+## New Contributors
+* @lysnikolaou made their first contribution in https://github.com/Instagram/LibCST/pull/1353
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.8.0...v1.8.1
+
+# 1.8.0 - 2025-05-27
+
+## Added
+* Allow configuring empty formatter lists in codemod CLI by @ngoldbaum in https://github.com/Instagram/LibCST/pull/1319
+* Publish several new binary wheels
+  * macos intel by @hadialqattan in https://github.com/Instagram/LibCST/pull/1316
+  * windows arm64 by @zsol in https://github.com/Instagram/LibCST/pull/1304
+  * 3.13 CPython free-threaded by @zsol in https://github.com/Instagram/LibCST/pull/1333
+  * (only on [test.pypi.org](https://test.pypi.org/project/libcst/#history)) 3.14 and 3.14 CPython free-threaded by @amyreese and @zsol in https://github.com/Instagram/LibCST/pull/1345 and https://github.com/Instagram/LibCST/pull/1331
+* Enable support for free-threaded CPython by @zsol in https://github.com/Instagram/LibCST/pull/1295 and https://github.com/Instagram/LibCST/pull/1335
+
+## Updated
+* update pyo3 to 0.25 by @ngoldbaum in https://github.com/Instagram/LibCST/pull/1324
+* Replace multiprocessing with ProcessPoolExecutor by @zsol in https://github.com/Instagram/LibCST/pull/1294
+* Support pipe syntax for Union types in codegen by @zsol in https://github.com/Instagram/LibCST/pull/1336
+
+## New Contributors
+* @hadialqattan made their first contribution in https://github.com/Instagram/LibCST/pull/1316
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.7.0...v1.8.0
+
+# 1.7.0 - 2025-03-13
+
+## Added
+* add free-threaded CI by @ngoldbaum in https://github.com/Instagram/LibCST/pull/1312
+
+## Updated
+* Remove dependency on `chic` and upgrade `annotate-snippets` by @zanieb in https://github.com/Instagram/LibCST/pull/1293
+* Update for Pyo3 0.23 by @ngoldbaum in https://github.com/Instagram/LibCST/pull/1289
+* Bump PyO3 to 0.23.5 by @mgorny in https://github.com/Instagram/LibCST/pull/1311
+
+## New Contributors
+* @zanieb made their first contribution in https://github.com/Instagram/LibCST/pull/1293
+* @ngoldbaum made their first contribution in https://github.com/Instagram/LibCST/pull/1289
+* @mgorny made their first contribution in https://github.com/Instagram/LibCST/pull/1311
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.6.0...v1.7.0
+
+# 1.6.0 - 2025-01-09
+
+## Fixed
+
+* rename: store state in scratch by @zsol in https://github.com/Instagram/LibCST/pull/1250
+* rename: handle imports via a parent module by @zsol in https://github.com/Instagram/LibCST/pull/1251
+* rename: Fix imports with aliases by @zsol in https://github.com/Instagram/LibCST/pull/1252
+* rename: don't leave trailing commas by @zsol in https://github.com/Instagram/LibCST/pull/1254
+* rename: don't eat commas unnecessarily by @zsol in https://github.com/Instagram/LibCST/pull/1256
+* rename: fix renaming toplevel names by @zsol in https://github.com/Instagram/LibCST/pull/1260
+* bump 3.12 to 3.13 in readme by @khameeteman in https://github.com/Instagram/LibCST/pull/1228
+
+## Added
+
+* Add codemod to convert `typing.Union` to `|` by @yangdanny97 in https://github.com/Instagram/LibCST/pull/1270
+* Add codemod to fix variadic callable annotations by @yangdanny97 in https://github.com/Instagram/LibCST/pull/1269
+* Add codemod to rename typing aliases of builtins by @yangdanny97 in https://github.com/Instagram/LibCST/pull/1267
+* Add typing classifier to pyproject.toml and badge to README by @yangdanny97 in https://github.com/Instagram/LibCST/pull/1272
+* Expose TypeAlias and TypeVar related structs in rust library by @Crozzers in https://github.com/Instagram/LibCST/pull/1274
+
+## Updated
+* Upgrade pyo3 to 0.22 by @jelmer in https://github.com/Instagram/LibCST/pull/1180
+
+## New Contributors
+* @yangdanny97 made their first contribution in https://github.com/Instagram/LibCST/pull/1270
+* @Crozzers made their first contribution in https://github.com/Instagram/LibCST/pull/1274
+* @jelmer made their first contribution in https://github.com/Instagram/LibCST/pull/1180
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.5.1...v1.6.0
+
+# 1.5.1 - 2024-11-18
+
+## Added
+
+* build wheels for musllinux by @MrMino in https://github.com/Instagram/LibCST/pull/1243
+
+## New Contributors
+* @MrMino made their first contribution in https://github.com/Instagram/LibCST/pull/1243
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.5.0...v1.5.1
+
+# 1.5.0 - 2024-10-10
+
+## Added
+* FullyQualifiedNameProvider: Optionally consider pyproject.toml files when determining a file's module name and package by @camillol in https://github.com/Instagram/LibCST/pull/1148
+* Add validation for If node by @kiri11 in https://github.com/Instagram/LibCST/pull/1177
+* include python 3.13 in build by @khameeteman in https://github.com/Instagram/LibCST/pull/1203
+
+## Fixed
+* fix various Match statement visitation errors by @zsol in https://github.com/Instagram/LibCST/pull/1161
+* Mention codemod -x flag in docs by @kiri11 in https://github.com/Instagram/LibCST/pull/1169
+* Clear warnings for each file in codemod cli by @kiri11 in https://github.com/Instagram/LibCST/pull/1184
+* Typo fix in codemods_tutorial.rst (trivial) by @wimglenn in https://github.com/Instagram/LibCST/pull/1208
+* fix certain matchers breaking under multiprocessing by initializing them late by @kiri11 in https://github.com/Instagram/LibCST/pull/1204
+
+## Updated
+* make libcst_native::tokenizer public by @zsol in https://github.com/Instagram/LibCST/pull/1182
+* Use `license` instead of `license-file` by @michel-slm in https://github.com/Instagram/LibCST/pull/1189
+* Drop codecov from CI and readme by @amyreese in https://github.com/Instagram/LibCST/pull/1192
+
+
+## New Contributors
+* @kiri11 made their first contribution in https://github.com/Instagram/LibCST/pull/1169
+* @grievejia made their first contribution in https://github.com/Instagram/LibCST/pull/1174
+* @michel-slm made their first contribution in https://github.com/Instagram/LibCST/pull/1189
+* @wimglenn made their first contribution in https://github.com/Instagram/LibCST/pull/1208
+* @khameeteman made their first contribution in https://github.com/Instagram/LibCST/pull/1203
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.4.0...v1.5.0
+
+# 1.4.0 - 2024-05-22
+
+## Fixed
+* Fix Literal parse error in RemoveImportsVisitor by @camillol in https://github.com/Instagram/LibCST/pull/1130
+* Don't reset context.scratch between files by @zsol in https://github.com/Instagram/LibCST/pull/1151
+* Various documentation fixes
+  * Typo fix FullRepoManager by @kit1980 in https://github.com/Instagram/LibCST/pull/1138
+  * ✏️ Fix tiny typo in `docs/source/metadata.rst` by @tiangolo in https://github.com/Instagram/LibCST/pull/1134
+  * ✏️ Fix typo in `docs/source/scope_tutorial.ipynb` by @tiangolo in https://github.com/Instagram/LibCST/pull/1135
+  * Update CONTRIBUTING.md by @zaicruvoir1rominet in https://github.com/Instagram/LibCST/pull/1142
+
+## Added
+
+* Add helper functions for common ways of filtering nodes by @zaicruvoir1rominet in https://github.com/Instagram/LibCST/pull/1137
+* Dump CST to .dot (graphviz) files by @zaicruvoir1rominet in https://github.com/Instagram/LibCST/pull/1147
+* Implement PEP-696 by @thereversiblewheel in https://github.com/Instagram/LibCST/pull/1141
+
+## New Contributors
+* @tiangolo made their first contribution in https://github.com/Instagram/LibCST/pull/1134
+* @camillol made their first contribution in https://github.com/Instagram/LibCST/pull/1130
+* @zaicruvoir1rominet made their first contribution in https://github.com/Instagram/LibCST/pull/1142
+* @thereversiblewheel made their first contribution in https://github.com/Instagram/LibCST/pull/1141
+
+**Full Changelog**: https://github.com/Instagram/LibCST/compare/v1.3.1...v1.4.0
+
+# 1.3.1 - 2024-04-03
+
+## Fixed
+* ImportError due to missing `mypy_extensions` dependency by @zsol in https://github.com/Instagram/LibCST/pull/1128
+
 # 1.3.0 - 2024-04-03
 
 ## Updated

CONTRIBUTING.md

@@ -9,12 +9,32 @@ pull requests.
 ## Pull Requests
 We actively welcome your pull requests.
 
-1. Fork the repo and create your branch from `main`.
-2. If you've added code that should be tested, add tests.
-3. If you've changed APIs, update the documentation.
-4. Ensure the test suite passes by `python -m unittest`.
-5. Make sure your code lints.
-6. If you haven't already, complete the Contributor License Agreement ("CLA").
+### Setup Your Environment
+
+1. Install a [Rust toolchain](https://rustup.rs) and [uv](https://docs.astral.sh/uv/)
+2. Fork the repo on your side
+3. Clone the repo
+   > git clone [your fork.git] libcst  
+   > cd libcst
+4. Sync with the main libcst version package
+   > git fetch --tags https://github.com/instagram/libcst
+5. Setup the env
+   > uv sync
+
+You are now ready to create your own branch from main, and contribute.
+Please provide tests (using unittest), and update the documentation (both docstrings
+and sphinx doc), if applicable.
+
+### Before Submitting Your Pull Request
+
+1. Format your code
+   > uv run poe format
+2. Run the type checker
+   > uv run poe typecheck
+3. Test your changes
+   > uv run poe test
+4. Check linters
+   > uv run poe lint
 
 ## Contributor License Agreement ("CLA")
 In order to accept your pull request, we need you to submit a CLA. You only need

MAINTAINERS.md

@@ -0,0 +1,12 @@
+# How to make a new release
+
+1. Add a new entry to `CHANGELOG.md` (I normally use the [new release page](https://github.com/Instagram/LibCST/releases/new) to generate a changelog, then manually group)
+    1. Follow the existing format: `Fixed`, `Added`, `Updated`, `Deprecated`, `Removed`, `New Contributors` sections, and the full changelog link at the bottom.
+    1. Mention only user-visible changes - improvements to CI, tests, or development workflow aren't noteworthy enough
+    1. Version bumps are generally not worth mentioning with some notable exceptions (like pyo3)
+    1. Group related PRs into one bullet point if it makes sense
+2. manually bump versions in `Cargo.toml` files in the repo
+3. run `cargo update -p libcst`
+4. make a new PR with the above changes, get it reviewed and landed
+5. make a new release on Github, create a new tag on publish, and copy the contents of the changelog entry in there
+6. after publishing, check out the repo at the new tag, and run `cd native; cargo +nightly publish -Z package-workspace -p libcst_derive -p libcst`

README.rst

@@ -4,7 +4,7 @@
 
 A Concrete Syntax Tree (CST) parser and serializer library for Python
 
-|support-ukraine| |readthedocs-badge| |ci-badge| |codecov-badge| |pypi-badge| |pypi-download| |notebook-badge|
+|support-ukraine| |readthedocs-badge| |ci-badge| |pypi-badge| |pypi-download| |notebook-badge| |types-badge|
 
 .. |support-ukraine| image:: https://img.shields.io/badge/Support-Ukraine-FFD500?style=flat&labelColor=005BBB
    :alt: Support Ukraine - Help Provide Humanitarian Aid to Ukraine.
@@ -18,10 +18,6 @@ A Concrete Syntax Tree (CST) parser and serializer library for Python
    :target: https://github.com/Instagram/LibCST/actions/workflows/build.yml?query=branch%3Amain
    :alt: Github Actions
 
-.. |codecov-badge| image:: https://codecov.io/gh/Instagram/LibCST/branch/main/graph/badge.svg
-   :target: https://codecov.io/gh/Instagram/LibCST/branch/main
-   :alt: CodeCov
-
 .. |pypi-badge| image:: https://img.shields.io/pypi/v/libcst.svg
    :target: https://pypi.org/project/libcst
    :alt: PYPI
@@ -35,9 +31,13 @@ A Concrete Syntax Tree (CST) parser and serializer library for Python
    :target: https://mybinder.org/v2/gh/Instagram/LibCST/main?filepath=docs%2Fsource%2Ftutorial.ipynb
    :alt: Notebook
 
+.. |types-badge| image:: https://img.shields.io/pypi/types/libcst
+   :target: https://pypi.org/project/libcst
+   :alt: PYPI - Types
+
 .. intro-start
 
-LibCST parses Python 3.0 -> 3.12 source code as a CST tree that keeps
+LibCST parses Python 3.0 -> 3.14 source code as a CST tree that keeps
 all formatting details (comments, whitespaces, parentheses, etc). It's useful for
 building automated refactoring (codemod) applications and linters.
 
@@ -148,49 +148,7 @@ Further Reading
 Development
 -----------
 
-You'll need a recent `Rust toolchain <https://rustup.rs>`_ for developing.
-
-We recommend using `hatch <https://hatch.pypa.io/>` for running tests, linters,
-etc.
-
-Then, start by setting up and building the project:
-
-.. code-block:: shell
-
-    git clone git@github.com:Instagram/LibCST.git libcst
-    cd libcst
-    hatch env create
-
-To run the project's test suite, you can:
-
-.. code-block:: shell
-
-    hatch run test
-
-You can also run individual tests by using unittest and specifying a module like
-this:
-
-.. code-block:: shell
-
-    hatch run python -m unittest libcst.tests.test_batched_visitor
-
-See the `unittest documentation <https://docs.python.org/3/library/unittest.html>`_
-for more examples of how to run tests.
-
-We have multiple linters, including copyright checks and
-`slotscheck <https://slotscheck.rtfd.io>`_ to check the correctness of class
-``__slots__``. To run all of the linters:
-
-.. code-block:: shell
-
-    hatch run lint
-
-We use `ufmt <https://ufmt.omnilib.dev/en/stable/>`_ to format code. To format
-changes to be conformant, run the following in the root:
-
-.. code-block:: shell
-
-    hatch run format
+See `CONTRIBUTING.md <CONTRIBUTING.md>`_ for more details.
 
 Building
 ~~~~~~~~
@@ -208,11 +166,11 @@ directory:
 
     cargo build
 
-To rebuild the ``libcst.native`` module, from the repo root:
+The ``libcst.native`` module should be rebuilt automatically, but to force it:
 
 .. code-block:: shell
 
-    hatch env prune && hatch env create
+    uv sync --reinstall-package libcst
 
 Type Checking
 ~~~~~~~~~~~~~
@@ -223,7 +181,7 @@ To verify types for the library, do the following in the root:
 
 .. code-block:: shell
 
-    hatch run typecheck
+    uv run poe typecheck
 
 Generating Documents
 ~~~~~~~~~~~~~~~~~~~~
@@ -232,7 +190,7 @@ To generate documents, do the following in the root:
 
 .. code-block:: shell
 
-    hatch run docs
+    uv run --group docs poe docs
 
 Future
 ======

codecov.yml

@@ -1,4 +0,0 @@
-coverage:
-  status:
-    project: no
-    patch: yes

docs/source/codemods_tutorial.rst

@@ -26,7 +26,7 @@ then edit the produced ``.libcst.codemod.yaml`` file::
     python3 -m libcst.tool initialize .
 
 The file includes provisions for customizing any generated code marker, calling an
-external code formatter such as `black <https://pypi.org/project/black/>`_, blackisting
+external code formatter such as `black <https://pypi.org/project/black/>`_, blacklisting
 patterns of files you never wish to touch and a list of modules that contain valid
 codemods that can be executed. If you want to write and run codemods specific to your
 repository or organization, you can add an in-repo module location to the list of
@@ -135,16 +135,18 @@ replaces any string which matches our string command-line argument with a consta
 It also takes care of adding the import required for the constant to be defined properly.
 
 Cool! Let's look at the command-line help for this codemod. Let's assume you saved it
-as ``constant_folding.py`` inside ``libcst.codemod.commands``. You can get help for the
+as ``constant_folding.py``. You can get help for the
 codemod by running the following command::
 
-    python3 -m libcst.tool codemod constant_folding.ConvertConstantCommand --help
+    python3 -m libcst.tool codemod -x constant_folding.ConvertConstantCommand --help
 
 Notice that along with the default arguments, the ``--string`` and ``--constant``
 arguments are present in the help, and the command-line description has been updated
 with the codemod's description string. You'll notice that the codemod also shows up
 on ``libcst.tool list``.
 
+And ``-x`` flag allows to load any module as a codemod in addition to the standard ones.
+
 ----------------
 Testing Codemods
 ----------------

docs/source/conf.py

@@ -196,6 +196,7 @@ intersphinx_mapping = {"python": ("https://docs.python.org/3", None)}
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
 
+
 # -- autodoc customization
 def strip_class_signature(app, what, name, obj, options, signature, return_annotation):
     if what == "class":
@@ -218,7 +219,7 @@ def setup(app):
 
 
 nbsphinx_prolog = r"""
-{% set docname = 'docs/source/' + env.doc2path(env.docname, base=None) %}
+{% set docname = 'docs/source/' + env.doc2path(env.docname, base=None)|string%}
 
 .. only:: html
 

docs/source/helpers.rst

@@ -32,3 +32,18 @@ Functions that assist in traversing an existing LibCST tree.
 .. autofunction:: libcst.helpers.get_full_name_for_node
 .. autofunction:: libcst.helpers.get_full_name_for_node_or_raise
 .. autofunction:: libcst.helpers.ensure_type
+
+Node fields filtering Helpers
+-----------------------------
+
+Function that assist when handling CST nodes' fields.
+
+.. autofunction:: libcst.helpers.filter_node_fields
+
+And lower level functions:
+
+.. autofunction:: libcst.helpers.get_node_fields
+.. autofunction:: libcst.helpers.is_whitespace_node_field
+.. autofunction:: libcst.helpers.is_syntax_node_field
+.. autofunction:: libcst.helpers.is_default_node_field
+.. autofunction:: libcst.helpers.get_field_default_value

docs/source/metadata.rst

@@ -94,7 +94,7 @@ declaring one of :class:`~libcst.metadata.PositionProvider` or
 most cases, :class:`~libcst.metadata.PositionProvider` is what you probably
 want.
 
-Node positions are is represented with :class:`~libcst.metadata.CodeRange`
+Node positions are represented with :class:`~libcst.metadata.CodeRange`
 objects. See :ref:`the above example<libcst-metadata-position-example>`.
 
 .. autoclass:: libcst.metadata.PositionProvider
@@ -134,7 +134,7 @@ New scopes are created for classes, functions, and comprehensions. Other block
 constructs like conditional statements, loops, and try…except don't create their
 own scope.
 
-There are five different type of scope in Python:
+There are five different types of scopes in Python:
 :class:`~libcst.metadata.BuiltinScope`,
 :class:`~libcst.metadata.GlobalScope`,
 :class:`~libcst.metadata.ClassScope`,
@@ -243,7 +243,7 @@ In Python, type checkers like `Mypy <https://github.com/python/mypy>`_ or
 and infer types for expressions.
 :class:`~libcst.metadata.TypeInferenceProvider` is provided by `Pyre Query API <https://pyre-check.org/docs/querying-pyre.html>`__
 which requires `setup watchman <https://pyre-check.org/docs/getting-started/>`_ for incremental typechecking.
-:class:`~libcst.metadata.FullRepoManger` is built for manage the inter process communication to Pyre.
+:class:`~libcst.metadata.FullRepoManager` is built for manage the inter process communication to Pyre.
 
 .. autoclass:: libcst.metadata.TypeInferenceProvider
    :no-undoc-members:

docs/source/scope_tutorial.ipynb

@@ -90,7 +90,7 @@
    "source": [
     "Warn on unused imports and undefined references\n",
     "===============================================\n",
-    "To find all unused imports, we iterate through :attr:`~libcst.metadata.Scope.assignments` and an assignment is unused when its :attr:`~libcst.metadata.BaseAssignment.references` is empty. To find all undefined references, we iterate through :attr:`~libcst.metadata.Scope.accesses` (we focus on :class:`~libcst.Import`/:class:`~libcst.ImportFrom` assignments) and an access is undefined reference when its :attr:`~libcst.metadata.Access.referents` is empty. When reporting the warning to developer, we'll want to report the line number and column offset along with the suggestion to make it more clear. We can get position information from :class:`~libcst.metadata.PositionProvider` and print the warnings as follows.\n"
+    "To find all unused imports, we iterate through :attr:`~libcst.metadata.Scope.assignments` and an assignment is unused when its :attr:`~libcst.metadata.BaseAssignment.references` is empty. To find all undefined references, we iterate through :attr:`~libcst.metadata.Scope.accesses` (we focus on :class:`~libcst.Import`/:class:`~libcst.ImportFrom` assignments) and an access is undefined reference when its :attr:`~libcst.metadata.Access.referents` is empty. When reporting the warning to the developer, we'll want to report the line number and column offset along with the suggestion to make it more clear. We can get position information from :class:`~libcst.metadata.PositionProvider` and print the warnings as follows.\n"
    ]
   },
   {
@@ -136,13 +136,13 @@
     "Automatically Remove Unused Import\n",
     "==================================\n",
     "Unused import is a commmon code suggestion provided by lint tool like `flake8 F401 <https://lintlyci.github.io/Flake8Rules/rules/F401.html>`_ ``imported but unused``.\n",
-    "Even though reporting unused import is already useful, with LibCST we can provide automatic fix to remove unused import. That can make the suggestion more actionable and save developer's time.\n",
+    "Even though reporting unused imports is already useful, with LibCST we can provide an automatic fix to remove unused imports. That can make the suggestion more actionable and save developer's time.\n",
     "\n",
     "An import statement may import multiple names, we want to remove those unused names from the import statement. If all the names in the import statement are not used, we remove the entire import.\n",
     "To remove the unused name, we implement ``RemoveUnusedImportTransformer`` by subclassing :class:`~libcst.CSTTransformer`. We overwrite ``leave_Import`` and ``leave_ImportFrom`` to modify the import statements.\n",
-    "When we find the import node in lookup table, we iterate through all ``names`` and keep used names in ``names_to_keep``.\n",
+    "When we find the import node in the lookup table, we iterate through all ``names`` and keep used names in ``names_to_keep``.\n",
     "If ``names_to_keep`` is empty, all names are unused and we remove the entire import node.\n",
-    "Otherwise, we update the import node and just removing partial names."
+    "Otherwise, we update the import node and just remove partial names."
    ]
   },
   {
@@ -195,7 +195,7 @@
     "raw_mimetype": "text/restructuredtext"
    },
    "source": [
-    "After the transform, we use ``.code`` to generate fixed code and all unused names are fixed as expected! The difflib is used to show only changed part and only import lines are updated as expected."
+    "After the transform, we use ``.code`` to generate the fixed code and all unused names are fixed as expected! The difflib is used to show only the changed part and only imported lines are updated as expected."
    ]
   },
   {

docs/source/tutorial.ipynb

@@ -1,24 +1,25 @@
 {
  "cells": [
   {
-   "cell_type": "raw",
    "metadata": {
     "raw_mimetype": "text/restructuredtext"
    },
+   "cell_type": "raw",
    "source": [
     "====================\n",
     "Parsing and Visiting\n",
     "====================\n",
     "\n",
-    "LibCST provides helpers to parse source code string as concrete syntax tree. In order to perform static analysis to identify patterns in the tree or modify the tree programmatically, we can use visitor pattern to traverse the tree. In this tutorial, we demonstrate a common three-step-workflow to build an automated refactoring (codemod) application:\n",
+    "LibCST provides helpers to parse source code string as a concrete syntax tree. In order to perform static analysis to identify patterns in the tree or modify the tree programmatically, we can use the visitor pattern to traverse the tree. In this tutorial, we demonstrate a common four-step-workflow to build an automated refactoring (codemod) application:\n",
     "\n",
     "1. `Parse Source Code <#Parse-Source-Code>`_\n",
-    "2. `Build Visitor or Transformer <#Build-Visitor-or-Transformer>`_\n",
-    "3. `Generate Source Code <#Generate-Source-Code>`_\n",
+    "2. `Display The Source Code CST <#Display-Source-Code-CST>`_\n",
+    "3. `Build Visitor or Transformer <#Build-Visitor-or-Transformer>`_\n",
+    "4. `Generate Source Code <#Generate-Source-Code>`_\n",
     "\n",
     "Parse Source Code\n",
     "=================\n",
-    "LibCST provides various helpers to parse source code as concrete syntax tree: :func:`~libcst.parse_module`, :func:`~libcst.parse_expression` and :func:`~libcst.parse_statement` (see :doc:`Parsing <parser>` for more detail). The default :class:`~libcst.CSTNode` repr provides pretty print formatting for reading the tree easily."
+    "LibCST provides various helpers to parse source code as a concrete syntax tree: :func:`~libcst.parse_module`, :func:`~libcst.parse_expression` and :func:`~libcst.parse_statement` (see :doc:`Parsing <parser>` for more detail)."
    ]
   },
   {
@@ -41,7 +42,42 @@
    "source": [
     "import libcst as cst\n",
     "\n",
-    "cst.parse_expression(\"1 + 2\")"
+    "source_tree = cst.parse_expression(\"1 + 2\")"
+   ]
+  },
+  {
+   "metadata": {
+    "raw_mimetype": "text/restructuredtext"
+   },
+   "cell_type": "raw",
+   "source": [
+    "|\n",
+    "Display Source Code CST\n",
+    "=======================\n",
+    "The default :class:`~libcst.CSTNode` repr provides pretty print formatting for displaying the entire CST tree."
+   ]
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "print(source_tree)"
+  },
+  {
+   "metadata": {},
+   "cell_type": "raw",
+   "source": "The entire CST tree may be overwhelming at times. To only focus on essential elements of the CST tree, LibCST provides the ``dump`` helper."
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": [
+    "from libcst.display import dump\n",
+    "\n",
+    "print(dump(source_tree))"
    ]
   },
   {
@@ -50,9 +86,11 @@
     "raw_mimetype": "text/restructuredtext"
    },
    "source": [
+    "  \n",
+    "|\n",
     "Example: add typing annotation from pyi stub file to Python source\n",
     "------------------------------------------------------------------\n",
-    "Python `typing annotation <https://mypy.readthedocs.io/en/latest/cheat_sheet_py3.html>`_ was added in Python 3.5. Some Python applications add typing annotations in separate ``pyi`` stub files in order to support old Python versions. When applications decide to stop supporting old Python versions, they'll want to automatically copy the type annotation from a pyi file to a source file. Here we demonstrate how to do that easliy using LibCST. The first step is to parse the pyi stub and source files as trees."
+    "Python `typing annotation <https://mypy.readthedocs.io/en/latest/cheat_sheet_py3.html>`_ was added in Python 3.5. Some Python applications add typing annotations in separate ``pyi`` stub files in order to support old Python versions. When applications decide to stop supporting old Python versions, they'll want to automatically copy the type annotation from a pyi file to a source file. Here we demonstrate how to do that easily using LibCST. The first step is to parse the pyi stub and source files as trees."
    ]
   },
   {
@@ -68,7 +106,7 @@
     "                self._replace(type=self.type.name))\n",
     "\n",
     "def tokenize(code, version_info, start_pos=(1, 0)):\n",
-    "    \"\"\"Generate tokens from a the source code (string).\"\"\"\n",
+    "    \"\"\"Generate tokens from the source code (string).\"\"\"\n",
     "    lines = split_lines(code, keepends=True)\n",
     "    return tokenize_lines(lines, version_info, start_pos=start_pos)\n",
     "'''\n",
@@ -92,10 +130,11 @@
     "raw_mimetype": "text/restructuredtext"
    },
    "source": [
+    "|\n",
     "Build Visitor or Transformer\n",
     "============================\n",
     "For traversing and modifying the tree, LibCST provides Visitor and Transformer classes similar to the `ast module <https://docs.python.org/3/library/ast.html#ast.NodeVisitor>`_. To implement a visitor (read only) or transformer (read/write), simply implement a subclass of :class:`~libcst.CSTVisitor` or :class:`~libcst.CSTTransformer` (see :doc:`Visitors <visitors>` for more detail).\n",
-    "In the typing example, we need to implement a visitor to collect typing annotation from the stub tree and a transformer to copy the annotation to the function signature. In the visitor, we implement ``visit_FunctionDef`` to collect annotations. Later in the transformer, we implement ``leave_FunctionDef`` to add the collected annotations."
+    "In the typing example, we need to implement a visitor to collect typing annotations from the stub tree and a transformer to copy the annotation to the function signature. In the visitor, we implement ``visit_FunctionDef`` to collect annotations. Later in the transformer, we implement ``leave_FunctionDef`` to add the collected annotations."
    ]
   },
   {
@@ -184,9 +223,10 @@
     "raw_mimetype": "text/restructuredtext"
    },
    "source": [
+    "|\n",
     "Generate Source Code\n",
     "====================\n",
-    "Generating the source code from a cst tree is as easy as accessing the :attr:`~libcst.Module.code` attribute on :class:`~libcst.Module`. After the code generation, we often use `ufmt <https://ufmt.omnilib.dev/en/stable/>`_ to reformate the code to keep a consistent coding style."
+    "Generating the source code from a cst tree is as easy as accessing the :attr:`~libcst.Module.code` attribute on :class:`~libcst.Module`. After the code generation, we often use `ufmt <https://ufmt.omnilib.dev/en/stable/>`_ to reformat the code to keep a consistent coding style."
    ]
   },
   {

libcst/__init__.py

@@ -4,7 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 
 from libcst._batched_visitor import BatchableCSTVisitor, visit_batched
-from libcst._exceptions import MetadataException, ParserSyntaxError
+from libcst._exceptions import CSTLogicError, MetadataException, ParserSyntaxError
 from libcst._flatten_sentinel import FlattenSentinel
 from libcst._maybe_sentinel import MaybeSentinel
 from libcst._metadata_dependent import MetadataDependent
@@ -29,6 +29,7 @@ from libcst._nodes.expression import (
     BaseSimpleComp,
     BaseSlice,
     BaseString,
+    BaseTemplatedStringContent,
     BinaryOperation,
     BooleanOperation,
     Call,
@@ -75,6 +76,9 @@ from libcst._nodes.expression import (
     StarredElement,
     Subscript,
     SubscriptElement,
+    TemplatedString,
+    TemplatedStringExpression,
+    TemplatedStringText,
     Tuple,
     UnaryOperation,
     Yield,
@@ -242,6 +246,7 @@ __all__ = [
     "CSTVisitorT",
     "FlattenSentinel",
     "MaybeSentinel",
+    "CSTLogicError",
     "MetadataException",
     "ParserSyntaxError",
     "PartialParserConfig",
@@ -267,6 +272,7 @@ __all__ = [
     "BaseElement",
     "BaseExpression",
     "BaseFormattedStringContent",
+    "BaseTemplatedStringContent",
     "BaseList",
     "BaseNumber",
     "BaseSet",
@@ -290,6 +296,9 @@ __all__ = [
     "FormattedString",
     "FormattedStringExpression",
     "FormattedStringText",
+    "TemplatedString",
+    "TemplatedStringText",
+    "TemplatedStringExpression",
     "From",
     "GeneratorExp",
     "IfExp",

libcst/_exceptions.py

@@ -4,16 +4,11 @@
 # LICENSE file in the root directory of this source tree.
 
 from enum import auto, Enum
-from typing import Any, Callable, final, Iterable, Optional, Sequence, Tuple, Union
+from typing import Any, Callable, final, Optional, Sequence, Tuple
 
-from libcst._parser.parso.pgen2.generator import ReservedString
-from libcst._parser.parso.python.token import PythonTokenTypes, TokenType
-from libcst._parser.types.token import Token
 from libcst._tabs import expand_tabs
 
-_EOF_STR: str = "end of file (EOF)"
-_INDENT_STR: str = "an indent"
-_DEDENT_STR: str = "a dedent"
+
 _NEWLINE_CHARS: str = "\r\n"
 
 
@@ -21,42 +16,10 @@ class EOFSentinel(Enum):
     EOF = auto()
 
 
-def get_expected_str(
-    encountered: Union[Token, EOFSentinel],
-    expected: Union[Iterable[Union[TokenType, ReservedString]], EOFSentinel],
-) -> str:
-    if (
-        isinstance(encountered, EOFSentinel)
-        or encountered.type is PythonTokenTypes.ENDMARKER
-    ):
-        encountered_str = _EOF_STR
-    elif encountered.type is PythonTokenTypes.INDENT:
-        encountered_str = _INDENT_STR
-    elif encountered.type is PythonTokenTypes.DEDENT:
-        encountered_str = _DEDENT_STR
-    else:
-        encountered_str = repr(encountered.string)
+class CSTLogicError(Exception):
+    """General purpose internal error within LibCST itself."""
 
-    if isinstance(expected, EOFSentinel):
-        expected_names = [_EOF_STR]
-    else:
-        expected_names = sorted(
-            [
-                repr(el.name) if isinstance(el, TokenType) else repr(el.value)
-                for el in expected
-            ]
-        )
-
-    if len(expected_names) > 10:
-        # There's too many possibilities, so it's probably not useful to list them.
-        # Instead, let's just abbreviate the message.
-        return f"Unexpectedly encountered {encountered_str}."
-    else:
-        if len(expected_names) == 1:
-            expected_str = expected_names[0]
-        else:
-            expected_str = f"{', '.join(expected_names[:-1])}, or {expected_names[-1]}"
-        return f"Encountered {encountered_str}, but expected {expected_str}."
+    pass
 
 
 # pyre-fixme[2]: 'Any' type isn't pyre-strict.

libcst/_nodes/base.py

@@ -8,6 +8,7 @@ from copy import deepcopy
 from dataclasses import dataclass, field, fields, replace
 from typing import Any, cast, ClassVar, Dict, List, Mapping, Sequence, TypeVar, Union
 
+from libcst import CSTLogicError
 from libcst._flatten_sentinel import FlattenSentinel
 from libcst._nodes.internal import CodegenState
 from libcst._removal_sentinel import RemovalSentinel
@@ -237,7 +238,7 @@ class CSTNode(ABC):
 
         # validate return type of the user-defined `visitor.on_leave` method
         if not isinstance(leave_result, (CSTNode, RemovalSentinel, FlattenSentinel)):
-            raise Exception(
+            raise CSTValidationError(
                 "Expected a node of type CSTNode or a RemovalSentinel, "
                 + f"but got a return value of {type(leave_result).__name__}"
             )
@@ -292,8 +293,7 @@ class CSTNode(ABC):
         return False
 
     @abstractmethod
-    def _codegen_impl(self, state: CodegenState) -> None:
-        ...
+    def _codegen_impl(self, state: CodegenState) -> None: ...
 
     def _codegen(self, state: CodegenState, **kwargs: Any) -> None:
         state.before_codegen(self)
@@ -383,7 +383,7 @@ class CSTNode(ABC):
         new_tree = self.visit(_ChildReplacementTransformer(old_node, new_node))
         if isinstance(new_tree, (FlattenSentinel, RemovalSentinel)):
             # The above transform never returns *Sentinel, so this isn't possible
-            raise Exception("Logic error, cannot get a *Sentinel here!")
+            raise CSTLogicError("Logic error, cannot get a *Sentinel here!")
         return new_tree
 
     def deep_remove(
@@ -400,7 +400,7 @@ class CSTNode(ABC):
 
         if isinstance(new_tree, FlattenSentinel):
             # The above transform never returns FlattenSentinel, so this isn't possible
-            raise Exception("Logic error, cannot get a FlattenSentinel here!")
+            raise CSTLogicError("Logic error, cannot get a FlattenSentinel here!")
 
         return new_tree
 
@@ -422,7 +422,7 @@ class CSTNode(ABC):
         new_tree = self.visit(_ChildWithChangesTransformer(old_node, changes))
         if isinstance(new_tree, (FlattenSentinel, RemovalSentinel)):
             # This is impossible with the above transform.
-            raise Exception("Logic error, cannot get a *Sentinel here!")
+            raise CSTLogicError("Logic error, cannot get a *Sentinel here!")
         return new_tree
 
     def __eq__(self: _CSTNodeSelfT, other: object) -> bool:

libcst/_nodes/expression.py

@@ -17,6 +17,8 @@ from tokenize import (
 )
 from typing import Callable, Generator, Literal, Optional, Sequence, Union
 
+from libcst import CSTLogicError
+
 from libcst._add_slots import add_slots
 from libcst._maybe_sentinel import MaybeSentinel
 from libcst._nodes.base import CSTCodegenError, CSTNode, CSTValidationError
@@ -666,7 +668,7 @@ class SimpleString(_BasePrefixedString):
         if len(quote) not in {1, 3}:
             # We shouldn't get here due to construction validation logic,
             # but handle the case anyway.
-            raise Exception(f"Invalid string {self.value}")
+            raise CSTLogicError(f"Invalid string {self.value}")
 
         # pyre-ignore We know via the above validation that we will only
         # ever return one of the four string literals.
@@ -956,6 +958,253 @@ class FormattedString(_BasePrefixedString):
             state.add_token(self.end)
 
 
+class BaseTemplatedStringContent(CSTNode, ABC):
+    """
+    The base type for :class:`TemplatedStringText` and
+    :class:`TemplatedStringExpression`. A :class:`TemplatedString` is composed of a
+    sequence of :class:`BaseTemplatedStringContent` parts.
+    """
+
+    __slots__ = ()
+
+
+@add_slots
+@dataclass(frozen=True)
+class TemplatedStringText(BaseTemplatedStringContent):
+    """
+    Part of a :class:`TemplatedString` that is not inside curly braces (``{`` or ``}``).
+    For example, in::
+
+        f"ab{cd}ef"
+
+    ``ab`` and ``ef`` are :class:`TemplatedStringText` nodes, but ``{cd}`` is a
+    :class:`TemplatedStringExpression`.
+    """
+
+    #: The raw string value, including any escape characters present in the source
+    #: code, not including any enclosing quotes.
+    value: str
+
+    def _visit_and_replace_children(
+        self, visitor: CSTVisitorT
+    ) -> "TemplatedStringText":
+        return TemplatedStringText(value=self.value)
+
+    def _codegen_impl(self, state: CodegenState) -> None:
+        state.add_token(self.value)
+
+
+@add_slots
+@dataclass(frozen=True)
+class TemplatedStringExpression(BaseTemplatedStringContent):
+    """
+    Part of a :class:`TemplatedString` that is inside curly braces (``{`` or ``}``),
+    including the surrounding curly braces. For example, in::
+
+        f"ab{cd}ef"
+
+    ``{cd}`` is a :class:`TemplatedStringExpression`, but ``ab`` and ``ef`` are
+    :class:`TemplatedStringText` nodes.
+
+    An t-string expression may contain ``conversion`` and ``format_spec`` suffixes that
+    control how the expression is converted to a string.
+    """
+
+    #: The expression we will evaluate and render when generating the string.
+    expression: BaseExpression
+
+    #: An optional conversion specifier, such as ``!s``, ``!r`` or ``!a``.
+    conversion: Optional[str] = None
+
+    #: An optional format specifier following the `format specification mini-language
+    #: <https://docs.python.org/3/library/string.html#formatspec>`_.
+    format_spec: Optional[Sequence[BaseTemplatedStringContent]] = None
+
+    #: Whitespace after the opening curly brace (``{``), but before the ``expression``.
+    whitespace_before_expression: BaseParenthesizableWhitespace = (
+        SimpleWhitespace.field("")
+    )
+
+    #: Whitespace after the ``expression``, but before the ``conversion``,
+    #: ``format_spec`` and the closing curly brace (``}``). Python does not
+    #: allow whitespace inside or after a ``conversion`` or ``format_spec``.
+    whitespace_after_expression: BaseParenthesizableWhitespace = SimpleWhitespace.field(
+        ""
+    )
+
+    #: Equal sign for Templated string expression uses self-documenting expressions,
+    #: such as ``f"{x=}"``. See the `Python 3.8 release notes
+    #: <https://docs.python.org/3/whatsnew/3.8.html#f-strings-support-for-self-documenting-expressions-and-debugging>`_.
+    equal: Optional[AssignEqual] = None
+
+    def _validate(self) -> None:
+        if self.conversion is not None and self.conversion not in ("s", "r", "a"):
+            raise CSTValidationError("Invalid t-string conversion.")
+
+    def _visit_and_replace_children(
+        self, visitor: CSTVisitorT
+    ) -> "TemplatedStringExpression":
+        format_spec = self.format_spec
+        return TemplatedStringExpression(
+            whitespace_before_expression=visit_required(
+                self,
+                "whitespace_before_expression",
+                self.whitespace_before_expression,
+                visitor,
+            ),
+            expression=visit_required(self, "expression", self.expression, visitor),
+            equal=visit_optional(self, "equal", self.equal, visitor),
+            whitespace_after_expression=visit_required(
+                self,
+                "whitespace_after_expression",
+                self.whitespace_after_expression,
+                visitor,
+            ),
+            conversion=self.conversion,
+            format_spec=(
+                visit_sequence(self, "format_spec", format_spec, visitor)
+                if format_spec is not None
+                else None
+            ),
+        )
+
+    def _codegen_impl(self, state: CodegenState) -> None:
+        state.add_token("{")
+        self.whitespace_before_expression._codegen(state)
+        self.expression._codegen(state)
+        equal = self.equal
+        if equal is not None:
+            equal._codegen(state)
+        self.whitespace_after_expression._codegen(state)
+        conversion = self.conversion
+        if conversion is not None:
+            state.add_token("!")
+            state.add_token(conversion)
+        format_spec = self.format_spec
+        if format_spec is not None:
+            state.add_token(":")
+            for spec in format_spec:
+                spec._codegen(state)
+        state.add_token("}")
+
+
+@add_slots
+@dataclass(frozen=True)
+class TemplatedString(_BasePrefixedString):
+    """
+    An "t-string". Template strings are a generalization of f-strings,
+    using a t in place of the f prefix. Instead of evaluating to str,
+    t-strings evaluate to a new type: Template
+
+    T-Strings are defined in 'PEP 750'
+
+    >>> import libcst as cst
+    >>> cst.parse_expression('t"ab{cd}ef"')
+    TemplatedString(
+        parts=[
+            TemplatedStringText(
+                value='ab',
+            ),
+            TemplatedStringExpression(
+                expression=Name(
+                    value='cd',
+                    lpar=[],
+                    rpar=[],
+                ),
+                conversion=None,
+                format_spec=None,
+                whitespace_before_expression=SimpleWhitespace(
+                    value='',
+                ),
+                whitespace_after_expression=SimpleWhitespace(
+                    value='',
+                ),
+                equal=None,
+            ),
+            TemplatedStringText(
+                value='ef',
+            ),
+        ],
+        start='t"',
+        end='"',
+        lpar=[],
+        rpar=[],
+    )
+    >>>
+    """
+
+    #: A templated string is composed as a series of :class:`TemplatedStringText` and
+    #: :class:`TemplatedStringExpression` parts.
+    parts: Sequence[BaseTemplatedStringContent]
+
+    #: The string prefix and the leading quote, such as ``t"``, ``T'``, ``tr"``, or
+    #: ``t"""``.
+    start: str = 't"'
+
+    #: The trailing quote. This must match the type of quote used in ``start``.
+    end: Literal['"', "'", '"""', "'''"] = '"'
+
+    lpar: Sequence[LeftParen] = ()
+    #: Sequence of parenthesis for precidence dictation.
+    rpar: Sequence[RightParen] = ()
+
+    def _validate(self) -> None:
+        super(_BasePrefixedString, self)._validate()
+
+        # Validate any prefix
+        prefix = self.prefix
+        if prefix not in ("t", "tr", "rt"):
+            raise CSTValidationError("Invalid t-string prefix.")
+
+        # Validate wrapping quotes
+        starttoken = self.start[len(prefix) :]
+        if starttoken != self.end:
+            raise CSTValidationError("t-string must have matching enclosing quotes.")
+
+        # Validate valid wrapping quote usage
+        if starttoken not in ('"', "'", '"""', "'''"):
+            raise CSTValidationError("Invalid t-string enclosing quotes.")
+
+    @property
+    def prefix(self) -> str:
+        """
+        Returns the string's prefix, if any exists. The prefix can be ``t``,
+        ``tr``, or ``rt``.
+        """
+
+        prefix = ""
+        for c in self.start:
+            if c in ['"', "'"]:
+                break
+            prefix += c
+        return prefix.lower()
+
+    @property
+    def quote(self) -> StringQuoteLiteral:
+        """
+        Returns the quotation used to denote the string. Can be either ``'``,
+        ``"``, ``'''`` or ``\"\"\"``.
+        """
+
+        return self.end
+
+    def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "TemplatedString":
+        return TemplatedString(
+            lpar=visit_sequence(self, "lpar", self.lpar, visitor),
+            start=self.start,
+            parts=visit_sequence(self, "parts", self.parts, visitor),
+            end=self.end,
+            rpar=visit_sequence(self, "rpar", self.rpar, visitor),
+        )
+
+    def _codegen_impl(self, state: CodegenState) -> None:
+        with self._parenthesize(state):
+            state.add_token(self.start)
+            for part in self.parts:
+                part._codegen(state)
+            state.add_token(self.end)
+
+
 @add_slots
 @dataclass(frozen=True)
 class ConcatenatedString(BaseString):
@@ -1010,7 +1259,7 @@ class ConcatenatedString(BaseString):
         elif isinstance(right, FormattedString):
             rightbytes = "b" in right.prefix
         else:
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
         if leftbytes != rightbytes:
             raise CSTValidationError("Cannot concatenate string and bytes.")
 
@@ -1647,9 +1896,9 @@ class Annotation(CSTNode):
     #: colon or arrow.
     annotation: BaseExpression
 
-    whitespace_before_indicator: Union[
-        BaseParenthesizableWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_before_indicator: Union[BaseParenthesizableWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
     whitespace_after_indicator: BaseParenthesizableWhitespace = SimpleWhitespace.field(
         " "
     )
@@ -1688,7 +1937,7 @@ class Annotation(CSTNode):
             if default_indicator == "->":
                 state.add_token(" ")
         else:
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
 
         # Now, output the indicator and the rest of the annotation
         state.add_token(default_indicator)
@@ -2101,9 +2350,9 @@ class Lambda(BaseExpression):
     rpar: Sequence[RightParen] = ()
 
     #: Whitespace after the lambda keyword, but before any argument or the colon.
-    whitespace_after_lambda: Union[
-        BaseParenthesizableWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_after_lambda: Union[BaseParenthesizableWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
         if position == ExpressionPosition.LEFT:
@@ -2601,9 +2850,9 @@ class From(CSTNode):
     item: BaseExpression
 
     #: The whitespace at the very start of this node.
-    whitespace_before_from: Union[
-        BaseParenthesizableWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_before_from: Union[BaseParenthesizableWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     #: The whitespace after the ``from`` keyword, but before the ``item``.
     whitespace_after_from: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
@@ -2662,9 +2911,9 @@ class Yield(BaseExpression):
     rpar: Sequence[RightParen] = ()
 
     #: Whitespace after the ``yield`` keyword, but before the ``value``.
-    whitespace_after_yield: Union[
-        BaseParenthesizableWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_after_yield: Union[BaseParenthesizableWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     def _validate(self) -> None:
         # Paren rules and such
@@ -2748,8 +2997,7 @@ class _BaseElementImpl(CSTNode, ABC):
         state: CodegenState,
         default_comma: bool = False,
         default_comma_whitespace: bool = False,  # False for a single-item collection
-    ) -> None:
-        ...
+    ) -> None: ...
 
 
 class BaseElement(_BaseElementImpl, ABC):

libcst/_nodes/op.py

@@ -43,8 +43,7 @@ class _BaseOneTokenOp(CSTNode, ABC):
         self.whitespace_after._codegen(state)
 
     @abstractmethod
-    def _get_token(self) -> str:
-        ...
+    def _get_token(self) -> str: ...
 
 
 class _BaseTwoTokenOp(CSTNode, ABC):
@@ -88,8 +87,7 @@ class _BaseTwoTokenOp(CSTNode, ABC):
         self.whitespace_after._codegen(state)
 
     @abstractmethod
-    def _get_tokens(self) -> Tuple[str, str]:
-        ...
+    def _get_tokens(self) -> Tuple[str, str]: ...
 
 
 class BaseUnaryOp(CSTNode, ABC):
@@ -115,8 +113,7 @@ class BaseUnaryOp(CSTNode, ABC):
         self.whitespace_after._codegen(state)
 
     @abstractmethod
-    def _get_token(self) -> str:
-        ...
+    def _get_token(self) -> str: ...
 
 
 class BaseBooleanOp(_BaseOneTokenOp, ABC):

libcst/_nodes/statement.py

@@ -7,7 +7,9 @@ import inspect
 import re
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
-from typing import Optional, Pattern, Sequence, Union
+from typing import Literal, Optional, Pattern, Sequence, Union
+
+from libcst import CSTLogicError
 
 from libcst._add_slots import add_slots
 from libcst._maybe_sentinel import MaybeSentinel
@@ -113,8 +115,7 @@ class BaseSmallStatement(CSTNode, ABC):
     @abstractmethod
     def _codegen_impl(
         self, state: CodegenState, default_semicolon: bool = False
-    ) -> None:
-        ...
+    ) -> None: ...
 
 
 @add_slots
@@ -273,9 +274,9 @@ class Return(BaseSmallStatement):
 
     #: Optional whitespace after the ``return`` keyword before the optional
     #: value expression.
-    whitespace_after_return: Union[
-        SimpleWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_after_return: Union[SimpleWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     #: Optional semicolon when this is used in a statement line. This semicolon
     #: owns the whitespace on both sides of it when it is used.
@@ -599,7 +600,12 @@ class If(BaseCompoundStatement):
     #: The whitespace appearing after the test expression but before the colon.
     whitespace_after_test: SimpleWhitespace = SimpleWhitespace.field("")
 
-    # TODO: _validate
+    def _validate(self) -> None:
+        if (
+            self.whitespace_before_test.empty
+            and not self.test._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
+        ):
+            raise CSTValidationError("Must have at least one space after 'if' keyword.")
 
     def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "If":
         return If(
@@ -1161,12 +1167,10 @@ class ImportAlias(CSTNode):
                 )
         try:
             self.evaluated_name
-        except Exception as e:
-            if str(e) == "Logic error!":
-                raise CSTValidationError(
-                    "The imported name must be a valid qualified name."
-                )
-            raise e
+        except CSTLogicError as e:
+            raise CSTValidationError(
+                "The imported name must be a valid qualified name."
+            ) from e
 
     def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ImportAlias":
         return ImportAlias(
@@ -1195,7 +1199,7 @@ class ImportAlias(CSTNode):
         elif isinstance(node, Attribute):
             return f"{self._name(node.value)}.{node.attr.value}"
         else:
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
 
     @property
     def evaluated_name(self) -> str:
@@ -2398,9 +2402,9 @@ class Raise(BaseSmallStatement):
     cause: Optional[From] = None
 
     #: Any whitespace appearing between the ``raise`` keyword and the exception.
-    whitespace_after_raise: Union[
-        SimpleWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_after_raise: Union[SimpleWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     #: Optional semicolon when this is used in a statement line. This semicolon
     #: owns the whitespace on both sides of it when it is used.
@@ -2854,17 +2858,16 @@ class MatchCase(CSTNode):
                 self, "whitespace_after_case", self.whitespace_after_case, visitor
             ),
             pattern=visit_required(self, "pattern", self.pattern, visitor),
-            # pyre-fixme[6]: Expected `SimpleWhitespace` for 4th param but got
-            #  `Optional[SimpleWhitespace]`.
-            whitespace_before_if=visit_optional(
+            whitespace_before_if=visit_required(
                 self, "whitespace_before_if", self.whitespace_before_if, visitor
             ),
-            # pyre-fixme[6]: Expected `SimpleWhitespace` for 5th param but got
-            #  `Optional[SimpleWhitespace]`.
-            whitespace_after_if=visit_optional(
+            whitespace_after_if=visit_required(
                 self, "whitespace_after_if", self.whitespace_after_if, visitor
             ),
             guard=visit_optional(self, "guard", self.guard, visitor),
+            whitespace_before_colon=visit_required(
+                self, "whitespace_before_colon", self.whitespace_before_colon, visitor
+            ),
             body=visit_required(self, "body", self.body, visitor),
         )
 
@@ -2883,6 +2886,9 @@ class MatchCase(CSTNode):
                 state.add_token("if")
                 self.whitespace_after_if._codegen(state)
                 guard._codegen(state)
+            else:
+                self.whitespace_before_if._codegen(state)
+                self.whitespace_after_if._codegen(state)
 
             self.whitespace_before_colon._codegen(state)
             state.add_token(":")
@@ -3382,6 +3388,7 @@ class MatchClass(MatchPattern):
             whitespace_after_kwds=visit_required(
                 self, "whitespace_after_kwds", self.whitespace_after_kwds, visitor
             ),
+            rpar=visit_sequence(self, "rpar", self.rpar, visitor),
         )
 
     def _codegen_impl(self, state: CodegenState) -> None:
@@ -3418,15 +3425,15 @@ class MatchAs(MatchPattern):
 
     #: Whitespace between ``pattern`` and the ``as`` keyword (if ``pattern`` is not
     #: ``None``)
-    whitespace_before_as: Union[
-        BaseParenthesizableWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_before_as: Union[BaseParenthesizableWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     #: Whitespace between the ``as`` keyword and ``name`` (if ``pattern`` is not
     #: ``None``)
-    whitespace_after_as: Union[
-        BaseParenthesizableWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_after_as: Union[BaseParenthesizableWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     #: Parenthesis at the beginning of the node
     lpar: Sequence[LeftParen] = ()
@@ -3469,6 +3476,13 @@ class MatchAs(MatchPattern):
                     state.add_token(" ")
                 elif isinstance(ws_after, BaseParenthesizableWhitespace):
                     ws_after._codegen(state)
+            else:
+                ws_before = self.whitespace_before_as
+                if isinstance(ws_before, BaseParenthesizableWhitespace):
+                    ws_before._codegen(state)
+                ws_after = self.whitespace_after_as
+                if isinstance(ws_after, BaseParenthesizableWhitespace):
+                    ws_after._codegen(state)
             if name is None:
                 state.add_token("_")
             else:
@@ -3653,8 +3667,34 @@ class TypeParam(CSTNode):
     #: with a comma only if a comma is required.
     comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
 
+    #: The equal sign used to denote assignment if there is a default.
+    equal: Union[AssignEqual, MaybeSentinel] = MaybeSentinel.DEFAULT
+
+    #: The star used to denote a variadic default
+    star: Literal["", "*"] = ""
+
+    #: The whitespace between the star and the type.
+    whitespace_after_star: SimpleWhitespace = SimpleWhitespace.field("")
+
+    #: Any optional default value, used when the argument is not supplied.
+    default: Optional[BaseExpression] = None
+
     def _codegen_impl(self, state: CodegenState, default_comma: bool = False) -> None:
         self.param._codegen(state)
+
+        equal = self.equal
+        if equal is MaybeSentinel.DEFAULT and self.default is not None:
+            state.add_token(" = ")
+        elif isinstance(equal, AssignEqual):
+            equal._codegen(state)
+
+        state.add_token(self.star)
+        self.whitespace_after_star._codegen(state)
+
+        default = self.default
+        if default is not None:
+            default._codegen(state)
+
         comma = self.comma
         if isinstance(comma, MaybeSentinel):
             if default_comma:
@@ -3663,10 +3703,27 @@ class TypeParam(CSTNode):
             comma._codegen(state)
 
     def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "TypeParam":
-        return TypeParam(
+        ret = TypeParam(
             param=visit_required(self, "param", self.param, visitor),
+            equal=visit_sentinel(self, "equal", self.equal, visitor),
+            star=self.star,
+            whitespace_after_star=visit_required(
+                self, "whitespace_after_star", self.whitespace_after_star, visitor
+            ),
+            default=visit_optional(self, "default", self.default, visitor),
             comma=visit_sentinel(self, "comma", self.comma, visitor),
         )
+        return ret
+
+    def _validate(self) -> None:
+        if self.default is None and isinstance(self.equal, AssignEqual):
+            raise CSTValidationError(
+                "Must have a default when specifying an AssignEqual."
+            )
+        if self.star and not (self.default or isinstance(self.equal, AssignEqual)):
+            raise CSTValidationError("Star can only be present if a default")
+        if isinstance(self.star, str) and self.star not in ("", "*"):
+            raise CSTValidationError("Must specify either '' or '*' for star.")
 
 
 @add_slots
@@ -3726,16 +3783,16 @@ class TypeAlias(BaseSmallStatement):
     #: Whitespace between the name and the type parameters (if they exist) or the ``=``.
     #: If not specified, :class:`MaybeSentinel` will be replaced with a single space if
     #: there are no type parameters, otherwise no spaces.
-    whitespace_after_name: Union[
-        SimpleWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_after_name: Union[SimpleWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     #: Whitespace between the type parameters and the ``=``. Always empty if there are
     #: no type parameters. If not specified, :class:`MaybeSentinel` will be replaced
     #: with a single space if there are type parameters.
-    whitespace_after_type_parameters: Union[
-        SimpleWhitespace, MaybeSentinel
-    ] = MaybeSentinel.DEFAULT
+    whitespace_after_type_parameters: Union[SimpleWhitespace, MaybeSentinel] = (
+        MaybeSentinel.DEFAULT
+    )
 
     #: Whitespace between the ``=`` and the value.
     whitespace_after_equals: SimpleWhitespace = SimpleWhitespace.field(" ")

libcst/_nodes/tests/test_atom.py

@@ -9,7 +9,6 @@ from typing import Any
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -1184,7 +1183,7 @@ class AtomTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)
 

libcst/_nodes/tests/test_binary_op.py

@@ -8,7 +8,6 @@ from typing import Any
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -189,4 +188,4 @@ class BinaryOperationTest(CSTNodeTest):
         )
     )
     def test_parse_error(self, **kwargs: Any) -> None:
-        self.assert_parses(**kwargs, expect_success=not is_native())
+        self.assert_parses(**kwargs, expect_success=False)

libcst/_nodes/tests/test_classdef.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -210,8 +209,6 @@ class ClassDefCreationTest(CSTNodeTest):
         )
     )
     def test_valid_native(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Disabled for pure python parser")
         self.validate_node(**kwargs)
 
     @data_provider(

libcst/_nodes/tests/test_dict.py

@@ -8,7 +8,6 @@ from typing import Any
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -188,6 +187,6 @@ class DictTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_funcdef.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock, parse_statement_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -741,8 +740,6 @@ class FunctionDefCreationTest(CSTNodeTest):
         )
     )
     def test_valid(self, **kwargs: Any) -> None:
-        if not is_native() and kwargs.get("native_only", False):
-            self.skipTest("Disabled for native parser")
         if "native_only" in kwargs:
             kwargs.pop("native_only")
         self.validate_node(**kwargs)
@@ -891,8 +888,6 @@ class FunctionDefCreationTest(CSTNodeTest):
         )
     )
     def test_valid_native(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Disabled for pure python parser")
         self.validate_node(**kwargs)
 
     @data_provider(
@@ -1052,7 +1047,9 @@ def _parse_statement_force_38(code: str) -> cst.BaseCompoundStatement:
         code, config=cst.PartialParserConfig(python_version="3.8")
     )
     if not isinstance(statement, cst.BaseCompoundStatement):
-        raise Exception("This function is expecting to parse compound statements only!")
+        raise ValueError(
+            "This function is expecting to parse compound statements only!"
+        )
     return statement
 
 
@@ -2221,8 +2218,6 @@ class FunctionDefParserTest(CSTNodeTest):
         )
     )
     def test_valid_38(self, node: cst.CSTNode, code: str, **kwargs: Any) -> None:
-        if not is_native() and kwargs.get("native_only", False):
-            self.skipTest("disabled for pure python parser")
         self.validate_node(node, code, _parse_statement_force_38)
 
     @data_provider(
@@ -2250,7 +2245,7 @@ class FunctionDefParserTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)
 
@@ -2269,6 +2264,4 @@ class FunctionDefParserTest(CSTNodeTest):
         )
     )
     def test_parse_error(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Skipped for non-native parser")
         self.assert_parses(**kwargs, expect_success=False, parser=parse_statement)

libcst/_nodes/tests/test_if.py

@@ -3,7 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any
+from typing import Any, Callable
 
 import libcst as cst
 from libcst import parse_statement
@@ -129,3 +129,21 @@ class IfTest(CSTNodeTest):
     )
     def test_valid(self, **kwargs: Any) -> None:
         self.validate_node(**kwargs)
+
+    @data_provider(
+        (
+            # Validate whitespace handling
+            (
+                lambda: cst.If(
+                    cst.Name("conditional"),
+                    cst.SimpleStatementSuite((cst.Pass(),)),
+                    whitespace_before_test=cst.SimpleWhitespace(""),
+                ),
+                "Must have at least one space after 'if' keyword.",
+            ),
+        )
+    )
+    def test_invalid(
+        self, get_node: Callable[[], cst.CSTNode], expected_re: str
+    ) -> None:
+        self.assert_invalid(get_node, expected_re)

libcst/_nodes/tests/test_list.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_expression, parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -126,6 +125,6 @@ class ListTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_match.py

@@ -3,17 +3,14 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any, Callable, Optional
+from typing import Any, Callable
 
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
-parser: Optional[Callable[[str], cst.CSTNode]] = (
-    parse_statement if is_native() else None
-)
+parser: Callable[[str], cst.CSTNode] = parse_statement
 
 
 class MatchTest(CSTNodeTest):

libcst/_nodes/tests/test_matrix_multiply.py

@@ -11,7 +11,6 @@ from libcst._nodes.tests.base import (
     parse_expression_as,
     parse_statement_as,
 )
-from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
 
@@ -70,6 +69,6 @@ class NamedExprTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_module.py

@@ -8,7 +8,7 @@ from typing import cast, Tuple
 import libcst as cst
 from libcst import parse_module, parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
+
 from libcst.metadata import CodeRange, MetadataWrapper, PositionProvider
 from libcst.testing.utils import data_provider
 
@@ -117,7 +117,7 @@ class ModuleTest(CSTNodeTest):
     def test_parser(
         self, *, code: str, expected: cst.Module, enabled_for_native: bool = True
     ) -> None:
-        if is_native() and not enabled_for_native:
+        if not enabled_for_native:
             self.skipTest("Disabled for native parser")
         self.assertEqual(parse_module(code), expected)
 

libcst/_nodes/tests/test_namedexpr.py

@@ -22,7 +22,9 @@ def _parse_statement_force_38(code: str) -> cst.BaseCompoundStatement:
         code, config=cst.PartialParserConfig(python_version="3.8")
     )
     if not isinstance(statement, cst.BaseCompoundStatement):
-        raise Exception("This function is expecting to parse compound statements only!")
+        raise ValueError(
+            "This function is expecting to parse compound statements only!"
+        )
     return statement
 
 

libcst/_nodes/tests/test_removal_behavior.py

@@ -95,7 +95,7 @@ class RemovalBehavior(CSTNodeTest):
         self, before: str, after: str, visitor: Type[CSTTransformer]
     ) -> None:
         if before.endswith("\n") or after.endswith("\n"):
-            raise Exception("Test cases should not be newline-terminated!")
+            raise ValueError("Test cases should not be newline-terminated!")
 
         # Test doesn't have newline termination case
         before_module = parse_module(before)

libcst/_nodes/tests/test_set.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
 
@@ -133,6 +132,6 @@ class ListTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_template_strings.py

@@ -0,0 +1,183 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Callable, Optional
+
+import libcst as cst
+from libcst import parse_expression
+from libcst._nodes.tests.base import CSTNodeTest
+from libcst.metadata import CodeRange
+from libcst.testing.utils import data_provider
+
+
+class TemplatedStringTest(CSTNodeTest):
+    @data_provider(
+        (
+            # Simple t-string with only text
+            (
+                cst.TemplatedString(
+                    parts=(cst.TemplatedStringText("hello world"),),
+                ),
+                't"hello world"',
+                True,
+            ),
+            # t-string with one expression
+            (
+                cst.TemplatedString(
+                    parts=(
+                        cst.TemplatedStringText("hello "),
+                        cst.TemplatedStringExpression(
+                            expression=cst.Name("name"),
+                        ),
+                    ),
+                ),
+                't"hello {name}"',
+                True,
+            ),
+            # t-string with multiple expressions
+            (
+                cst.TemplatedString(
+                    parts=(
+                        cst.TemplatedStringText("a="),
+                        cst.TemplatedStringExpression(expression=cst.Name("a")),
+                        cst.TemplatedStringText(", b="),
+                        cst.TemplatedStringExpression(expression=cst.Name("b")),
+                    ),
+                ),
+                't"a={a}, b={b}"',
+                True,
+                CodeRange((1, 0), (1, 15)),
+            ),
+            # t-string with nested expression
+            (
+                cst.TemplatedString(
+                    parts=(
+                        cst.TemplatedStringText("sum="),
+                        cst.TemplatedStringExpression(
+                            expression=cst.BinaryOperation(
+                                left=cst.Name("a"),
+                                operator=cst.Add(),
+                                right=cst.Name("b"),
+                            )
+                        ),
+                    ),
+                ),
+                't"sum={a + b}"',
+                True,
+            ),
+            # t-string with spacing in expression
+            (
+                cst.TemplatedString(
+                    parts=(
+                        cst.TemplatedStringText("x = "),
+                        cst.TemplatedStringExpression(
+                            whitespace_before_expression=cst.SimpleWhitespace(" "),
+                            expression=cst.Name("x"),
+                            whitespace_after_expression=cst.SimpleWhitespace(" "),
+                        ),
+                    ),
+                ),
+                't"x = { x }"',
+                True,
+            ),
+            # t-string with escaped braces
+            (
+                cst.TemplatedString(
+                    parts=(cst.TemplatedStringText("{{foo}}"),),
+                ),
+                't"{{foo}}"',
+                True,
+            ),
+            # t-string with only an expression
+            (
+                cst.TemplatedString(
+                    parts=(
+                        cst.TemplatedStringExpression(expression=cst.Name("value")),
+                    ),
+                ),
+                't"{value}"',
+                True,
+            ),
+            # t-string with whitespace and newlines
+            (
+                cst.TemplatedString(
+                    parts=(
+                        cst.TemplatedStringText("line1\\n"),
+                        cst.TemplatedStringExpression(expression=cst.Name("x")),
+                        cst.TemplatedStringText("\\nline2"),
+                    ),
+                ),
+                't"line1\\n{x}\\nline2"',
+                True,
+            ),
+            # t-string with parenthesis (not typical, but test node construction)
+            (
+                cst.TemplatedString(
+                    lpar=(cst.LeftParen(),),
+                    parts=(cst.TemplatedStringText("foo"),),
+                    rpar=(cst.RightParen(),),
+                ),
+                '(t"foo")',
+                True,
+            ),
+            # t-string with whitespace in delimiters
+            (
+                cst.TemplatedString(
+                    lpar=(cst.LeftParen(whitespace_after=cst.SimpleWhitespace(" ")),),
+                    parts=(cst.TemplatedStringText("foo"),),
+                    rpar=(cst.RightParen(whitespace_before=cst.SimpleWhitespace(" ")),),
+                ),
+                '( t"foo" )',
+                True,
+            ),
+            # Test TemplatedStringText and TemplatedStringExpression individually
+            (
+                cst.TemplatedStringText("abc"),
+                "abc",
+                False,
+                CodeRange((1, 0), (1, 3)),
+            ),
+            (
+                cst.TemplatedStringExpression(expression=cst.Name("foo")),
+                "{foo}",
+                False,
+                CodeRange((1, 0), (1, 5)),
+            ),
+        )
+    )
+    def test_valid(
+        self,
+        node: cst.CSTNode,
+        code: str,
+        check_parsing: bool,
+        position: Optional[CodeRange] = None,
+    ) -> None:
+        if check_parsing:
+            self.validate_node(node, code, parse_expression, expected_position=position)
+        else:
+            self.validate_node(node, code, expected_position=position)
+
+    @data_provider(
+        (
+            (
+                lambda: cst.TemplatedString(
+                    parts=(cst.TemplatedStringText("foo"),),
+                    lpar=(cst.LeftParen(),),
+                ),
+                "left paren without right paren",
+            ),
+            (
+                lambda: cst.TemplatedString(
+                    parts=(cst.TemplatedStringText("foo"),),
+                    rpar=(cst.RightParen(),),
+                ),
+                "right paren without left paren",
+            ),
+        )
+    )
+    def test_invalid(
+        self, get_node: Callable[[], cst.CSTNode], expected_re: str
+    ) -> None:
+        self.assert_invalid(get_node, expected_re)

libcst/_nodes/tests/test_try.py

@@ -3,18 +3,15 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any, Callable, Optional
+from typing import Any, Callable
 
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
-native_parse_statement: Optional[Callable[[str], cst.CSTNode]] = (
-    parse_statement if is_native() else None
-)
+native_parse_statement: Callable[[str], cst.CSTNode] = parse_statement
 
 
 class TryTest(CSTNodeTest):
@@ -347,6 +344,34 @@ class TryTest(CSTNodeTest):
                 ),
                 "code": "try: pass\nexcept foo()as bar: pass\n",
             },
+            # PEP758 - Multiple exceptions with no parentheses
+            {
+                "node": cst.Try(
+                    cst.SimpleStatementSuite((cst.Pass(),)),
+                    handlers=[
+                        cst.ExceptHandler(
+                            cst.SimpleStatementSuite((cst.Pass(),)),
+                            type=cst.Tuple(
+                                elements=[
+                                    cst.Element(
+                                        value=cst.Name(
+                                            value="ValueError",
+                                        ),
+                                    ),
+                                    cst.Element(
+                                        value=cst.Name(
+                                            value="RuntimeError",
+                                        ),
+                                    ),
+                                ],
+                                lpar=[],
+                                rpar=[],
+                            ),
+                        )
+                    ],
+                ),
+                "code": "try: pass\nexcept ValueError, RuntimeError: pass\n",
+            },
         )
     )
     def test_valid(self, **kwargs: Any) -> None:
@@ -579,6 +604,38 @@ class TryStarTest(CSTNodeTest):
                 "parser": native_parse_statement,
                 "expected_position": CodeRange((1, 0), (5, 13)),
             },
+            # PEP758 - Multiple exceptions with no parentheses
+            {
+                "node": cst.TryStar(
+                    cst.SimpleStatementSuite((cst.Pass(),)),
+                    handlers=[
+                        cst.ExceptStarHandler(
+                            cst.SimpleStatementSuite((cst.Pass(),)),
+                            type=cst.Tuple(
+                                elements=[
+                                    cst.Element(
+                                        value=cst.Name(
+                                            value="ValueError",
+                                        ),
+                                        comma=cst.Comma(
+                                            whitespace_after=cst.SimpleWhitespace(" ")
+                                        ),
+                                    ),
+                                    cst.Element(
+                                        value=cst.Name(
+                                            value="RuntimeError",
+                                        ),
+                                    ),
+                                ],
+                                lpar=[],
+                                rpar=[],
+                            ),
+                        )
+                    ],
+                ),
+                "code": "try: pass\nexcept* ValueError, RuntimeError: pass\n",
+                "parser": native_parse_statement,
+            },
         )
     )
     def test_valid(self, **kwargs: Any) -> None:

libcst/_nodes/tests/test_tuple.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_expression, parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -286,6 +285,6 @@ class TupleTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_type_alias.py

@@ -8,7 +8,6 @@ from typing import Any
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -56,11 +55,82 @@ class TypeAliasCreationTest(CSTNodeTest):
                 "code": "type foo[T: str, *Ts, **KW] = bar | baz",
                 "expected_position": CodeRange((1, 0), (1, 39)),
             },
+            {
+                "node": cst.TypeAlias(
+                    cst.Name("foo"),
+                    type_parameters=cst.TypeParameters(
+                        [
+                            cst.TypeParam(
+                                cst.TypeVar(cst.Name("T")), default=cst.Name("str")
+                            ),
+                        ]
+                    ),
+                    value=cst.Name("bar"),
+                ),
+                "code": "type foo[T = str] = bar",
+                "expected_position": CodeRange((1, 0), (1, 23)),
+            },
+            {
+                "node": cst.TypeAlias(
+                    cst.Name("foo"),
+                    type_parameters=cst.TypeParameters(
+                        [
+                            cst.TypeParam(
+                                cst.ParamSpec(cst.Name("P")),
+                                default=cst.List(
+                                    elements=[
+                                        cst.Element(cst.Name("int")),
+                                        cst.Element(cst.Name("str")),
+                                    ]
+                                ),
+                            ),
+                        ]
+                    ),
+                    value=cst.Name("bar"),
+                ),
+                "code": "type foo[**P = [int, str]] = bar",
+                "expected_position": CodeRange((1, 0), (1, 32)),
+            },
+            {
+                "node": cst.TypeAlias(
+                    cst.Name("foo"),
+                    type_parameters=cst.TypeParameters(
+                        [
+                            cst.TypeParam(
+                                cst.TypeVarTuple(cst.Name("T")),
+                                equal=cst.AssignEqual(),
+                                default=cst.Name("default"),
+                                star="*",
+                            ),
+                        ]
+                    ),
+                    value=cst.Name("bar"),
+                ),
+                "code": "type foo[*T = *default] = bar",
+                "expected_position": CodeRange((1, 0), (1, 29)),
+            },
+            {
+                "node": cst.TypeAlias(
+                    cst.Name("foo"),
+                    type_parameters=cst.TypeParameters(
+                        [
+                            cst.TypeParam(
+                                cst.TypeVarTuple(cst.Name("T")),
+                                equal=cst.AssignEqual(),
+                                default=cst.Name("default"),
+                                star="*",
+                                whitespace_after_star=cst.SimpleWhitespace("  "),
+                            ),
+                        ]
+                    ),
+                    value=cst.Name("bar"),
+                ),
+                "code": "type foo[*T = *  default] = bar",
+                "expected_position": CodeRange((1, 0), (1, 31)),
+            },
         )
     )
     def test_valid(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Disabled in the old parser")
         self.validate_node(**kwargs)
 
 
@@ -125,9 +195,58 @@ class TypeAliasParserTest(CSTNodeTest):
                 "code": "type  foo [T:str,**  KW , ]  =  bar ; \n",
                 "parser": parse_statement,
             },
+            {
+                "node": cst.SimpleStatementLine(
+                    [
+                        cst.TypeAlias(
+                            cst.Name("foo"),
+                            type_parameters=cst.TypeParameters(
+                                [
+                                    cst.TypeParam(
+                                        cst.TypeVarTuple(cst.Name("P")),
+                                        star="*",
+                                        equal=cst.AssignEqual(),
+                                        default=cst.Name("default"),
+                                    ),
+                                ]
+                            ),
+                            value=cst.Name("bar"),
+                            whitespace_after_name=cst.SimpleWhitespace(" "),
+                            whitespace_after_type_parameters=cst.SimpleWhitespace(" "),
+                        )
+                    ]
+                ),
+                "code": "type foo [*P = *default] = bar\n",
+                "parser": parse_statement,
+            },
+            {
+                "node": cst.SimpleStatementLine(
+                    [
+                        cst.TypeAlias(
+                            cst.Name("foo"),
+                            type_parameters=cst.TypeParameters(
+                                [
+                                    cst.TypeParam(
+                                        cst.TypeVarTuple(cst.Name("P")),
+                                        star="*",
+                                        whitespace_after_star=cst.SimpleWhitespace(
+                                            "   "
+                                        ),
+                                        equal=cst.AssignEqual(),
+                                        default=cst.Name("default"),
+                                    ),
+                                ]
+                            ),
+                            value=cst.Name("bar"),
+                            whitespace_after_name=cst.SimpleWhitespace(" "),
+                            whitespace_after_type_parameters=cst.SimpleWhitespace(" "),
+                        )
+                    ]
+                ),
+                "code": "type foo [*P = *   default] = bar\n",
+                "parser": parse_statement,
+            },
         )
     )
     def test_valid(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Disabled in the old parser")
         self.validate_node(**kwargs)

libcst/_nodes/tests/test_with.py

@@ -7,9 +7,7 @@ from typing import Any
 
 import libcst as cst
 from libcst import parse_statement, PartialParserConfig
-from libcst._maybe_sentinel import MaybeSentinel
 from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock, parse_statement_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -187,14 +185,14 @@ class WithTest(CSTNodeTest):
                         cst.WithItem(
                             cst.Call(
                                 cst.Name("context_mgr"),
-                                lpar=() if is_native() else (cst.LeftParen(),),
-                                rpar=() if is_native() else (cst.RightParen(),),
+                                lpar=(),
+                                rpar=(),
                             )
                         ),
                     ),
                     cst.SimpleStatementSuite((cst.Pass(),)),
-                    lpar=(cst.LeftParen() if is_native() else MaybeSentinel.DEFAULT),
-                    rpar=(cst.RightParen() if is_native() else MaybeSentinel.DEFAULT),
+                    lpar=(cst.LeftParen()),
+                    rpar=(cst.RightParen()),
                     whitespace_after_with=cst.SimpleWhitespace(""),
                 ),
                 "code": "with(context_mgr()): pass\n",
@@ -233,7 +231,7 @@ class WithTest(CSTNodeTest):
                     rpar=cst.RightParen(whitespace_before=cst.SimpleWhitespace(" ")),
                 ),
                 "code": ("with ( foo(),\n" "       bar(), ): pass\n"),  # noqa
-                "parser": parse_statement if is_native() else None,
+                "parser": parse_statement,
                 "expected_position": CodeRange((1, 0), (2, 21)),
             },
         )
@@ -310,7 +308,7 @@ class WithTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)
 

libcst/_nodes/tests/test_yield.py

@@ -8,7 +8,6 @@ from typing import Any, Callable, Optional
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, parse_statement_as
-from libcst._parser.entrypoints import is_native
 from libcst.helpers import ensure_type
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
@@ -241,6 +240,6 @@ class YieldParsingTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_parser/_parsing_check.py

@@ -0,0 +1,53 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Iterable, Union
+
+from libcst._exceptions import EOFSentinel
+from libcst._parser.parso.pgen2.generator import ReservedString
+from libcst._parser.parso.python.token import PythonTokenTypes, TokenType
+from libcst._parser.types.token import Token
+
+_EOF_STR: str = "end of file (EOF)"
+_INDENT_STR: str = "an indent"
+_DEDENT_STR: str = "a dedent"
+
+
+def get_expected_str(
+    encountered: Union[Token, EOFSentinel],
+    expected: Union[Iterable[Union[TokenType, ReservedString]], EOFSentinel],
+) -> str:
+    if (
+        isinstance(encountered, EOFSentinel)
+        or encountered.type is PythonTokenTypes.ENDMARKER
+    ):
+        encountered_str = _EOF_STR
+    elif encountered.type is PythonTokenTypes.INDENT:
+        encountered_str = _INDENT_STR
+    elif encountered.type is PythonTokenTypes.DEDENT:
+        encountered_str = _DEDENT_STR
+    else:
+        encountered_str = repr(encountered.string)
+
+    if isinstance(expected, EOFSentinel):
+        expected_names = [_EOF_STR]
+    else:
+        expected_names = sorted(
+            [
+                repr(el.name) if isinstance(el, TokenType) else repr(el.value)
+                for el in expected
+            ]
+        )
+
+    if len(expected_names) > 10:
+        # There's too many possibilities, so it's probably not useful to list them.
+        # Instead, let's just abbreviate the message.
+        return f"Unexpectedly encountered {encountered_str}."
+    else:
+        if len(expected_names) == 1:
+            expected_str = expected_names[0]
+        else:
+            expected_str = f"{', '.join(expected_names[:-1])}, or {expected_names[-1]}"
+        return f"Encountered {encountered_str}, but expected {expected_str}."

libcst/_parser/base_parser.py

@@ -26,12 +26,8 @@
 from dataclasses import dataclass, field
 from typing import Generic, Iterable, List, Sequence, TypeVar, Union
 
-from libcst._exceptions import (
-    EOFSentinel,
-    get_expected_str,
-    ParserSyntaxError,
-    PartialParserSyntaxError,
-)
+from libcst._exceptions import EOFSentinel, ParserSyntaxError, PartialParserSyntaxError
+from libcst._parser._parsing_check import get_expected_str
 from libcst._parser.parso.pgen2.generator import DFAState, Grammar, ReservedString
 from libcst._parser.parso.python.token import TokenType
 from libcst._parser.types.token import Token
@@ -103,7 +99,7 @@ class BaseParser(Generic[_TokenT, _TokenTypeT, _NodeT]):
     def parse(self) -> _NodeT:
         # Ensure that we don't re-use parsers.
         if self.__was_parse_called:
-            raise Exception("Each parser object may only be used to parse once.")
+            raise ValueError("Each parser object may only be used to parse once.")
         self.__was_parse_called = True
 
         for token in self.tokens:
@@ -129,11 +125,9 @@ class BaseParser(Generic[_TokenT, _TokenTypeT, _NodeT]):
 
     def convert_nonterminal(
         self, nonterminal: str, children: Sequence[_NodeT]
-    ) -> _NodeT:
-        ...
+    ) -> _NodeT: ...
 
-    def convert_terminal(self, token: _TokenT) -> _NodeT:
-        ...
+    def convert_terminal(self, token: _TokenT) -> _NodeT: ...
 
     def _add_token(self, token: _TokenT) -> None:
         """

libcst/_parser/conversions/expression.py

@@ -12,7 +12,8 @@ from tokenize import (
     Intnumber as INTNUMBER_RE,
 )
 
-from libcst._exceptions import PartialParserSyntaxError
+from libcst import CSTLogicError
+from libcst._exceptions import ParserSyntaxError, PartialParserSyntaxError
 from libcst._maybe_sentinel import MaybeSentinel
 from libcst._nodes.expression import (
     Arg,
@@ -327,7 +328,12 @@ def convert_boolop(
     # Convert all of the operations that have no precedence in a loop
     for op, rightexpr in grouper(rightexprs, 2):
         if op.string not in BOOLOP_TOKEN_LUT:
-            raise Exception(f"Unexpected token '{op.string}'!")
+            raise ParserSyntaxError(
+                f"Unexpected token '{op.string}'!",
+                lines=config.lines,
+                raw_line=0,
+                raw_column=0,
+            )
         leftexpr = BooleanOperation(
             left=leftexpr,
             # pyre-ignore Pyre thinks that the type of the LUT is CSTNode.
@@ -420,7 +426,12 @@ def convert_comp_op(
             )
         else:
             # this should be unreachable
-            raise Exception(f"Unexpected token '{op.string}'!")
+            raise ParserSyntaxError(
+                f"Unexpected token '{op.string}'!",
+                lines=config.lines,
+                raw_line=0,
+                raw_column=0,
+            )
     else:
         # A two-token comparison
         leftcomp, rightcomp = children
@@ -451,7 +462,12 @@ def convert_comp_op(
             )
         else:
             # this should be unreachable
-            raise Exception(f"Unexpected token '{leftcomp.string} {rightcomp.string}'!")
+            raise ParserSyntaxError(
+                f"Unexpected token '{leftcomp.string} {rightcomp.string}'!",
+                lines=config.lines,
+                raw_line=0,
+                raw_column=0,
+            )
 
 
 @with_production("star_expr", "'*' expr")
@@ -493,7 +509,12 @@ def convert_binop(
     # Convert all of the operations that have no precedence in a loop
     for op, rightexpr in grouper(rightexprs, 2):
         if op.string not in BINOP_TOKEN_LUT:
-            raise Exception(f"Unexpected token '{op.string}'!")
+            raise ParserSyntaxError(
+                f"Unexpected token '{op.string}'!",
+                lines=config.lines,
+                raw_line=0,
+                raw_column=0,
+            )
         leftexpr = BinaryOperation(
             left=leftexpr,
             # pyre-ignore Pyre thinks that the type of the LUT is CSTNode.
@@ -540,7 +561,12 @@ def convert_factor(
             )
         )
     else:
-        raise Exception(f"Unexpected token '{op.string}'!")
+        raise ParserSyntaxError(
+            f"Unexpected token '{op.string}'!",
+            lines=config.lines,
+            raw_line=0,
+            raw_column=0,
+        )
 
     return WithLeadingWhitespace(
         UnaryOperation(operator=opnode, expression=factor.value), op.whitespace_before
@@ -651,7 +677,7 @@ def convert_atom_expr_trailer(
             )
         else:
             # This is an invalid trailer, so lets give up
-            raise Exception("Logic error!")
+            raise CSTLogicError()
     return WithLeadingWhitespace(atom, whitespace_before)
 
 
@@ -870,9 +896,19 @@ def convert_atom_basic(
                 Imaginary(child.string), child.whitespace_before
             )
         else:
-            raise Exception(f"Unparseable number {child.string}")
+            raise ParserSyntaxError(
+                f"Unparseable number {child.string}",
+                lines=config.lines,
+                raw_line=0,
+                raw_column=0,
+            )
     else:
-        raise Exception(f"Logic error, unexpected token {child.type.name}")
+        raise ParserSyntaxError(
+            f"Logic error, unexpected token {child.type.name}",
+            lines=config.lines,
+            raw_line=0,
+            raw_column=0,
+        )
 
 
 @with_production("atom_squarebrackets", "'[' [testlist_comp_list] ']'")
@@ -1447,7 +1483,7 @@ def convert_arg_assign_comp_for(
         if equal.string == ":=":
             val = convert_namedexpr_test(config, children)
             if not isinstance(val, WithLeadingWhitespace):
-                raise Exception(
+                raise TypeError(
                     f"convert_namedexpr_test returned {val!r}, not WithLeadingWhitespace"
                 )
             return Arg(value=val.value)

libcst/_parser/conversions/params.py

@@ -6,6 +6,7 @@
 
 from typing import Any, List, Optional, Sequence, Union
 
+from libcst import CSTLogicError
 from libcst._exceptions import PartialParserSyntaxError
 from libcst._maybe_sentinel import MaybeSentinel
 from libcst._nodes.expression import (
@@ -121,7 +122,7 @@ def convert_argslist(  # noqa: C901
                 # Example code:
                 #     def fn(*abc, *): ...
                 # This should be unreachable, the grammar already disallows it.
-                raise Exception(
+                raise ValueError(
                     "Cannot have multiple star ('*') markers in a single argument "
                     + "list."
                 )
@@ -136,7 +137,7 @@ def convert_argslist(  # noqa: C901
                 # Example code:
                 # def fn(foo, /, *, /, bar): ...
                 # This should be unreachable, the grammar already disallows it.
-                raise Exception(
+                raise ValueError(
                     "Cannot have multiple slash ('/') markers in a single argument "
                     + "list."
                 )
@@ -168,7 +169,7 @@ def convert_argslist(  # noqa: C901
                 # Example code:
                 #     def fn(**kwargs, trailing=None)
                 # This should be unreachable, the grammar already disallows it.
-                raise Exception("Cannot have any arguments after a kwargs expansion.")
+                raise ValueError("Cannot have any arguments after a kwargs expansion.")
         elif (
             isinstance(param.star, str) and param.star == "*" and param.default is None
         ):
@@ -181,7 +182,7 @@ def convert_argslist(  # noqa: C901
                 # Example code:
                 #     def fn(*first, *second): ...
                 # This should be unreachable, the grammar already disallows it.
-                raise Exception(
+                raise ValueError(
                     "Expected a keyword argument but found a starred positional "
                     + "argument expansion."
                 )
@@ -197,13 +198,13 @@ def convert_argslist(  # noqa: C901
                 # Example code:
                 #     def fn(**first, **second)
                 # This should be unreachable, the grammar already disallows it.
-                raise Exception(
+                raise ValueError(
                     "Multiple starred keyword argument expansions are not allowed in a "
                     + "single argument list"
                 )
         else:
             # The state machine should never end up here.
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
 
         return current_param
 

libcst/_parser/conversions/statement.py

@@ -6,7 +6,8 @@
 
 from typing import Any, Dict, List, Optional, Sequence, Tuple, Type
 
-from libcst._exceptions import PartialParserSyntaxError
+from libcst import CSTLogicError
+from libcst._exceptions import ParserSyntaxError, PartialParserSyntaxError
 from libcst._maybe_sentinel import MaybeSentinel
 from libcst._nodes.expression import (
     Annotation,
@@ -283,7 +284,9 @@ def convert_annassign(config: ParserConfig, children: Sequence[Any]) -> Any:
             whitespace_after=parse_simple_whitespace(config, equal.whitespace_after),
         )
     else:
-        raise Exception("Invalid parser state!")
+        raise ParserSyntaxError(
+            "Invalid parser state!", lines=config.lines, raw_line=0, raw_column=0
+        )
 
     return AnnAssignPartial(
         annotation=Annotation(
@@ -319,7 +322,13 @@ def convert_annassign(config: ParserConfig, children: Sequence[Any]) -> Any:
 def convert_augassign(config: ParserConfig, children: Sequence[Any]) -> Any:
     op, expr = children
     if op.string not in AUGOP_TOKEN_LUT:
-        raise Exception(f"Unexpected token '{op.string}'!")
+        raise ParserSyntaxError(
+            f"Unexpected token '{op.string}'!",
+            lines=config.lines,
+            raw_line=0,
+            raw_column=0,
+        )
+
     return AugAssignPartial(
         # pyre-ignore Pyre seems to think that the value of this LUT is CSTNode
         operator=AUGOP_TOKEN_LUT[op.string](
@@ -447,7 +456,7 @@ def convert_import_relative(config: ParserConfig, children: Sequence[Any]) -> An
             # This should be the dotted name, and we can't get more than
             # one, but lets be sure anyway
             if dotted_name is not None:
-                raise Exception("Logic error!")
+                raise CSTLogicError()
             dotted_name = child
 
     return ImportRelativePartial(relative=tuple(dots), module=dotted_name)
@@ -644,7 +653,7 @@ def convert_raise_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
             item=source.value,
         )
     else:
-        raise Exception("Logic error!")
+        raise CSTLogicError()
 
     return WithLeadingWhitespace(
         Raise(whitespace_after_raise=whitespace_after_raise, exc=exc, cause=cause),
@@ -893,7 +902,7 @@ def convert_try_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
         if isinstance(clause, Token):
             if clause.string == "else":
                 if orelse is not None:
-                    raise Exception("Logic error!")
+                    raise CSTLogicError("Logic error!")
                 orelse = Else(
                     leading_lines=parse_empty_lines(config, clause.whitespace_before),
                     whitespace_before_colon=parse_simple_whitespace(
@@ -903,7 +912,7 @@ def convert_try_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
                 )
             elif clause.string == "finally":
                 if finalbody is not None:
-                    raise Exception("Logic error!")
+                    raise CSTLogicError("Logic error!")
                 finalbody = Finally(
                     leading_lines=parse_empty_lines(config, clause.whitespace_before),
                     whitespace_before_colon=parse_simple_whitespace(
@@ -912,7 +921,7 @@ def convert_try_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
                     body=suite,
                 )
             else:
-                raise Exception("Logic error!")
+                raise CSTLogicError("Logic error!")
         elif isinstance(clause, ExceptClausePartial):
             handlers.append(
                 ExceptHandler(
@@ -927,7 +936,7 @@ def convert_try_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
                 )
             )
         else:
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
 
     return Try(
         leading_lines=parse_empty_lines(config, trytoken.whitespace_before),
@@ -1333,7 +1342,7 @@ def convert_asyncable_stmt(config: ParserConfig, children: Sequence[Any]) -> Any
             asynchronous=asyncnode, leading_lines=leading_lines
         )
     else:
-        raise Exception("Logic error!")
+        raise CSTLogicError("Logic error!")
 
 
 @with_production("suite", "simple_stmt_suite | indented_suite")

libcst/_parser/entrypoints.py

@@ -9,7 +9,6 @@ parser. A parser entrypoint should take the source code and some configuration
 information
 """
 
-import os
 from functools import partial
 from typing import Union
 
@@ -17,19 +16,12 @@ from libcst._nodes.base import CSTNode
 from libcst._nodes.expression import BaseExpression
 from libcst._nodes.module import Module
 from libcst._nodes.statement import BaseCompoundStatement, SimpleStatementLine
-from libcst._parser.detect_config import convert_to_utf8, detect_config
-from libcst._parser.grammar import get_grammar, validate_grammar
-from libcst._parser.python_parser import PythonCSTParser
+from libcst._parser.detect_config import convert_to_utf8
 from libcst._parser.types.config import PartialParserConfig
 
 _DEFAULT_PARTIAL_PARSER_CONFIG: PartialParserConfig = PartialParserConfig()
 
 
-def is_native() -> bool:
-    typ = os.environ.get("LIBCST_PARSER_TYPE")
-    return typ != "pure"
-
-
 def _parse(
     entrypoint: str,
     source: Union[str, bytes],
@@ -38,57 +30,21 @@ def _parse(
     detect_trailing_newline: bool,
     detect_default_newline: bool,
 ) -> CSTNode:
-    if is_native():
-        from libcst.native import parse_expression, parse_module, parse_statement
 
-        encoding, source_str = convert_to_utf8(source, partial=config)
+    encoding, source_str = convert_to_utf8(source, partial=config)
 
-        if entrypoint == "file_input":
-            parse = partial(parse_module, encoding=encoding)
-        elif entrypoint == "stmt_input":
-            parse = parse_statement
-        elif entrypoint == "expression_input":
-            parse = parse_expression
-        else:
-            raise ValueError(f"Unknown parser entry point: {entrypoint}")
+    from libcst import native
 
-        return parse(source_str)
-    return _pure_python_parse(
-        entrypoint,
-        source,
-        config,
-        detect_trailing_newline=detect_trailing_newline,
-        detect_default_newline=detect_default_newline,
-    )
+    if entrypoint == "file_input":
+        parse = partial(native.parse_module, encoding=encoding)
+    elif entrypoint == "stmt_input":
+        parse = native.parse_statement
+    elif entrypoint == "expression_input":
+        parse = native.parse_expression
+    else:
+        raise ValueError(f"Unknown parser entry point: {entrypoint}")
 
-
-def _pure_python_parse(
-    entrypoint: str,
-    source: Union[str, bytes],
-    config: PartialParserConfig,
-    *,
-    detect_trailing_newline: bool,
-    detect_default_newline: bool,
-) -> CSTNode:
-    detection_result = detect_config(
-        source,
-        partial=config,
-        detect_trailing_newline=detect_trailing_newline,
-        detect_default_newline=detect_default_newline,
-    )
-    validate_grammar()
-    grammar = get_grammar(config.parsed_python_version, config.future_imports)
-
-    parser = PythonCSTParser(
-        tokens=detection_result.tokens,
-        config=detection_result.config,
-        pgen_grammar=grammar,
-        start_nonterminal=entrypoint,
-    )
-    # The parser has an Any return type, we can at least refine it to CSTNode here.
-    result = parser.parse()
-    assert isinstance(result, CSTNode)
-    return result
+    return parse(source_str)
 
 
 def parse_module(

libcst/_parser/grammar.py

@@ -319,7 +319,7 @@ def validate_grammar() -> None:
             production_name = fn_productions[0].name
             expected_name = f"convert_{production_name}"
             if fn.__name__ != expected_name:
-                raise Exception(
+                raise ValueError(
                     f"The conversion function for '{production_name}' "
                     + f"must be called '{expected_name}', not '{fn.__name__}'."
                 )
@@ -330,7 +330,7 @@ def _get_version_comparison(version: str) -> Tuple[str, PythonVersionInfo]:
         return (version[:2], parse_version_string(version[2:].strip()))
     if version[:1] in (">", "<"):
         return (version[:1], parse_version_string(version[1:].strip()))
-    raise Exception(f"Invalid version comparison specifier '{version}'")
+    raise ValueError(f"Invalid version comparison specifier '{version}'")
 
 
 def _compare_versions(
@@ -350,7 +350,7 @@ def _compare_versions(
         return actual_version > requested_version
     if comparison == "<":
         return actual_version < requested_version
-    raise Exception(f"Invalid version comparison specifier '{comparison}'")
+    raise ValueError(f"Invalid version comparison specifier '{comparison}'")
 
 
 def _should_include(
@@ -405,7 +405,7 @@ def get_nonterminal_conversions(
             if not _should_include_future(fn_production.future, future_imports):
                 continue
             if fn_production.name in conversions:
-                raise Exception(
+                raise ValueError(
                     f"Found duplicate '{fn_production.name}' production in grammar"
                 )
             conversions[fn_production.name] = fn

libcst/_parser/parso/pgen2/generator.py

@@ -72,9 +72,9 @@ class DFAState(Generic[_TokenTypeT]):
     def __init__(self, from_rule: str, nfa_set: Set[NFAState], final: NFAState) -> None:
         self.from_rule = from_rule
         self.nfa_set = nfa_set
-        self.arcs: Mapping[
-            str, DFAState
-        ] = {}  # map from terminals/nonterminals to DFAState
+        self.arcs: Mapping[str, DFAState] = (
+            {}
+        )  # map from terminals/nonterminals to DFAState
         # In an intermediary step we set these nonterminal arcs (which has the
         # same structure as arcs). These don't contain terminals anymore.
         self.nonterminal_arcs: Mapping[str, DFAState] = {}
@@ -259,7 +259,7 @@ def generate_grammar(bnf_grammar: str, token_namespace: Any) -> Grammar[Any]:
 
     _calculate_tree_traversal(rule_to_dfas)
     if start_nonterminal is None:
-        raise Exception("could not find starting nonterminal!")
+        raise ValueError("could not find starting nonterminal!")
     return Grammar(start_nonterminal, rule_to_dfas, reserved_strings)
 
 

libcst/_parser/parso/python/token.py

@@ -27,7 +27,7 @@ try:
         ERROR_DEDENT: TokenType = native_token_type.ERROR_DEDENT
 
 except ImportError:
-    from libcst._parser.parso.python.py_token import (  # noqa F401
+    from libcst._parser.parso.python.py_token import (  # noqa: F401
         PythonTokenTypes,
         TokenType,
     )

libcst/_parser/parso/python/tokenize.py

@@ -36,6 +36,7 @@ from collections import namedtuple
 from dataclasses import dataclass
 from typing import Dict, Generator, Iterable, Optional, Pattern, Set, Tuple
 
+from libcst import CSTLogicError
 from libcst._parser.parso.python.token import PythonTokenTypes
 from libcst._parser.parso.utils import PythonVersionInfo, split_lines
 
@@ -522,14 +523,14 @@ def _tokenize_lines_py36_or_below(  # noqa: C901
 
         if contstr:  # continued string
             if endprog is None:
-                raise Exception("Logic error!")
+                raise CSTLogicError("Logic error!")
             endmatch = endprog.match(line)
             if endmatch:
                 pos = endmatch.end(0)
                 if contstr_start is None:
-                    raise Exception("Logic error!")
+                    raise CSTLogicError("Logic error!")
                 if stashed is not None:
-                    raise Exception("Logic error!")
+                    raise CSTLogicError("Logic error!")
                 yield PythonToken(STRING, contstr + line[:pos], contstr_start, prefix)
                 contstr = ""
                 contline = None
@@ -547,7 +548,7 @@ def _tokenize_lines_py36_or_below(  # noqa: C901
                     )
                     if string:
                         if stashed is not None:
-                            raise Exception("Logic error!")
+                            raise CSTLogicError("Logic error!")
                         yield PythonToken(
                             FSTRING_STRING,
                             string,
@@ -572,7 +573,7 @@ def _tokenize_lines_py36_or_below(  # noqa: C901
                 pos += quote_length
                 if fstring_end_token is not None:
                     if stashed is not None:
-                        raise Exception("Logic error!")
+                        raise CSTLogicError("Logic error!")
                     yield fstring_end_token
                     continue
 
@@ -885,12 +886,12 @@ def _tokenize_lines_py37_or_above(  # noqa: C901
 
         if contstr:  # continued string
             if endprog is None:
-                raise Exception("Logic error!")
+                raise CSTLogicError("Logic error!")
             endmatch = endprog.match(line)
             if endmatch:
                 pos = endmatch.end(0)
                 if contstr_start is None:
-                    raise Exception("Logic error!")
+                    raise CSTLogicError("Logic error!")
                 yield PythonToken(STRING, contstr + line[:pos], contstr_start, prefix)
                 contstr = ""
                 contline = None

libcst/_parser/parso/tests/test_utils.py

@@ -39,8 +39,8 @@ class ParsoUtilsTest(UnitTest):
             # Invalid line breaks
             ("a\vb", ["a\vb"], False),
             ("a\vb", ["a\vb"], True),
-            ("\x1C", ["\x1C"], False),
-            ("\x1C", ["\x1C"], True),
+            ("\x1c", ["\x1c"], False),
+            ("\x1c", ["\x1c"], True),
         )
     )
     def test_split_lines(self, string, expected_result, keepends):

libcst/_parser/parso/utils.py

@@ -29,9 +29,9 @@ from typing import Optional, Sequence, Tuple, Union
 _NON_LINE_BREAKS = (
     "\v",  # Vertical Tabulation 0xB
     "\f",  # Form Feed 0xC
-    "\x1C",  # File Separator
-    "\x1D",  # Group Separator
-    "\x1E",  # Record Separator
+    "\x1c",  # File Separator
+    "\x1d",  # Group Separator
+    "\x1e",  # Record Separator
     "\x85",  # Next Line (NEL - Equivalent to CR+LF.
     # Used to mark end-of-line on some IBM mainframes.)
     "\u2028",  # Line Separator

libcst/_parser/production_decorator.py

@@ -39,7 +39,7 @@ def with_production(
         # pyre-ignore: Pyre doesn't think that fn has a __name__ attribute
         fn_name = fn.__name__
         if not fn_name.startswith("convert_"):
-            raise Exception(
+            raise ValueError(
                 "A function with a production must be named 'convert_X', not "
                 + f"'{fn_name}'."
             )

libcst/_parser/py_whitespace_parser.py

@@ -5,6 +5,7 @@
 
 from typing import List, Optional, Sequence, Tuple, Union
 
+from libcst import CSTLogicError, ParserSyntaxError
 from libcst._nodes.whitespace import (
     Comment,
     COMMENT_RE,
@@ -103,10 +104,13 @@ def parse_trailing_whitespace(
 ) -> TrailingWhitespace:
     trailing_whitespace = _parse_trailing_whitespace(config, state)
     if trailing_whitespace is None:
-        raise Exception(
+        raise ParserSyntaxError(
             "Internal Error: Failed to parse TrailingWhitespace. This should never "
             + "happen because a TrailingWhitespace is never optional in the grammar, "
-            + "so this error should've been caught by parso first."
+            + "so this error should've been caught by parso first.",
+            lines=config.lines,
+            raw_line=state.line,
+            raw_column=state.column,
         )
     return trailing_whitespace
 
@@ -177,7 +181,9 @@ def _parse_indent(
         if state.column == len(line_str) and state.line == len(config.lines):
             # We're at EOF, treat this as a failed speculative parse
             return False
-        raise Exception("Internal Error: Column should be 0 when parsing an indent.")
+        raise CSTLogicError(
+            "Internal Error: Column should be 0 when parsing an indent."
+        )
     if line_str.startswith(absolute_indent, state.column):
         state.column += len(absolute_indent)
         return True
@@ -206,7 +212,12 @@ def _parse_newline(
         newline_str = newline_match.group(0)
         state.column += len(newline_str)
         if state.column != len(line_str):
-            raise Exception("Internal Error: Found a newline, but it wasn't the EOL.")
+            raise ParserSyntaxError(
+                "Internal Error: Found a newline, but it wasn't the EOL.",
+                lines=config.lines,
+                raw_line=state.line,
+                raw_column=state.column,
+            )
         if state.line < len(config.lines):
             # this newline was the end of a line, and there's another line,
             # therefore we should move to the next line

libcst/_parser/tests/test_parse_errors.py

@@ -10,7 +10,6 @@ from unittest.mock import patch
 
 import libcst as cst
 from libcst._nodes.base import CSTValidationError
-from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider, UnitTest
 
 
@@ -174,8 +173,6 @@ class ParseErrorsTest(UnitTest):
             parse_fn()
         # make sure str() doesn't blow up
         self.assertIn("Syntax Error", str(cm.exception))
-        if not is_native():
-            self.assertEqual(str(cm.exception), expected)
 
     def test_native_fallible_into_py(self) -> None:
         with patch("libcst._nodes.expression.Name._validate") as await_validate:

libcst/_parser/types/config.py

@@ -27,9 +27,9 @@ except ImportError:
 
 BaseWhitespaceParserConfig = config_mod.BaseWhitespaceParserConfig
 ParserConfig = config_mod.ParserConfig
-parser_config_asdict: Callable[
-    [ParserConfig], Mapping[str, Any]
-] = config_mod.parser_config_asdict
+parser_config_asdict: Callable[[ParserConfig], Mapping[str, Any]] = (
+    config_mod.parser_config_asdict
+)
 
 
 class AutoConfig(Enum):

libcst/_parser/types/token.py

@@ -9,4 +9,4 @@ try:
 
     Token = tokenize.Token
 except ImportError:
-    from libcst._parser.types.py_token import Token  # noqa F401
+    from libcst._parser.types.py_token import Token  # noqa: F401

libcst/_position.py

@@ -40,12 +40,10 @@ class CodeRange:
     end: CodePosition
 
     @overload
-    def __init__(self, start: CodePosition, end: CodePosition) -> None:
-        ...
+    def __init__(self, start: CodePosition, end: CodePosition) -> None: ...
 
     @overload
-    def __init__(self, start: Tuple[int, int], end: Tuple[int, int]) -> None:
-        ...
+    def __init__(self, start: Tuple[int, int], end: Tuple[int, int]) -> None: ...
 
     def __init__(self, start: _CodePositionT, end: _CodePositionT) -> None:
         if isinstance(start, tuple) and isinstance(end, tuple):

libcst/codegen/gen_matcher_classes.py

@@ -8,7 +8,7 @@ from dataclasses import dataclass, fields
 from typing import Generator, List, Optional, Sequence, Set, Tuple, Type, Union
 
 import libcst as cst
-from libcst import ensure_type, parse_expression
+from libcst import CSTLogicError, ensure_type, parse_expression
 from libcst.codegen.gather import all_libcst_nodes, typeclasses
 
 CST_DIR: Set[str] = set(dir(cst))
@@ -16,6 +16,109 @@ CLASS_RE = r"<class \'(.*?)\'>"
 OPTIONAL_RE = r"typing\.Union\[([^,]*?), NoneType]"
 
 
+class NormalizeUnions(cst.CSTTransformer):
+    """
+    Convert a binary operation with | operators into a Union type.
+    For example, converts `foo | bar | baz` into `typing.Union[foo, bar, baz]`.
+    Special case: converts `foo | None` or `None | foo` into `typing.Optional[foo]`.
+    Also flattens nested typing.Union types.
+    """
+
+    def leave_Subscript(
+        self, original_node: cst.Subscript, updated_node: cst.Subscript
+    ) -> cst.Subscript:
+        # Check if this is a typing.Union
+        if (
+            isinstance(updated_node.value, cst.Attribute)
+            and isinstance(updated_node.value.value, cst.Name)
+            and updated_node.value.attr.value == "Union"
+            and updated_node.value.value.value == "typing"
+        ):
+            # Collect all operands from any nested Unions
+            operands: List[cst.BaseExpression] = []
+            for slc in updated_node.slice:
+                if not isinstance(slc.slice, cst.Index):
+                    continue
+                value = slc.slice.value
+                # If this is a nested Union, add its elements
+                if (
+                    isinstance(value, cst.Subscript)
+                    and isinstance(value.value, cst.Attribute)
+                    and isinstance(value.value.value, cst.Name)
+                    and value.value.attr.value == "Union"
+                    and value.value.value.value == "typing"
+                ):
+                    operands.extend(
+                        nested_slc.slice.value
+                        for nested_slc in value.slice
+                        if isinstance(nested_slc.slice, cst.Index)
+                    )
+                else:
+                    operands.append(value)
+
+            # flatten operands into a Union type
+            return cst.Subscript(
+                cst.Attribute(cst.Name("typing"), cst.Name("Union")),
+                [cst.SubscriptElement(cst.Index(operand)) for operand in operands],
+            )
+        return updated_node
+
+    def leave_BinaryOperation(
+        self, original_node: cst.BinaryOperation, updated_node: cst.BinaryOperation
+    ) -> Union[cst.BinaryOperation, cst.Subscript]:
+        if not updated_node.operator.deep_equals(cst.BitOr()):
+            return updated_node
+
+        def flatten_binary_op(node: cst.BaseExpression) -> List[cst.BaseExpression]:
+            """Flatten a binary operation tree into a list of operands."""
+            if not isinstance(node, cst.BinaryOperation):
+                # If it's a Union type, extract its elements
+                if (
+                    isinstance(node, cst.Subscript)
+                    and isinstance(node.value, cst.Attribute)
+                    and isinstance(node.value.value, cst.Name)
+                    and node.value.attr.value == "Union"
+                    and node.value.value.value == "typing"
+                ):
+                    return [
+                        slc.slice.value
+                        for slc in node.slice
+                        if isinstance(slc.slice, cst.Index)
+                    ]
+                return [node]
+            if not node.operator.deep_equals(cst.BitOr()):
+                return [node]
+
+            left_operands = flatten_binary_op(node.left)
+            right_operands = flatten_binary_op(node.right)
+            return left_operands + right_operands
+
+        # Flatten the binary operation tree into a list of operands
+        operands = flatten_binary_op(updated_node)
+
+        # Check for Optional case (None in union)
+        none_count = sum(
+            1 for op in operands if isinstance(op, cst.Name) and op.value == "None"
+        )
+        if none_count == 1 and len(operands) == 2:
+            # This is an Optional case - find the non-None operand
+            non_none = next(
+                op
+                for op in operands
+                if not (isinstance(op, cst.Name) and op.value == "None")
+            )
+            return cst.Subscript(
+                cst.Attribute(cst.Name("typing"), cst.Name("Optional")),
+                [cst.SubscriptElement(cst.Index(non_none))],
+            )
+
+        # Regular Union case
+        return cst.Subscript(
+            cst.Attribute(cst.Name("typing"), cst.Name("Union")),
+            [cst.SubscriptElement(cst.Index(operand)) for operand in operands],
+        )
+
+
 class CleanseFullTypeNames(cst.CSTTransformer):
     def leave_Call(
         self, original_node: cst.Call, updated_node: cst.Call
@@ -180,9 +283,9 @@ class AddWildcardsToSequenceUnions(cst.CSTTransformer):
                 # type blocks, even for sequence types.
                 return
             if len(node.slice) != 1:
-                raise Exception(
+                raise ValueError(
                     "Unexpected number of sequence elements inside Sequence type "
-                    + "annotation!"
+                    "annotation!"
                 )
             nodeslice = node.slice[0].slice
             if isinstance(nodeslice, cst.Index):
@@ -346,10 +449,14 @@ def _get_clean_type_from_subscript(
     if typecst.value.deep_equals(cst.Name("Sequence")):
         # Lets attempt to widen the sequence type and alias it.
         if len(typecst.slice) != 1:
-            raise Exception("Logic error, Sequence shouldn't have more than one param!")
+            raise CSTLogicError(
+                "Logic error, Sequence shouldn't have more than one param!"
+            )
         inner_type = typecst.slice[0].slice
         if not isinstance(inner_type, cst.Index):
-            raise Exception("Logic error, expecting Index for only Sequence element!")
+            raise CSTLogicError(
+                "Logic error, expecting Index for only Sequence element!"
+            )
         inner_type = inner_type.value
 
         if isinstance(inner_type, cst.Subscript):
@@ -357,7 +464,9 @@ def _get_clean_type_from_subscript(
         elif isinstance(inner_type, (cst.Name, cst.SimpleString)):
             clean_inner_type = _get_clean_type_from_expression(aliases, inner_type)
         else:
-            raise Exception("Logic error, unexpected type in Sequence!")
+            raise CSTLogicError(
+                f"Logic error, unexpected type in Sequence: {type(inner_type)}!"
+            )
 
         return _get_wrapped_union_type(
             typecst.deep_replace(inner_type, clean_inner_type),
@@ -386,9 +495,12 @@ def _get_clean_type_and_aliases(
     typestr = re.sub(OPTIONAL_RE, r"typing.Optional[\1]", typestr)
 
     # Now, parse the expression with LibCST.
-    cleanser = CleanseFullTypeNames()
+
     typecst = parse_expression(typestr)
-    typecst = typecst.visit(cleanser)
+    typecst = typecst.visit(NormalizeUnions())
+    assert isinstance(typecst, cst.BaseExpression)
+    typecst = typecst.visit(CleanseFullTypeNames())
+    assert isinstance(typecst, cst.BaseExpression)
     aliases: List[Alias] = []
 
     # Now, convert the type to allow for MetadataMatchType and MatchIfTrue values.
@@ -397,7 +509,7 @@ def _get_clean_type_and_aliases(
     elif isinstance(typecst, (cst.Name, cst.SimpleString)):
         clean_type = _get_clean_type_from_expression(aliases, typecst)
     else:
-        raise Exception("Logic error, unexpected top level type!")
+        raise CSTLogicError(f"Logic error, unexpected top level type: {type(typecst)}!")
 
     # Now, insert OneOf/AllOf and MatchIfTrue into unions so we can typecheck their usage.
     # This allows us to put OneOf[SomeType] or MatchIfTrue[cst.SomeType] into any

libcst/codegen/tests/test_codegen_clean.py

@@ -3,6 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+import difflib
 import os
 import os.path
 
@@ -20,12 +21,20 @@ class TestCodegenClean(UnitTest):
         new_code: str,
         module_name: str,
     ) -> None:
-        self.assertTrue(
-            old_code == new_code,
-            f"{module_name} needs new codegen, see "
-            + "`python -m libcst.codegen.generate --help` "
-            + "for instructions, or run `python -m libcst.codegen.generate all`",
-        )
+        if old_code != new_code:
+            diff = difflib.unified_diff(
+                old_code.splitlines(keepends=True),
+                new_code.splitlines(keepends=True),
+                fromfile="old_code",
+                tofile="new_code",
+            )
+            diff_str = "".join(diff)
+            self.fail(
+                f"{module_name} needs new codegen, see "
+                + "`python -m libcst.codegen.generate --help` "
+                + "for instructions, or run `python -m libcst.codegen.generate all`. "
+                + f"Diff:\n{diff_str}"
+            )
 
     def test_codegen_clean_visitor_functions(self) -> None:
         """
@@ -123,3 +132,50 @@ class TestCodegenClean(UnitTest):
 
         # Now that we've done simple codegen, verify that it matches.
         self.assert_code_matches(old_code, new_code, "libcst.matchers._return_types")
+
+    def test_normalize_unions(self) -> None:
+        """
+        Verifies that NormalizeUnions correctly converts binary operations with |
+        into Union types, with special handling for Optional cases.
+        """
+        import libcst as cst
+        from libcst.codegen.gen_matcher_classes import NormalizeUnions
+
+        def assert_transforms_to(input_code: str, expected_code: str) -> None:
+            input_cst = cst.parse_expression(input_code)
+            expected_cst = cst.parse_expression(expected_code)
+
+            result = input_cst.visit(NormalizeUnions())
+            assert isinstance(
+                result, cst.BaseExpression
+            ), f"Expected BaseExpression, got {type(result)}"
+
+            result_code = cst.Module(body=()).code_for_node(result)
+            expected_code_str = cst.Module(body=()).code_for_node(expected_cst)
+
+            self.assertEqual(
+                result_code,
+                expected_code_str,
+                f"Expected {expected_code_str}, got {result_code}",
+            )
+
+        # Test regular union case
+        assert_transforms_to("foo | bar | baz", "typing.Union[foo, bar, baz]")
+
+        # Test Optional case (None on right)
+        assert_transforms_to("foo | None", "typing.Optional[foo]")
+
+        # Test Optional case (None on left)
+        assert_transforms_to("None | foo", "typing.Optional[foo]")
+
+        # Test case with more than 2 operands including None (should remain Union)
+        assert_transforms_to("foo | bar | None", "typing.Union[foo, bar, None]")
+
+        # Flatten existing Union types
+        assert_transforms_to(
+            "typing.Union[foo, typing.Union[bar, baz]]", "typing.Union[foo, bar, baz]"
+        )
+        # Merge two kinds of union types
+        assert_transforms_to(
+            "foo | typing.Union[bar, baz]", "typing.Union[foo, bar, baz]"
+        )

libcst/codemod/_cli.py

@@ -8,20 +8,25 @@ Provides helpers for CLI interaction.
 """
 
 import difflib
+import functools
 import os.path
 import re
 import subprocess
 import sys
 import time
 import traceback
-from dataclasses import dataclass, replace
-from multiprocessing import cpu_count, Pool
+from concurrent.futures import as_completed, Executor
+from copy import deepcopy
+from dataclasses import dataclass
+from multiprocessing import cpu_count
 from pathlib import Path
-from typing import Any, AnyStr, cast, Dict, List, Optional, Sequence, Union
+from typing import AnyStr, Callable, cast, Dict, List, Optional, Sequence, Type, Union
+from warnings import warn
 
 from libcst import parse_module, PartialParserConfig
 from libcst.codemod._codemod import Codemod
-from libcst.codemod._dummy_pool import DummyPool
+from libcst.codemod._context import CodemodContext
+from libcst.codemod._dummy_pool import DummyExecutor
 from libcst.codemod._runner import (
     SkipFile,
     SkipReason,
@@ -46,7 +51,7 @@ def invoke_formatter(formatter_args: Sequence[str], code: AnyStr) -> AnyStr:
 
     # Make sure there is something to run
     if len(formatter_args) == 0:
-        raise Exception("No formatter configured but code formatting requested.")
+        raise ValueError("No formatter configured but code formatting requested.")
 
     # Invoke the formatter, giving it the code as stdin and assuming the formatted
     # code comes from stdout.
@@ -210,11 +215,52 @@ class ExecutionConfig:
     unified_diff: Optional[int] = None
 
 
-def _execute_transform(  # noqa: C901
-    transformer: Codemod,
+def _prepare_context(
+    repo_root: str,
     filename: str,
-    config: ExecutionConfig,
-) -> ExecutionResult:
+    scratch: Dict[str, object],
+    repo_manager: Optional[FullRepoManager],
+) -> CodemodContext:
+    # determine the module and package name for this file
+    try:
+        module_name_and_package = calculate_module_and_package(repo_root, filename)
+        mod_name = module_name_and_package.name
+        pkg_name = module_name_and_package.package
+    except ValueError as ex:
+        print(f"Failed to determine module name for {filename}: {ex}", file=sys.stderr)
+        mod_name = None
+        pkg_name = None
+    return CodemodContext(
+        scratch=scratch,
+        filename=filename,
+        full_module_name=mod_name,
+        full_package_name=pkg_name,
+        metadata_manager=repo_manager,
+    )
+
+
+def _instantiate_transformer(
+    transformer: Union[Codemod, Type[Codemod]],
+    repo_root: str,
+    filename: str,
+    original_scratch: Dict[str, object],
+    codemod_kwargs: Dict[str, object],
+    repo_manager: Optional[FullRepoManager],
+) -> Codemod:
+    if isinstance(transformer, type):
+        return transformer(  # type: ignore
+            context=_prepare_context(repo_root, filename, {}, repo_manager),
+            **codemod_kwargs,
+        )
+    transformer.context = _prepare_context(
+        repo_root, filename, deepcopy(original_scratch), repo_manager
+    )
+    return transformer
+
+
+def _check_for_skip(
+    filename: str, config: ExecutionConfig
+) -> Union[ExecutionResult, bytes]:
     for pattern in config.blacklist_patterns:
         if re.fullmatch(pattern, filename):
             return ExecutionResult(
@@ -226,48 +272,47 @@ def _execute_transform(  # noqa: C901
                 ),
             )
 
-    try:
-        with open(filename, "rb") as fp:
-            oldcode = fp.read()
+    with open(filename, "rb") as fp:
+        oldcode = fp.read()
 
-        # Skip generated files
-        if (
-            not config.include_generated
-            and config.generated_code_marker.encode("utf-8") in oldcode
-        ):
-            return ExecutionResult(
-                filename=filename,
-                changed=False,
-                transform_result=TransformSkip(
-                    skip_reason=SkipReason.GENERATED,
-                    skip_description="Generated file.",
-                ),
-            )
-
-        # Somewhat gross hack to provide the filename in the transform's context.
-        # We do this after the fork so that a context that was initialized with
-        # some defaults before calling parallel_exec_transform_with_prettyprint
-        # will be updated per-file.
-        transformer.context = replace(
-            transformer.context,
+    # Skip generated files
+    if (
+        not config.include_generated
+        and config.generated_code_marker.encode("utf-8") in oldcode
+    ):
+        return ExecutionResult(
             filename=filename,
-            scratch={},
+            changed=False,
+            transform_result=TransformSkip(
+                skip_reason=SkipReason.GENERATED,
+                skip_description="Generated file.",
+            ),
         )
+    return oldcode
 
-        # determine the module and package name for this file
-        try:
-            module_name_and_package = calculate_module_and_package(
-                config.repo_root or ".", filename
-            )
-            transformer.context = replace(
-                transformer.context,
-                full_module_name=module_name_and_package.name,
-                full_package_name=module_name_and_package.package,
-            )
-        except ValueError as ex:
-            print(
-                f"Failed to determine module name for {filename}: {ex}", file=sys.stderr
-            )
+
+def _execute_transform(
+    transformer: Union[Codemod, Type[Codemod]],
+    filename: str,
+    config: ExecutionConfig,
+    original_scratch: Dict[str, object],
+    codemod_args: Optional[Dict[str, object]],
+    repo_manager: Optional[FullRepoManager],
+) -> ExecutionResult:
+    warnings: list[str] = []
+    try:
+        oldcode = _check_for_skip(filename, config)
+        if isinstance(oldcode, ExecutionResult):
+            return oldcode
+
+        transformer_instance = _instantiate_transformer(
+            transformer,
+            config.repo_root or ".",
+            filename,
+            original_scratch,
+            codemod_args or {},
+            repo_manager,
+        )
 
         # Run the transform, bail if we failed or if we aren't formatting code
         try:
@@ -279,55 +324,26 @@ def _execute_transform(  # noqa: C901
                     else PartialParserConfig()
                 ),
             )
-            output_tree = transformer.transform_module(input_tree)
+            output_tree = transformer_instance.transform_module(input_tree)
             newcode = output_tree.bytes
             encoding = output_tree.encoding
-        except KeyboardInterrupt:
-            return ExecutionResult(
-                filename=filename, changed=False, transform_result=TransformExit()
-            )
+            warnings.extend(transformer_instance.context.warnings)
         except SkipFile as ex:
+            warnings.extend(transformer_instance.context.warnings)
             return ExecutionResult(
                 filename=filename,
                 changed=False,
                 transform_result=TransformSkip(
                     skip_reason=SkipReason.OTHER,
                     skip_description=str(ex),
-                    warning_messages=transformer.context.warnings,
-                ),
-            )
-        except Exception as ex:
-            return ExecutionResult(
-                filename=filename,
-                changed=False,
-                transform_result=TransformFailure(
-                    error=ex,
-                    traceback_str=traceback.format_exc(),
-                    warning_messages=transformer.context.warnings,
+                    warning_messages=warnings,
                 ),
             )
 
         # Call formatter if needed, but only if we actually changed something in this
         # file
         if config.format_code and newcode != oldcode:
-            try:
-                newcode = invoke_formatter(config.formatter_args, newcode)
-            except KeyboardInterrupt:
-                return ExecutionResult(
-                    filename=filename,
-                    changed=False,
-                    transform_result=TransformExit(),
-                )
-            except Exception as ex:
-                return ExecutionResult(
-                    filename=filename,
-                    changed=False,
-                    transform_result=TransformFailure(
-                        error=ex,
-                        traceback_str=traceback.format_exc(),
-                        warning_messages=transformer.context.warnings,
-                    ),
-                )
+            newcode = invoke_formatter(config.formatter_args, newcode)
 
         # Format as unified diff if needed, otherwise save it back
         changed = oldcode != newcode
@@ -350,13 +366,14 @@ def _execute_transform(  # noqa: C901
         return ExecutionResult(
             filename=filename,
             changed=changed,
-            transform_result=TransformSuccess(
-                warning_messages=transformer.context.warnings, code=newcode
-            ),
+            transform_result=TransformSuccess(warning_messages=warnings, code=newcode),
         )
+
     except KeyboardInterrupt:
         return ExecutionResult(
-            filename=filename, changed=False, transform_result=TransformExit()
+            filename=filename,
+            changed=False,
+            transform_result=TransformExit(warning_messages=warnings),
         )
     except Exception as ex:
         return ExecutionResult(
@@ -365,7 +382,7 @@ def _execute_transform(  # noqa: C901
             transform_result=TransformFailure(
                 error=ex,
                 traceback_str=traceback.format_exc(),
-                warning_messages=transformer.context.warnings,
+                warning_messages=warnings,
             ),
         )
 
@@ -418,7 +435,7 @@ class Progress:
         operations still to do.
         """
 
-        if files_finished <= 0:
+        if files_finished <= 0 or elapsed_seconds == 0:
             # Technically infinite but calculating sounds better.
             return "[calculating]"
 
@@ -502,15 +519,8 @@ class ParallelTransformResult:
     skips: int
 
 
-# Unfortunate wrapper required since there is no `istarmap_unordered`...
-def _execute_transform_wrap(
-    job: Dict[str, Any],
-) -> ExecutionResult:
-    return _execute_transform(**job)
-
-
 def parallel_exec_transform_with_prettyprint(  # noqa: C901
-    transform: Codemod,
+    transform: Union[Codemod, Type[Codemod]],
     files: Sequence[str],
     *,
     jobs: Optional[int] = None,
@@ -526,38 +536,49 @@ def parallel_exec_transform_with_prettyprint(  # noqa: C901
     blacklist_patterns: Sequence[str] = (),
     python_version: Optional[str] = None,
     repo_root: Optional[str] = None,
+    codemod_args: Optional[Dict[str, object]] = None,
 ) -> ParallelTransformResult:
     """
-    Given a list of files and an instantiated codemod we should apply to them,
-    fork and apply the codemod in parallel to all of the files, including any
-    configured formatter. The ``jobs`` parameter controls the maximum number of
-    in-flight transforms, and needs to be at least 1. If not included, the number
-    of jobs will automatically be set to the number of CPU cores. If ``unified_diff``
-    is set to a number, changes to files will be printed to stdout with
-    ``unified_diff`` lines of context. If it is set to ``None`` or left out, files
-    themselves will be updated with changes and formatting. If a
-    ``python_version`` is provided, then we will parse each source file using
-    this version. Otherwise, we will use the version of the currently executing python
+    Given a list of files and a codemod we should apply to them, fork and apply the
+    codemod in parallel to all of the files, including any configured formatter. The
+    ``jobs`` parameter controls the maximum number of in-flight transforms, and needs to
+    be at least 1. If not included, the number of jobs will automatically be set to the
+    number of CPU cores. If ``unified_diff`` is set to a number, changes to files will
+    be printed to stdout with ``unified_diff`` lines of context. If it is set to
+    ``None`` or left out, files themselves will be updated with changes and formatting.
+    If a ``python_version`` is provided, then we will parse each source file using this
+    version. Otherwise, we will use the version of the currently executing python
     binary.
 
-    A progress indicator as well as any generated warnings will be printed to stderr.
-    To supress the interactive progress indicator, set ``hide_progress`` to ``True``.
-    Files that include the generated code marker will be skipped unless the
-    ``include_generated`` parameter is set to ``True``. Similarly, files that match
-    a supplied blacklist of regex patterns will be skipped. Warnings for skipping
-    both blacklisted and generated files will be printed to stderr along with
-    warnings generated by the codemod unless ``hide_blacklisted`` and
-    ``hide_generated`` are set to ``True``. Files that were successfully codemodded
-    will not be printed to stderr unless ``show_successes`` is set to ``True``.
+    A progress indicator as well as any generated warnings will be printed to stderr. To
+    supress the interactive progress indicator, set ``hide_progress`` to ``True``. Files
+    that include the generated code marker will be skipped unless the
+    ``include_generated`` parameter is set to ``True``. Similarly, files that match a
+    supplied blacklist of regex patterns will be skipped. Warnings for skipping both
+    blacklisted and generated files will be printed to stderr along with warnings
+    generated by the codemod unless ``hide_blacklisted`` and ``hide_generated`` are set
+    to ``True``. Files that were successfully codemodded will not be printed to stderr
+    unless ``show_successes`` is set to ``True``.
 
-    To make this API possible, we take an instantiated transform. This is due to
-    the fact that lambdas are not pickleable and pickling functions is undefined.
-    This means we're implicitly relying on fork behavior on UNIX-like systems, and
-    this function will not work on Windows systems. To create a command-line utility
-    that runs on Windows, please instead see
-    :func:`~libcst.codemod.exec_transform_with_prettyprint`.
+    We take a :class:`~libcst.codemod._codemod.Codemod` class, or an instantiated
+    :class:`~libcst.codemod._codemod.Codemod`. In the former case, the codemod will be
+    instantiated for each file, with ``codemod_args`` passed in to the constructor.
+    Passing an already instantiated :class:`~libcst.codemod._codemod.Codemod` is
+    deprecated, because it leads to sharing of the
+    :class:`~libcst.codemod._codemod.Codemod` instance across files, which is a common
+    source of hard-to-track-down bugs when the :class:`~libcst.codemod._codemod.Codemod`
+    tracks its state on the instance.
     """
 
+    if isinstance(transform, Codemod):
+        warn(
+            "Passing transformer instances to `parallel_exec_transform_with_prettyprint` "
+            "is deprecated and will break in a future version. "
+            "Please pass the transformer class instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
     # Ensure that we have no duplicates, otherwise we might get race conditions
     # on write.
     files = sorted({os.path.abspath(f) for f in files})
@@ -572,11 +593,12 @@ def parallel_exec_transform_with_prettyprint(  # noqa: C901
     )
 
     if jobs < 1:
-        raise Exception("Must have at least one job to process!")
+        raise ValueError("Must have at least one job to process!")
 
     if total == 0:
         return ParallelTransformResult(successes=0, failures=0, skips=0, warnings=0)
 
+    metadata_manager: Optional[FullRepoManager] = None
     if repo_root is not None:
         # Make sure if there is a root that we have the absolute path to it.
         repo_root = os.path.abspath(repo_root)
@@ -589,10 +611,7 @@ def parallel_exec_transform_with_prettyprint(  # noqa: C901
             transform.get_inherited_dependencies(),
         )
         metadata_manager.resolve_cache()
-        transform.context = replace(
-            transform.context,
-            metadata_manager=metadata_manager,
-        )
+
     print("Executing codemod...", file=sys.stderr)
 
     config = ExecutionConfig(
@@ -606,13 +625,16 @@ def parallel_exec_transform_with_prettyprint(  # noqa: C901
         python_version=python_version,
     )
 
+    pool_impl: Callable[[], Executor]
     if total == 1 or jobs == 1:
         # Simple case, we should not pay for process overhead.
-        # Let's just use a dummy synchronous pool.
+        # Let's just use a dummy synchronous executor.
         jobs = 1
-        pool_impl = DummyPool
-    else:
-        pool_impl = Pool
+        pool_impl = DummyExecutor
+    elif getattr(sys, "_is_gil_enabled", lambda: True)():  # pyre-ignore[16]
+        from concurrent.futures import ProcessPoolExecutor
+
+        pool_impl = functools.partial(ProcessPoolExecutor, max_workers=jobs)
         # Warm the parser, pre-fork.
         parse_module(
             "",
@@ -622,25 +644,35 @@ def parallel_exec_transform_with_prettyprint(  # noqa: C901
                 else PartialParserConfig()
             ),
         )
+    else:
+        from concurrent.futures import ThreadPoolExecutor
+
+        pool_impl = functools.partial(ThreadPoolExecutor, max_workers=jobs)
 
     successes: int = 0
     failures: int = 0
     warnings: int = 0
     skips: int = 0
+    original_scratch = (
+        deepcopy(transform.context.scratch) if isinstance(transform, Codemod) else {}
+    )
 
-    with pool_impl(processes=jobs) as p:  # type: ignore
-        args = [
-            {
-                "transformer": transform,
-                "filename": filename,
-                "config": config,
-            }
-            for filename in files
-        ]
+    with pool_impl() as executor:  # type: ignore
         try:
-            for result in p.imap_unordered(
-                _execute_transform_wrap, args, chunksize=chunksize
-            ):
+            futures = [
+                executor.submit(
+                    _execute_transform,
+                    transformer=transform,
+                    filename=filename,
+                    config=config,
+                    original_scratch=original_scratch,
+                    codemod_args=codemod_args,
+                    repo_manager=metadata_manager,
+                )
+                for filename in files
+            ]
+            for future in as_completed(futures):
+                result = future.result()
                 # Print an execution result, keep track of failures
                 _print_parallel_result(
                     result,

libcst/codemod/_codemod.py

@@ -56,9 +56,9 @@ class Codemod(MetadataDependent, ABC):
         """
         module = self.context.module
         if module is None:
-            raise Exception(
+            raise ValueError(
                 f"Attempted access of {self.__class__.__name__}.module outside of "
-                + "transform_module()."
+                "transform_module()."
             )
         return module
 

libcst/codemod/_command.py

@@ -3,12 +3,14 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 #
+from __future__ import annotations
+
 import argparse
 import inspect
 from abc import ABC, abstractmethod
-from typing import Dict, Generator, List, Type, TypeVar
+from typing import Dict, Generator, List, Tuple, Type, TypeVar
 
-from libcst import Module
+from libcst import CSTNode, Module
 from libcst.codemod._codemod import Codemod
 from libcst.codemod._context import CodemodContext
 from libcst.codemod._visitor import ContextAwareTransformer
@@ -65,6 +67,28 @@ class CodemodCommand(Codemod, ABC):
         """
         ...
 
+    # Lightweight wrappers for RemoveImportsVisitor static functions
+    def remove_unused_import(
+        self,
+        module: str,
+        obj: str | None = None,
+        asname: str | None = None,
+    ) -> None:
+        RemoveImportsVisitor.remove_unused_import(self.context, module, obj, asname)
+
+    def remove_unused_import_by_node(self, node: CSTNode) -> None:
+        RemoveImportsVisitor.remove_unused_import_by_node(self.context, node)
+
+    # Lightweight wrappers for AddImportsVisitor static functions
+    def add_needed_import(
+        self,
+        module: str,
+        obj: str | None = None,
+        asname: str | None = None,
+        relative: int = 0,
+    ) -> None:
+        AddImportsVisitor.add_needed_import(self.context, module, obj, asname, relative)
+
     def transform_module(self, tree: Module) -> Module:
         # Overrides (but then calls) Codemod's transform_module to provide
         # a spot where additional supported transforms can be attached and run.
@@ -75,13 +99,13 @@ class CodemodCommand(Codemod, ABC):
         # have a static method that other transforms can use which takes
         # a context and other optional args and modifies its own context key
         # accordingly. We import them here so that we don't have circular imports.
-        supported_transforms: Dict[str, Type[Codemod]] = {
-            AddImportsVisitor.CONTEXT_KEY: AddImportsVisitor,
-            RemoveImportsVisitor.CONTEXT_KEY: RemoveImportsVisitor,
-        }
+        supported_transforms: List[Tuple[str, Type[Codemod]]] = [
+            (AddImportsVisitor.CONTEXT_KEY, AddImportsVisitor),
+            (RemoveImportsVisitor.CONTEXT_KEY, RemoveImportsVisitor),
+        ]
 
         # For any visitors that we support auto-running, run them here if needed.
-        for key, transform in supported_transforms.items():
+        for key, transform in supported_transforms:
             if key in self.context.scratch:
                 # We have work to do, so lets run this.
                 tree = self._instantiate_and_run(transform, tree)

libcst/codemod/_dummy_pool.py

@@ -3,37 +3,47 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+import sys
+from concurrent.futures import Executor, Future
 from types import TracebackType
-from typing import Callable, Generator, Iterable, Optional, Type, TypeVar
+from typing import Callable, Optional, Type, TypeVar
 
-RetT = TypeVar("RetT")
-ArgT = TypeVar("ArgT")
+if sys.version_info >= (3, 10):
+    from typing import ParamSpec
+else:
+    from typing_extensions import ParamSpec
+
+Return = TypeVar("Return")
+Params = ParamSpec("Params")
 
 
-class DummyPool:
+class DummyExecutor(Executor):
     """
-    Synchronous dummy `multiprocessing.Pool` analogue.
+    Synchronous dummy `concurrent.futures.Executor` analogue.
     """
 
-    def __init__(self, processes: Optional[int] = None) -> None:
-        pass
-
-    def imap_unordered(
+    def submit(
         self,
-        func: Callable[[ArgT], RetT],
-        iterable: Iterable[ArgT],
-        chunksize: Optional[int] = None,
-    ) -> Generator[RetT, None, None]:
-        for args in iterable:
-            yield func(args)
+        fn: Callable[Params, Return],
+        /,
+        *args: Params.args,
+        **kwargs: Params.kwargs,
+    ) -> Future[Return]:
+        future: Future[Return] = Future()
+        try:
+            result = fn(*args, **kwargs)
+            future.set_result(result)
+        except Exception as exc:
+            future.set_exception(exc)
+        return future
 
-    def __enter__(self) -> "DummyPool":
+    def __enter__(self) -> "DummyExecutor":
         return self
 
     def __exit__(
         self,
-        exc_type: Optional[Type[Exception]],
-        exc: Optional[Exception],
-        tb: Optional[TracebackType],
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
     ) -> None:
         pass

libcst/codemod/_visitor.py

@@ -6,7 +6,7 @@
 from typing import Mapping
 
 import libcst as cst
-from libcst import MetadataDependent
+from libcst import MetadataDependent, MetadataException
 from libcst.codemod._codemod import Codemod
 from libcst.codemod._context import CodemodContext
 from libcst.matchers import MatcherDecoratableTransformer, MatcherDecoratableVisitor
@@ -69,14 +69,14 @@ class ContextAwareVisitor(MatcherDecoratableVisitor, MetadataDependent):
         if dependencies:
             wrapper = self.context.wrapper
             if wrapper is None:
-                raise Exception(
+                raise MetadataException(
                     f"Attempting to instantiate {self.__class__.__name__} outside of "
                     + "an active transform. This means that metadata hasn't been "
                     + "calculated and we cannot successfully create this visitor."
                 )
             for dep in dependencies:
                 if dep not in wrapper._metadata:
-                    raise Exception(
+                    raise MetadataException(
                         f"Attempting to access metadata {dep.__name__} that was not a "
                         + "declared dependency of parent transform! This means it is "
                         + "not possible to compute this value. Please ensure that all "
@@ -101,7 +101,7 @@ class ContextAwareVisitor(MatcherDecoratableVisitor, MetadataDependent):
         """
         module = self.context.module
         if module is None:
-            raise Exception(
+            raise ValueError(
                 f"Attempted access of {self.__class__.__name__}.module outside of "
                 + "transform_module()."
             )

libcst/codemod/commands/convert_format_to_fstring.py

@@ -9,6 +9,8 @@ from typing import Generator, List, Optional, Sequence, Set, Tuple
 
 import libcst as cst
 import libcst.matchers as m
+from libcst import CSTLogicError
+from libcst._exceptions import ParserSyntaxError
 from libcst.codemod import (
     CodemodContext,
     ContextAwareTransformer,
@@ -23,7 +25,7 @@ def _get_lhs(field: cst.BaseExpression) -> cst.BaseExpression:
     elif isinstance(field, (cst.Attribute, cst.Subscript)):
         return _get_lhs(field.value)
     else:
-        raise Exception("Unsupported node type!")
+        raise TypeError("Unsupported node type!")
 
 
 def _find_expr_from_field_name(
@@ -48,7 +50,7 @@ def _find_expr_from_field_name(
     if isinstance(lhs, cst.Integer):
         index = int(lhs.value)
         if index < 0 or index >= len(args):
-            raise Exception(f"Logic error, arg sequence {index} out of bounds!")
+            raise CSTLogicError(f"Logic error, arg sequence {index} out of bounds!")
     elif isinstance(lhs, cst.Name):
         for i, arg in enumerate(args):
             kw = arg.keyword
@@ -58,10 +60,12 @@ def _find_expr_from_field_name(
                 index = i
                 break
         if index is None:
-            raise Exception(f"Logic error, arg name {lhs.value} out of bounds!")
+            raise CSTLogicError(f"Logic error, arg name {lhs.value} out of bounds!")
 
     if index is None:
-        raise Exception(f"Logic error, unsupported fieldname expression {fieldname}!")
+        raise CSTLogicError(
+            f"Logic error, unsupported fieldname expression {fieldname}!"
+        )
 
     # Format it!
     return field_expr.deep_replace(lhs, args[index].value)
@@ -141,7 +145,7 @@ def _get_tokens(  # noqa: C901
                 in_brackets -= 1
 
                 if in_brackets < 0:
-                    raise Exception("Stray } in format string!")
+                    raise ValueError("Stray } in format string!")
 
                 if in_brackets == 0:
                     field_name, format_spec, conversion = _get_field(format_accum)
@@ -158,9 +162,11 @@ def _get_tokens(  # noqa: C901
             format_accum += char
 
     if in_brackets > 0:
-        raise Exception("Stray { in format string!")
+        raise ParserSyntaxError(
+            "Stray { in format string!", lines=[string], raw_line=0, raw_column=0
+        )
     if format_accum:
-        raise Exception("Logic error!")
+        raise CSTLogicError("Logic error!")
 
     # Yield the last bit of information
     yield (prefix, None, None, None)
@@ -188,7 +194,7 @@ class SwitchStringQuotesTransformer(ContextAwareTransformer):
     def __init__(self, context: CodemodContext, avoid_quote: str) -> None:
         super().__init__(context)
         if avoid_quote not in {'"', "'"}:
-            raise Exception("Must specify either ' or \" single quote to avoid.")
+            raise ValueError("Must specify either ' or \" single quote to avoid.")
         self.avoid_quote: str = avoid_quote
         self.replace_quote: str = '"' if avoid_quote == "'" else "'"
 
@@ -296,7 +302,7 @@ class ConvertFormatStringCommand(VisitorBasedCodemodCommand):
                     ) in format_spec_tokens:
                         if spec_format_spec is not None:
                             # This shouldn't be possible, we don't allow it in the spec!
-                            raise Exception("Logic error!")
+                            raise CSTLogicError("Logic error!")
                         if spec_literal_text:
                             format_spec_parts.append(
                                 cst.FormattedStringText(spec_literal_text)

libcst/codemod/commands/convert_namedtuple_to_dataclass.py

@@ -25,7 +25,9 @@ class ConvertNamedTupleToDataclassCommand(VisitorBasedCodemodCommand):
     NamedTuple-specific attributes and methods.
     """
 
-    DESCRIPTION: str = "Convert NamedTuple class declarations to Python 3.7 dataclasses using the @dataclass decorator."
+    DESCRIPTION: str = (
+        "Convert NamedTuple class declarations to Python 3.7 dataclasses using the @dataclass decorator."
+    )
     METADATA_DEPENDENCIES: Sequence[ProviderT] = (QualifiedNameProvider,)
 
     # The 'NamedTuple' we are interested in

libcst/codemod/commands/convert_percent_format_to_fstring.py

@@ -53,12 +53,12 @@ class EscapeStringQuote(cst.CSTTransformer):
                         original_node.prefix + quo + original_node.raw_value + quo
                     )
                     if escaped_string.evaluated_value != original_node.evaluated_value:
-                        raise Exception(
+                        raise ValueError(
                             f"Failed to escape string:\n  original:{original_node.value}\n  escaped:{escaped_string.value}"
                         )
                     else:
                         return escaped_string
-            raise Exception(
+            raise ValueError(
                 f"Cannot find a good quote for escaping the SimpleString: {original_node.value}"
             )
         return original_node
@@ -97,9 +97,11 @@ class ConvertPercentFormatStringCommand(VisitorBasedCodemodCommand):
                 parts.append(cst.FormattedStringText(value=token))
             expressions: List[cst.CSTNode] = list(
                 *itertools.chain(
-                    [elm.value for elm in expr.elements]
-                    if isinstance(expr, cst.Tuple)
-                    else [expr]
+                    (
+                        [elm.value for elm in expr.elements]
+                        if isinstance(expr, cst.Tuple)
+                        else [expr]
+                    )
                     for expr in exprs
                 )
             )

libcst/codemod/commands/convert_union_to_or.py

@@ -0,0 +1,56 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+# pyre-strict
+
+import libcst as cst
+from libcst.codemod import VisitorBasedCodemodCommand
+from libcst.codemod.visitors import RemoveImportsVisitor
+from libcst.metadata import QualifiedName, QualifiedNameProvider, QualifiedNameSource
+
+
+class ConvertUnionToOrCommand(VisitorBasedCodemodCommand):
+    DESCRIPTION: str = "Convert `Union[A, B]` to `A | B` in Python 3.10+"
+
+    METADATA_DEPENDENCIES = (QualifiedNameProvider,)
+
+    def leave_Subscript(
+        self, original_node: cst.Subscript, updated_node: cst.Subscript
+    ) -> cst.BaseExpression:
+        """
+        Given a subscript, check if it's a Union - if so, either flatten the members
+        into a nested BitOr (if multiple members) or unwrap the type (if only one member).
+        """
+        if not QualifiedNameProvider.has_name(
+            self,
+            original_node,
+            QualifiedName(name="typing.Union", source=QualifiedNameSource.IMPORT),
+        ):
+            return updated_node
+        types = [
+            cst.ensure_type(
+                cst.ensure_type(s, cst.SubscriptElement).slice, cst.Index
+            ).value
+            for s in updated_node.slice
+        ]
+        if len(types) == 1:
+            return types[0]
+        else:
+            replacement = cst.BinaryOperation(
+                left=types[0], right=types[1], operator=cst.BitOr()
+            )
+            for type_ in types[2:]:
+                replacement = cst.BinaryOperation(
+                    left=replacement, right=type_, operator=cst.BitOr()
+                )
+            return replacement
+
+    def leave_Module(
+        self, original_node: cst.Module, updated_node: cst.Module
+    ) -> cst.Module:
+        RemoveImportsVisitor.remove_unused_import(
+            self.context, module="typing", obj="Union"
+        )
+        return updated_node

libcst/codemod/commands/fix_pyre_directives.py

@@ -7,6 +7,7 @@ from typing import Dict, Sequence, Union
 
 import libcst
 import libcst.matchers as m
+from libcst import CSTLogicError
 from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand
 from libcst.helpers import insert_header_comments
 
@@ -29,12 +30,12 @@ class FixPyreDirectivesCommand(VisitorBasedCodemodCommand):
 
     def visit_Module_header(self, node: libcst.Module) -> None:
         if self.in_module_header:
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
         self.in_module_header = True
 
     def leave_Module_header(self, node: libcst.Module) -> None:
         if not self.in_module_header:
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
         self.in_module_header = False
 
     def leave_EmptyLine(

libcst/codemod/commands/fix_variadic_callable.py

@@ -0,0 +1,40 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+# pyre-strict
+
+import libcst as cst
+import libcst.matchers as m
+from libcst.codemod import VisitorBasedCodemodCommand
+from libcst.metadata import QualifiedName, QualifiedNameProvider, QualifiedNameSource
+
+
+class FixVariadicCallableCommmand(VisitorBasedCodemodCommand):
+    DESCRIPTION: str = (
+        "Fix incorrect variadic callable type annotations from `Callable[[...], T]` to `Callable[..., T]``"
+    )
+
+    METADATA_DEPENDENCIES = (QualifiedNameProvider,)
+
+    def leave_Subscript(
+        self, original_node: cst.Subscript, updated_node: cst.Subscript
+    ) -> cst.BaseExpression:
+        if QualifiedNameProvider.has_name(
+            self,
+            original_node,
+            QualifiedName(name="typing.Callable", source=QualifiedNameSource.IMPORT),
+        ):
+            node_matches = len(updated_node.slice) == 2 and m.matches(
+                updated_node.slice[0],
+                m.SubscriptElement(
+                    slice=m.Index(value=m.List(elements=[m.Element(m.Ellipsis())]))
+                ),
+            )
+
+            if node_matches:
+                slices = list(updated_node.slice)
+                slices[0] = cst.SubscriptElement(cst.Index(cst.Ellipsis()))
+                return updated_node.with_changes(slice=slices)
+        return updated_node

libcst/codemod/commands/rename.py

@@ -15,7 +15,7 @@ from libcst.metadata import QualifiedNameProvider
 
 
 def leave_import_decorator(
-    method: Callable[..., Union[cst.Import, cst.ImportFrom]]
+    method: Callable[..., Union[cst.Import, cst.ImportFrom]],
 ) -> Callable[..., Union[cst.Import, cst.ImportFrom]]:
     # We want to record any 'as name' that is relevant but only after we leave the corresponding Import/ImportFrom node since
     # we don't want the 'as name' to interfere with children 'Name' and 'Attribute' nodes.
@@ -92,14 +92,43 @@ class RenameCommand(VisitorBasedCodemodCommand):
         self.old_module: str = old_module
         self.old_mod_or_obj: str = old_mod_or_obj
 
-        self.as_name: Optional[Tuple[str, str]] = None
+    @property
+    def as_name(self) -> Optional[Tuple[str, str]]:
+        if "as_name" not in self.context.scratch:
+            self.context.scratch["as_name"] = None
+        return self.context.scratch["as_name"]
 
-        # A set of nodes that have been renamed to help with the cleanup of now potentially unused
-        # imports, during import cleanup in `leave_Module`.
-        self.scheduled_removals: Set[cst.CSTNode] = set()
-        # If an import has been renamed while inside an `Import` or `ImportFrom` node, we want to flag
-        # this so that we do not end up with two of the same import.
-        self.bypass_import = False
+    @as_name.setter
+    def as_name(self, value: Optional[Tuple[str, str]]) -> None:
+        self.context.scratch["as_name"] = value
+
+    @property
+    def scheduled_removals(
+        self,
+    ) -> Set[Union[cst.CSTNode, Tuple[str, Optional[str], Optional[str]]]]:
+        """A set of nodes that have been renamed to help with the cleanup of now potentially unused
+        imports, during import cleanup in `leave_Module`. Can also contain tuples that can be passed
+        directly to RemoveImportsVisitor.remove_unused_import()."""
+        if "scheduled_removals" not in self.context.scratch:
+            self.context.scratch["scheduled_removals"] = set()
+        return self.context.scratch["scheduled_removals"]
+
+    @scheduled_removals.setter
+    def scheduled_removals(
+        self, value: Set[Union[cst.CSTNode, Tuple[str, Optional[str], Optional[str]]]]
+    ) -> None:
+        self.context.scratch["scheduled_removals"] = value
+
+    @property
+    def bypass_import(self) -> bool:
+        """A flag to indicate that an import has been renamed while inside an `Import` or `ImportFrom` node."""
+        if "bypass_import" not in self.context.scratch:
+            self.context.scratch["bypass_import"] = False
+        return self.context.scratch["bypass_import"]
+
+    @bypass_import.setter
+    def bypass_import(self, value: bool) -> None:
+        self.context.scratch["bypass_import"] = value
 
     def visit_Import(self, node: cst.Import) -> None:
         for import_alias in node.names:
@@ -118,38 +147,42 @@ class RenameCommand(VisitorBasedCodemodCommand):
     ) -> cst.Import:
         new_names = []
         for import_alias in updated_node.names:
+            # We keep the original import_alias here in case it's used by other symbols.
+            # It will be removed later in RemoveImportsVisitor if it's unused.
+            new_names.append(import_alias)
             import_alias_name = import_alias.name
             import_alias_full_name = get_full_name_for_node(import_alias_name)
             if import_alias_full_name is None:
-                raise Exception("Could not parse full name for ImportAlias.name node.")
+                raise ValueError("Could not parse full name for ImportAlias.name node.")
 
-            if isinstance(import_alias_name, cst.Name) and self.old_name.startswith(
-                import_alias_full_name + "."
-            ):
-                # Might, be in use elsewhere in the code, so schedule a potential removal, and add another alias.
-                new_names.append(import_alias)
-                replacement_module = self.gen_replacement_module(import_alias_full_name)
-                self.bypass_import = True
-                if replacement_module != import_alias_name.value:
-                    self.scheduled_removals.add(original_node)
-                    new_names.append(
-                        cst.ImportAlias(name=cst.Name(value=replacement_module))
-                    )
-            elif isinstance(
-                import_alias_name, cst.Attribute
-            ) and self.old_name.startswith(import_alias_full_name + "."):
-                # Same idea as above.
-                new_names.append(import_alias)
+            if self.old_name.startswith(import_alias_full_name + "."):
                 replacement_module = self.gen_replacement_module(import_alias_full_name)
+                if not replacement_module:
+                    # here import_alias_full_name isn't an exact match for old_name
+                    # don't add an import here, it will be handled either in more
+                    # specific import aliases or at the very end
+                    continue
                 self.bypass_import = True
                 if replacement_module != import_alias_full_name:
                     self.scheduled_removals.add(original_node)
-                    new_name_node: Union[
-                        cst.Attribute, cst.Name
-                    ] = self.gen_name_or_attr_node(replacement_module)
+                    new_name_node: Union[cst.Attribute, cst.Name] = (
+                        self.gen_name_or_attr_node(replacement_module)
+                    )
                     new_names.append(cst.ImportAlias(name=new_name_node))
-            else:
-                new_names.append(import_alias)
+            elif (
+                import_alias_full_name == self.new_name
+                and import_alias.asname is not None
+            ):
+                self.bypass_import = True
+                # Add removal tuple instead of calling directly
+                self.scheduled_removals.add(
+                    (
+                        import_alias.evaluated_name,
+                        None,
+                        import_alias.evaluated_alias,
+                    )
+                )
+                new_names.append(import_alias.with_changes(asname=None))
 
         return updated_node.with_changes(names=new_names)
 
@@ -181,7 +214,7 @@ class RenameCommand(VisitorBasedCodemodCommand):
             return updated_node
 
         else:
-            new_names = []
+            new_names: list[cst.ImportAlias] = []
             for import_alias in names:
                 alias_name = get_full_name_for_node(import_alias.name)
                 if alias_name is not None:
@@ -198,9 +231,9 @@ class RenameCommand(VisitorBasedCodemodCommand):
                             self.scheduled_removals.add(original_node)
                             continue
 
-                        new_import_alias_name: Union[
-                            cst.Attribute, cst.Name
-                        ] = self.gen_name_or_attr_node(replacement_obj)
+                        new_import_alias_name: Union[cst.Attribute, cst.Name] = (
+                            self.gen_name_or_attr_node(replacement_obj)
+                        )
                         # Rename on the spot only if this is the only imported name under the module.
                         if len(names) == 1:
                             updated_node = updated_node.with_changes(
@@ -219,6 +252,10 @@ class RenameCommand(VisitorBasedCodemodCommand):
                             # This import might be in use elsewhere in the code, so schedule a potential removal.
                             self.scheduled_removals.add(original_node)
                         new_names.append(import_alias)
+            if isinstance(new_names[-1].comma, cst.Comma) and updated_node.rpar is None:
+                new_names[-1] = new_names[-1].with_changes(
+                    comma=cst.MaybeSentinel.DEFAULT
+                )
 
             return updated_node.with_changes(names=new_names)
         return updated_node
@@ -249,7 +286,7 @@ class RenameCommand(VisitorBasedCodemodCommand):
     ) -> Union[cst.Name, cst.Attribute]:
         full_name_for_node = get_full_name_for_node(original_node)
         if full_name_for_node is None:
-            raise Exception("Could not parse full name for Attribute node.")
+            raise ValueError("Could not parse full name for Attribute node.")
         full_replacement_name = self.gen_replacement(full_name_for_node)
 
         # If a node has no associated QualifiedName, we are still inside an import statement.
@@ -265,10 +302,14 @@ class RenameCommand(VisitorBasedCodemodCommand):
             if not inside_import_statement:
                 self.scheduled_removals.add(original_node.value)
             if full_replacement_name == self.new_name:
-                return updated_node.with_changes(
-                    value=cst.parse_expression(new_value),
-                    attr=cst.Name(value=new_attr.rstrip(".")),
-                )
+                value = cst.parse_expression(new_value)
+                if new_attr:
+                    return updated_node.with_changes(
+                        value=value,
+                        attr=cst.Name(value=new_attr.rstrip(".")),
+                    )
+                assert isinstance(value, (cst.Name, cst.Attribute))
+                return value
 
             return self.gen_name_or_attr_node(new_attr)
 
@@ -277,14 +318,17 @@ class RenameCommand(VisitorBasedCodemodCommand):
     def leave_Module(
         self, original_node: cst.Module, updated_node: cst.Module
     ) -> cst.Module:
-        for removal_node in self.scheduled_removals:
-            RemoveImportsVisitor.remove_unused_import_by_node(
-                self.context, removal_node
-            )
+        for removal in self.scheduled_removals:
+            if isinstance(removal, tuple):
+                RemoveImportsVisitor.remove_unused_import(
+                    self.context, removal[0], removal[1], removal[2]
+                )
+            else:
+                RemoveImportsVisitor.remove_unused_import_by_node(self.context, removal)
         # If bypass_import is False, we know that no import statements were directly renamed, and the fact
         # that we have any `self.scheduled_removals` tells us we encountered a matching `old_name` in the code.
         if not self.bypass_import and self.scheduled_removals:
-            if self.new_module:
+            if self.new_module and self.new_module != "builtins":
                 new_obj: Optional[str] = (
                     self.new_mod_or_obj.split(".")[0] if self.new_mod_or_obj else None
                 )
@@ -303,10 +347,14 @@ class RenameCommand(VisitorBasedCodemodCommand):
                     module_as_name[0] + ".", module_as_name[1] + ".", 1
                 )
 
-        if original_name == self.old_mod_or_obj:
+        if self.old_module and original_name == self.old_mod_or_obj:
             return self.new_mod_or_obj
-        elif original_name == ".".join([self.old_module, self.old_mod_or_obj]):
-            return self.new_name
+        elif original_name == self.old_name:
+            return (
+                self.new_mod_or_obj
+                if (not self.bypass_import and self.new_mod_or_obj)
+                else self.new_name
+            )
         elif original_name.endswith("." + self.old_mod_or_obj):
             return self.new_mod_or_obj
         else:
@@ -320,7 +368,7 @@ class RenameCommand(VisitorBasedCodemodCommand):
     ) -> Union[cst.Attribute, cst.Name]:
         name_or_attr_node: cst.BaseExpression = cst.parse_expression(dotted_expression)
         if not isinstance(name_or_attr_node, (cst.Name, cst.Attribute)):
-            raise Exception(
+            raise ValueError(
                 "`parse_expression()` on dotted path returned non-Attribute-or-Name."
             )
         return name_or_attr_node

libcst/codemod/commands/rename_typing_generic_aliases.py

@@ -0,0 +1,37 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+# pyre-strict
+from functools import partial
+from typing import cast, Generator
+
+from libcst.codemod import Codemod, MagicArgsCodemodCommand
+from libcst.codemod.commands.rename import RenameCommand
+
+
+class RenameTypingGenericAliases(MagicArgsCodemodCommand):
+    DESCRIPTION: str = (
+        "Rename typing module aliases of builtin generics in Python 3.9+, for example: `typing.List` -> `list`"
+    )
+
+    MAPPING: dict[str, str] = {
+        "typing.List": "builtins.list",
+        "typing.Tuple": "builtins.tuple",
+        "typing.Dict": "builtins.dict",
+        "typing.FrozenSet": "builtins.frozenset",
+        "typing.Set": "builtins.set",
+        "typing.Type": "builtins.type",
+    }
+
+    def get_transforms(self) -> Generator[type[Codemod], None, None]:
+        for from_type, to_type in self.MAPPING.items():
+            yield cast(
+                type[Codemod],
+                partial(
+                    RenameCommand,
+                    old_name=from_type,
+                    new_name=to_type,
+                ),
+            )

libcst/codemod/commands/tests/test_convert_union_to_or.py

@@ -0,0 +1,86 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+# pyre-strict
+
+from libcst.codemod import CodemodTest
+from libcst.codemod.commands.convert_union_to_or import ConvertUnionToOrCommand
+
+
+class TestConvertUnionToOrCommand(CodemodTest):
+    TRANSFORM = ConvertUnionToOrCommand
+
+    def test_simple_union(self) -> None:
+        before = """
+            from typing import Union
+            x: Union[int, str]
+        """
+        after = """
+            x: int | str
+        """
+        self.assertCodemod(before, after)
+
+    def test_nested_union(self) -> None:
+        before = """
+            from typing import Union
+            x: Union[int, Union[str, float]]
+        """
+        after = """
+            x: int | str | float
+        """
+        self.assertCodemod(before, after)
+
+    def test_single_type_union(self) -> None:
+        before = """
+            from typing import Union
+            x: Union[int]
+        """
+        after = """
+            x: int
+        """
+        self.assertCodemod(before, after)
+
+    def test_union_with_alias(self) -> None:
+        before = """
+            import typing as t
+            x: t.Union[int, str]
+        """
+        after = """
+            import typing as t
+            x: int | str
+        """
+        self.assertCodemod(before, after)
+
+    def test_union_with_unused_import(self) -> None:
+        before = """
+            from typing import Union, List
+            x: Union[int, str]
+        """
+        after = """
+            from typing import List
+            x: int | str
+        """
+        self.assertCodemod(before, after)
+
+    def test_union_no_import(self) -> None:
+        before = """
+            x: Union[int, str]
+        """
+        after = """
+            x: Union[int, str]
+        """
+        self.assertCodemod(before, after)
+
+    def test_union_in_function(self) -> None:
+        before = """
+            from typing import Union
+            def foo(x: Union[int, str]) -> Union[float, None]:
+                ...
+        """
+        after = """
+            def foo(x: int | str) -> float | None:
+                ...
+        """
+        self.assertCodemod(before, after)

libcst/codemod/commands/tests/test_fix_variadic_callable.py

@@ -0,0 +1,92 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+# pyre-strict
+
+from libcst.codemod import CodemodTest
+from libcst.codemod.commands.fix_variadic_callable import FixVariadicCallableCommmand
+
+
+class TestFixVariadicCallableCommmand(CodemodTest):
+    TRANSFORM = FixVariadicCallableCommmand
+
+    def test_callable_typing(self) -> None:
+        before = """
+            from typing import Callable
+            x: Callable[[...], int] = ...
+        """
+        after = """
+            from typing import Callable
+            x: Callable[..., int] = ...
+        """
+        self.assertCodemod(before, after)
+
+    def test_callable_typing_alias(self) -> None:
+        before = """
+            import typing as t
+            x: t.Callable[[...], int] = ...
+        """
+        after = """
+            import typing as t
+            x: t.Callable[..., int] = ...
+        """
+        self.assertCodemod(before, after)
+
+    def test_callable_import_alias(self) -> None:
+        before = """
+            from typing import Callable as C
+            x: C[[...], int] = ...
+        """
+        after = """
+            from typing import Callable as C
+            x: C[..., int] = ...
+        """
+        self.assertCodemod(before, after)
+
+    def test_callable_with_optional(self) -> None:
+        before = """
+            from typing import Callable
+            def foo(bar: Optional[Callable[[...], int]]) -> Callable[[...], int]:
+                ...
+        """
+        after = """
+            from typing import Callable
+            def foo(bar: Optional[Callable[..., int]]) -> Callable[..., int]:
+                ...
+        """
+        self.assertCodemod(before, after)
+
+    def test_callable_with_arguments(self) -> None:
+        before = """
+            from typing import Callable
+            x: Callable[[int], int]
+        """
+        after = """
+            from typing import Callable
+            x: Callable[[int], int]
+        """
+        self.assertCodemod(before, after)
+
+    def test_callable_with_variadic_arguments(self) -> None:
+        before = """
+            from typing import Callable
+            x: Callable[[int, int, ...], int]
+        """
+        after = """
+            from typing import Callable
+            x: Callable[[int, int, ...], int]
+        """
+        self.assertCodemod(before, after)
+
+    def test_callable_no_arguments(self) -> None:
+        before = """
+            from typing import Callable
+            x: Callable
+        """
+        after = """
+            from typing import Callable
+            x: Callable
+        """
+        self.assertCodemod(before, after)

libcst/codemod/commands/tests/test_rename.py

@@ -28,6 +28,19 @@ class TestRenameCommand(CodemodTest):
 
         self.assertCodemod(before, after, old_name="foo.bar", new_name="baz.qux")
 
+    def test_rename_to_builtin(self) -> None:
+        before = """
+            from typing import List
+            x: List[int] = []
+        """
+        after = """
+            x: list[int] = []
+        """
+
+        self.assertCodemod(
+            before, after, old_name="typing.List", new_name="builtins.list"
+        )
+
     def test_rename_name_asname(self) -> None:
         before = """
             from foo import bar as bla
@@ -111,6 +124,27 @@ class TestRenameCommand(CodemodTest):
             new_name="baz.quux",
         )
 
+    def test_rename_attr_asname_2(self) -> None:
+        before = """
+            import foo.qux as bar
+
+            def test() -> None:
+                bar.z(5)
+        """
+        after = """
+            import baz.quux
+
+            def test() -> None:
+                baz.quux.z(5)
+        """
+
+        self.assertCodemod(
+            before,
+            after,
+            old_name="foo.qux",
+            new_name="baz.quux",
+        )
+
     def test_rename_module_import(self) -> None:
         before = """
             import a.b
@@ -361,6 +395,28 @@ class TestRenameCommand(CodemodTest):
             new_name="d.z",
         )
 
+    def test_comma_import(self) -> None:
+        before = """
+            import a, b, c
+
+            class Foo(a.z):
+                bar: b.bar
+                baz: c.baz
+        """
+        after = """
+            import a, b, d
+
+            class Foo(a.z):
+                bar: b.bar
+                baz: d.baz
+        """
+        self.assertCodemod(
+            before,
+            after,
+            old_name="c.baz",
+            new_name="d.baz",
+        )
+
     def test_other_import_froms_untouched(self) -> None:
         before = """
             from a import b, c, d
@@ -384,6 +440,61 @@ class TestRenameCommand(CodemodTest):
             new_name="f.b",
         )
 
+    def test_comma_import_from(self) -> None:
+        before = """
+            from a import b, c, d
+
+            class Foo(b):
+                bar: c.bar
+                baz: d.baz
+        """
+        after = """
+            from a import b, c
+            from f import d
+
+            class Foo(b):
+                bar: c.bar
+                baz: d.baz
+        """
+        self.assertCodemod(
+            before,
+            after,
+            old_name="a.d",
+            new_name="f.d",
+        )
+
+    def test_comma_import_from_parens(self) -> None:
+        before = """
+            from a import (
+                b,
+                c,
+                d,
+            )
+            from x import (y,)
+
+            class Foo(b):
+                bar: c.bar
+                baz: d.baz
+        """
+        after = """
+            from a import (
+                b,
+                c,
+                )
+            from x import (y,)
+            from f import d
+
+            class Foo(b):
+                bar: c.bar
+                baz: d.baz
+        """
+        self.assertCodemod(
+            before,
+            after,
+            old_name="a.d",
+            new_name="f.d",
+        )
+
     def test_no_removal_of_import_in_use(self) -> None:
         before = """
             import a
@@ -705,3 +816,72 @@ class TestRenameCommand(CodemodTest):
             old_name="a.b.qux",
             new_name="a:b.qux",
         )
+
+    def test_import_parent_module(self) -> None:
+        before = """
+            import a
+            a.b.c(a.b.c.d)
+        """
+        after = """
+            from z import c
+
+            c(c.d)
+        """
+        self.assertCodemod(before, after, old_name="a.b.c", new_name="z.c")
+
+    def test_import_parent_module_2(self) -> None:
+        before = """
+            import a.b
+            a.b.c.d(a.b.c.d.x)
+        """
+        after = """
+            from z import c
+            
+            c(c.x)
+        """
+        self.assertCodemod(before, after, old_name="a.b.c.d", new_name="z.c")
+
+    def test_import_parent_module_3(self) -> None:
+        before = """
+            import a
+            a.b.c(a.b.c.d)
+        """
+        after = """
+            import z.c
+
+            z.c(z.c.d)
+        """
+        self.assertCodemod(before, after, old_name="a.b.c", new_name="z.c:")
+
+    def test_import_parent_module_asname(self) -> None:
+        before = """
+            import a.b as alias
+            alias.c(alias.c.d)
+        """
+        after = """
+            import z
+            z.c(z.c.d)
+        """
+        self.assertCodemod(before, after, old_name="a.b.c", new_name="z.c")
+
+    def test_push_down_toplevel_names(self) -> None:
+        before = """
+            import foo
+            foo.baz()
+        """
+        after = """
+            import quux.foo
+            quux.foo.baz()
+        """
+        self.assertCodemod(before, after, old_name="foo", new_name="quux.foo")
+
+    def test_push_down_toplevel_names_with_asname(self) -> None:
+        before = """
+            import foo as bar
+            bar.baz()
+        """
+        after = """
+            import quux.foo
+            quux.foo.baz()
+        """
+        self.assertCodemod(before, after, old_name="foo", new_name="quux.foo")

libcst/codemod/commands/tests/test_rename_typing_generic_aliases.py

@@ -0,0 +1,33 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+# pyre-strict
+
+from libcst.codemod import CodemodTest
+from libcst.codemod.commands.rename_typing_generic_aliases import (
+    RenameTypingGenericAliases,
+)
+
+
+class TestRenameCommand(CodemodTest):
+    TRANSFORM = RenameTypingGenericAliases
+
+    def test_rename_typing_generic_alias(self) -> None:
+        before = """
+            from typing import List, Set, Dict, FrozenSet, Tuple
+            x: List[int] = []
+            y: Set[int] = set()
+            z: Dict[str, int] = {}
+            a: FrozenSet[str] = frozenset()
+            b: Tuple[int, str] = (1, "hello")
+        """
+        after = """
+            x: list[int] = []
+            y: set[int] = set()
+            z: dict[str, int] = {}
+            a: frozenset[str] = frozenset()
+            b: tuple[int, str] = (1, "hello")
+        """
+        self.assertCodemod(before, after)

libcst/codemod/tests/test_codemod_cli.py

@@ -8,10 +8,11 @@
 import platform
 import subprocess
 import sys
+import tempfile
 from pathlib import Path
 from unittest import skipIf
 
-from libcst._parser.entrypoints import is_native
+from libcst.codemod import CodemodTest
 from libcst.testing.utils import UnitTest
 
 
@@ -35,16 +36,10 @@ class TestCodemodCLI(UnitTest):
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
-        if not is_native():
-            self.assertIn(
-                "ParserSyntaxError: Syntax Error @ 14:11.",
-                rlt.stderr.decode("utf-8"),
-            )
-        else:
-            self.assertIn(
-                "error: cannot format -: Cannot parse: 13:10:     async with AsyncExitStack() as stack:",
-                rlt.stderr.decode("utf-8"),
-            )
+        self.assertIn(
+            "error: cannot format -: Cannot parse for target version Python 3.6: 13:10:     async with AsyncExitStack() as stack:",
+            rlt.stderr.decode("utf-8"),
+        )
 
     def test_codemod_external(self) -> None:
         # Test running the NOOP command as an "external command"
@@ -63,3 +58,62 @@ class TestCodemodCLI(UnitTest):
             stderr=subprocess.STDOUT,
         )
         assert "Finished codemodding 1 files!" in output
+
+    def test_warning_messages_several_files(self) -> None:
+        code = """
+        def baz() -> str:
+            return "{}: {}".format(*baz)
+        """
+        with tempfile.TemporaryDirectory() as tmpdir:
+            p = Path(tmpdir)
+            (p / "mod1.py").write_text(CodemodTest.make_fixture_data(code))
+            (p / "mod2.py").write_text(CodemodTest.make_fixture_data(code))
+            (p / "mod3.py").write_text(CodemodTest.make_fixture_data(code))
+            output = subprocess.run(
+                [
+                    sys.executable,
+                    "-m",
+                    "libcst.tool",
+                    "codemod",
+                    "convert_format_to_fstring.ConvertFormatStringCommand",
+                    str(p),
+                ],
+                encoding="utf-8",
+                stderr=subprocess.PIPE,
+            )
+            # Each module will generate a warning, so we should get 3 warnings in total
+            self.assertIn(
+                "- 3 warnings were generated.",
+                output.stderr,
+            )
+
+    def test_matcher_decorators_multiprocessing(self) -> None:
+        file_count = 5
+        code = """
+        def baz(): # type: int
+            return 5
+        """
+        with tempfile.TemporaryDirectory() as tmpdir:
+            p = Path(tmpdir)
+            # Using more than chunksize=4 files to trigger multiprocessing
+            for i in range(file_count):
+                (p / f"mod{i}.py").write_text(CodemodTest.make_fixture_data(code))
+            output = subprocess.run(
+                [
+                    sys.executable,
+                    "-m",
+                    "libcst.tool",
+                    "codemod",
+                    # Good candidate since it uses matcher decorators
+                    "convert_type_comments.ConvertTypeComments",
+                    str(p),
+                    "--jobs",
+                    str(file_count),
+                ],
+                encoding="utf-8",
+                stderr=subprocess.PIPE,
+            )
+            self.assertIn(
+                f"Transformed {file_count} files successfully.",
+                output.stderr,
+            )

libcst/codemod/tests/test_command_helpers.py

@@ -0,0 +1,325 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+from typing import Union
+
+import libcst as cst
+from libcst.codemod import CodemodTest, VisitorBasedCodemodCommand
+
+
+class TestRemoveUnusedImportHelper(CodemodTest):
+    """Tests for the remove_unused_import helper method in CodemodCommand."""
+
+    def test_remove_unused_import_simple(self) -> None:
+        """
+        Test that remove_unused_import helper method works correctly.
+        """
+
+        class RemoveBarImport(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Use the helper method to schedule removal
+                self.remove_unused_import("bar")
+
+        before = """
+            import bar
+            import baz
+
+            def foo() -> None:
+                pass
+        """
+        after = """
+            import baz
+
+            def foo() -> None:
+                pass
+        """
+
+        self.TRANSFORM = RemoveBarImport
+        self.assertCodemod(before, after)
+
+    def test_remove_unused_import_from_simple(self) -> None:
+        """
+        Test that remove_unused_import helper method works correctly with from imports.
+        """
+
+        class RemoveBarFromImport(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Use the helper method to schedule removal
+                self.remove_unused_import("a.b.c", "bar")
+
+        before = """
+            from a.b.c import bar, baz
+
+            def foo() -> None:
+                baz()
+        """
+        after = """
+            from a.b.c import baz
+
+            def foo() -> None:
+                baz()
+        """
+
+        self.TRANSFORM = RemoveBarFromImport
+        self.assertCodemod(before, after)
+
+    def test_remove_unused_import_with_alias(self) -> None:
+        """
+        Test that remove_unused_import helper method works correctly with aliased imports.
+        """
+
+        class RemoveBarAsQuxImport(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Use the helper method to schedule removal
+                self.remove_unused_import("a.b.c", "bar", "qux")
+
+        before = """
+            from a.b.c import bar as qux, baz
+
+            def foo() -> None:
+                baz()
+        """
+        after = """
+            from a.b.c import baz
+
+            def foo() -> None:
+                baz()
+        """
+
+        self.TRANSFORM = RemoveBarAsQuxImport
+        self.assertCodemod(before, after)
+
+
+class TestRemoveUnusedImportByNodeHelper(CodemodTest):
+    """Tests for the remove_unused_import_by_node helper method in CodemodCommand."""
+
+    def test_remove_unused_import_by_node_simple(self) -> None:
+        """
+        Test that remove_unused_import_by_node helper method works correctly.
+        """
+
+        class RemoveBarCallAndImport(VisitorBasedCodemodCommand):
+            METADATA_DEPENDENCIES = (
+                cst.metadata.QualifiedNameProvider,
+                cst.metadata.ScopeProvider,
+            )
+
+            def leave_SimpleStatementLine(
+                self,
+                original_node: cst.SimpleStatementLine,
+                updated_node: cst.SimpleStatementLine,
+            ) -> Union[cst.RemovalSentinel, cst.SimpleStatementLine]:
+                # Remove any statement that calls bar()
+                if cst.matchers.matches(
+                    updated_node,
+                    cst.matchers.SimpleStatementLine(
+                        body=[cst.matchers.Expr(cst.matchers.Call())]
+                    ),
+                ):
+                    call = cst.ensure_type(updated_node.body[0], cst.Expr).value
+                    if cst.matchers.matches(
+                        call, cst.matchers.Call(func=cst.matchers.Name("bar"))
+                    ):
+                        # Use the helper method to remove imports referenced by this node
+                        self.remove_unused_import_by_node(original_node)
+                        return cst.RemoveFromParent()
+                return updated_node
+
+        before = """
+            from foo import bar, baz
+
+            def fun() -> None:
+                bar()
+                baz()
+        """
+        after = """
+            from foo import baz
+
+            def fun() -> None:
+                baz()
+        """
+
+        self.TRANSFORM = RemoveBarCallAndImport
+        self.assertCodemod(before, after)
+
+
+class TestAddNeededImportHelper(CodemodTest):
+    """Tests for the add_needed_import helper method in CodemodCommand."""
+
+    def test_add_needed_import_simple(self) -> None:
+        """
+        Test that add_needed_import helper method works correctly.
+        """
+
+        class AddBarImport(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Use the helper method to schedule import addition
+                self.add_needed_import("bar")
+
+        before = """
+            def foo() -> None:
+                pass
+        """
+        after = """
+            import bar
+
+            def foo() -> None:
+                pass
+        """
+
+        self.TRANSFORM = AddBarImport
+        self.assertCodemod(before, after)
+
+    def test_add_needed_import_from_simple(self) -> None:
+        """
+        Test that add_needed_import helper method works correctly with from imports.
+        """
+
+        class AddBarFromImport(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Use the helper method to schedule import addition
+                self.add_needed_import("a.b.c", "bar")
+
+        before = """
+            def foo() -> None:
+                pass
+        """
+        after = """
+            from a.b.c import bar
+
+            def foo() -> None:
+                pass
+        """
+
+        self.TRANSFORM = AddBarFromImport
+        self.assertCodemod(before, after)
+
+    def test_add_needed_import_with_alias(self) -> None:
+        """
+        Test that add_needed_import helper method works correctly with aliased imports.
+        """
+
+        class AddBarAsQuxImport(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Use the helper method to schedule import addition
+                self.add_needed_import("a.b.c", "bar", "qux")
+
+        before = """
+            def foo() -> None:
+                pass
+        """
+        after = """
+            from a.b.c import bar as qux
+
+            def foo() -> None:
+                pass
+        """
+
+        self.TRANSFORM = AddBarAsQuxImport
+        self.assertCodemod(before, after)
+
+    def test_add_needed_import_relative(self) -> None:
+        """
+        Test that add_needed_import helper method works correctly with relative imports.
+        """
+
+        class AddRelativeImport(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Use the helper method to schedule relative import addition
+                self.add_needed_import("c", "bar", relative=2)
+
+        before = """
+            def foo() -> None:
+                pass
+        """
+        after = """
+            from ..c import bar
+
+            def foo() -> None:
+                pass
+        """
+
+        self.TRANSFORM = AddRelativeImport
+        self.assertCodemod(before, after)
+
+
+class TestCombinedHelpers(CodemodTest):
+    """Tests for combining add_needed_import and remove_unused_import helper methods."""
+
+    def test_add_and_remove_imports(self) -> None:
+        """
+        Test that both helper methods work correctly when used together.
+        """
+
+        class ReplaceBarWithBaz(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Add new import and remove old one
+                self.add_needed_import("new_module", "baz")
+                self.remove_unused_import("old_module", "bar")
+
+        before = """
+            from other_module import qux
+            from old_module import bar
+
+            def foo() -> None:
+                pass
+        """
+        after = """
+            from other_module import qux
+            from new_module import baz
+
+            def foo() -> None:
+                pass
+        """
+
+        self.TRANSFORM = ReplaceBarWithBaz
+        self.assertCodemod(before, after)
+
+    def test_add_and_remove_same_import(self) -> None:
+        """
+        Test that both helper methods work correctly when used together.
+        """
+
+        class AddAndRemoveBar(VisitorBasedCodemodCommand):
+            def visit_Module(self, node: cst.Module) -> None:
+                # Add new import and remove old one
+                self.add_needed_import("hello_module", "bar")
+                self.remove_unused_import("hello_module", "bar")
+
+        self.TRANSFORM = AddAndRemoveBar
+
+        before = """
+            from other_module import baz
+
+            def foo() -> None:
+                pass
+        """
+        # Should remain unchanged
+        self.assertCodemod(before, before)
+
+        before = """
+            from other_module import baz
+            from hello_module import bar
+
+            def foo() -> None:
+                bar.func()
+        """
+        self.assertCodemod(before, before)
+
+        before = """
+            from other_module import baz
+            from hello_module import bar
+
+            def foo() -> None:
+                pass
+        """
+
+        after = """
+            from other_module import baz
+
+            def foo() -> None:
+                pass
+        """
+        self.assertCodemod(before, after)

libcst/codemod/visitors/_add_imports.py

@@ -7,7 +7,7 @@ from collections import defaultdict
 from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
 
 import libcst
-from libcst import matchers as m, parse_statement
+from libcst import CSTLogicError, matchers as m, parse_statement
 from libcst._nodes.statement import Import, ImportFrom, SimpleStatementLine
 from libcst.codemod._context import CodemodContext
 from libcst.codemod._visitor import ContextAwareTransformer
@@ -107,7 +107,7 @@ class AddImportsVisitor(ContextAwareTransformer):
     ) -> List[ImportItem]:
         imports = context.scratch.get(AddImportsVisitor.CONTEXT_KEY, [])
         if not isinstance(imports, list):
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
         return imports
 
     @staticmethod
@@ -136,7 +136,7 @@ class AddImportsVisitor(ContextAwareTransformer):
         """
 
         if module == "__future__" and obj is None:
-            raise Exception("Cannot import __future__ directly!")
+            raise ValueError("Cannot import __future__ directly!")
         imports = AddImportsVisitor._get_imports_from_context(context)
         imports.append(ImportItem(module, obj, asname, relative))
         context.scratch[AddImportsVisitor.CONTEXT_KEY] = imports
@@ -157,9 +157,9 @@ class AddImportsVisitor(ContextAwareTransformer):
         # Verify that the imports are valid
         for imp in imps:
             if imp.module == "__future__" and imp.obj_name is None:
-                raise Exception("Cannot import __future__ directly!")
+                raise ValueError("Cannot import __future__ directly!")
             if imp.module == "__future__" and imp.alias is not None:
-                raise Exception("Cannot import __future__ objects with aliases!")
+                raise ValueError("Cannot import __future__ objects with aliases!")
 
         # Resolve relative imports if we have a module name
         imps = [imp.resolve_relative(self.context.full_package_name) for imp in imps]

libcst/codemod/visitors/_apply_type_annotations.py

@@ -534,15 +534,20 @@ class _TypeCollectorDequalifier(cst.CSTTransformer):
     def __init__(self, type_collector: "TypeCollector") -> None:
         self.type_collector = type_collector
 
-    def leave_Name(self, original_node: cst.Name, updated_node: cst.Name) -> cst.Name:
+    def leave_Name(
+        self, original_node: cst.Name, updated_node: cst.Name
+    ) -> NameOrAttribute:
         qualified_name = _get_unique_qualified_name(self.type_collector, original_node)
         should_qualify = self.type_collector._handle_qualification_and_should_qualify(
             qualified_name, original_node
         )
         self.type_collector.annotations.names.add(qualified_name)
         if should_qualify:
-            qualified_node = cst.parse_module(qualified_name)
-            return qualified_node  # pyre-ignore[7]
+            parts = qualified_name.split(".")
+            qualified_node = cst.Name(parts[0])
+            for p in parts[1:]:
+                qualified_node = cst.Attribute(qualified_node, cst.Name(p))
+            return qualified_node
         else:
             return original_node
 

libcst/codemod/visitors/_gather_string_annotation_names.py

@@ -44,6 +44,11 @@ class GatherNamesFromStringAnnotationsVisitor(ContextAwareVisitor):
     def leave_Annotation(self, original_node: cst.Annotation) -> None:
         self._annotation_stack.pop()
 
+    def visit_Subscript(self, node: cst.Subscript) -> bool:
+        qnames = self.get_metadata(QualifiedNameProvider, node)
+        # A Literal["foo"] should not be interpreted as a use of the symbol "foo".
+        return not any(qn.name == "typing.Literal" for qn in qnames)
+
     def visit_Call(self, node: cst.Call) -> bool:
         qnames = self.get_metadata(QualifiedNameProvider, node)
         if any(qn.name in self._typing_functions for qn in qnames):
@@ -71,7 +76,11 @@ class GatherNamesFromStringAnnotationsVisitor(ContextAwareVisitor):
         value = node.evaluated_value
         if value is None:
             return
-        mod = cst.parse_module(value)
+        try:
+            mod = cst.parse_module(value)
+        except cst.ParserSyntaxError:
+            # Not all strings inside a type annotation are meant to be valid Python code.
+            return
         extracted_nodes = m.extractall(
             mod,
             m.Name(

libcst/codemod/visitors/_remove_imports.py

@@ -6,6 +6,7 @@
 from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
 
 import libcst as cst
+from libcst import CSTLogicError
 from libcst.codemod._context import CodemodContext
 from libcst.codemod._visitor import ContextAwareTransformer, ContextAwareVisitor
 from libcst.codemod.visitors._gather_unused_imports import GatherUnusedImportsVisitor
@@ -45,7 +46,7 @@ class RemovedNodeVisitor(ContextAwareVisitor):
             self.context.full_package_name, import_node
         )
         if module_name is None:
-            raise Exception("Cannot look up absolute module from relative import!")
+            raise ValueError("Cannot look up absolute module from relative import!")
 
         # We know any local names will refer to this as an alias if
         # there is one, and as the original name if there is not one
@@ -72,7 +73,9 @@ class RemovedNodeVisitor(ContextAwareVisitor):
         # Look up the scope for this node, remove the import that caused it to exist.
         metadata_wrapper = self.context.wrapper
         if metadata_wrapper is None:
-            raise Exception("Cannot look up import, metadata is not computed for node!")
+            raise ValueError(
+                "Cannot look up import, metadata is not computed for node!"
+            )
         scope_provider = metadata_wrapper.resolve(ScopeProvider)
         try:
             scope = scope_provider[node]
@@ -185,7 +188,7 @@ class RemoveImportsVisitor(ContextAwareTransformer):
     ) -> List[Tuple[str, Optional[str], Optional[str]]]:
         unused_imports = context.scratch.get(RemoveImportsVisitor.CONTEXT_KEY, [])
         if not isinstance(unused_imports, list):
-            raise Exception("Logic error!")
+            raise CSTLogicError("Logic error!")
         return unused_imports
 
     @staticmethod
@@ -255,7 +258,7 @@ class RemoveImportsVisitor(ContextAwareTransformer):
                 context.full_package_name, node
             )
             if module_name is None:
-                raise Exception("Cannot look up absolute module from relative import!")
+                raise ValueError("Cannot look up absolute module from relative import!")
             for import_alias in names:
                 RemoveImportsVisitor.remove_unused_import(
                     context,

libcst/codemod/visitors/tests/test_apply_type_annotations.py

@@ -61,6 +61,28 @@ class TestApplyAnnotationsVisitor(CodemodTest):
         )
         self.assertCodemod(before, after, context_override=context)
 
+    def run_test_case_twice(
+        self,
+        stub: str,
+        before: str,
+        after: str,
+    ) -> None:
+        context = CodemodContext()
+        ApplyTypeAnnotationsVisitor.store_stub_in_context(
+            context, parse_module(textwrap.dedent(stub.rstrip()))
+        )
+        r1 = ApplyTypeAnnotationsVisitor(context).transform_module(
+            parse_module(textwrap.dedent(before.rstrip()))
+        )
+
+        context = CodemodContext()
+        ApplyTypeAnnotationsVisitor.store_stub_in_context(
+            context, parse_module(textwrap.dedent(stub.rstrip()))
+        )
+        r2 = ApplyTypeAnnotationsVisitor(context).transform_module(r1)
+        assert r1.code == textwrap.dedent(after.rstrip())
+        assert r2.code == textwrap.dedent(after.rstrip())
+
     @data_provider(
         {
             "simple": (
@@ -1965,3 +1987,29 @@ class TestApplyAnnotationsVisitor(CodemodTest):
     )
     def test_no_duplicate_annotations(self, stub: str, before: str, after: str) -> None:
         self.run_simple_test_case(stub=stub, before=before, after=after)
+
+    @data_provider(
+        {
+            "qualifier_jank": (
+                """
+                from module.submodule import B
+                M: B
+                class Foo: ...
+                """,
+                """
+                from module import B
+                M = B()
+                class Foo: pass
+                """,
+                """
+                from module import B
+                import module.submodule
+
+                M: module.submodule.B = B()
+                class Foo: pass
+                """,
+            ),
+        }
+    )
+    def test_idempotent(self, stub: str, before: str, after: str) -> None:
+        self.run_test_case_twice(stub=stub, before=before, after=after)

libcst/codemod/visitors/tests/test_gather_string_annotation_names.py

@@ -80,3 +80,14 @@ class TestGatherNamesFromStringAnnotationsVisitor(UnitTest):
             visitor.names,
             {"api", "api.http_exceptions", "api.http_exceptions.HttpException"},
         )
+
+    def test_literals(self) -> None:
+        visitor = self.gather_names(
+            """
+            from typing import Literal
+            a: Literal["in"]
+            b: list[Literal["1x"]]
+            c: Literal["Any"]
+            """
+        )
+        self.assertEqual(visitor.names, set())

libcst/display/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from libcst.display.graphviz import dump_graphviz
+from libcst.display.text import dump
+
+__all__ = [
+    "dump",
+    "dump_graphviz",
+]

libcst/display/graphviz.py

@@ -0,0 +1,187 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+import textwrap
+from collections.abc import Sequence
+
+from libcst import CSTNode
+from libcst.helpers import filter_node_fields
+
+
+_syntax_style = ', color="#777777", fillcolor="#eeeeee"'
+_value_style = ', color="#3e99ed", fillcolor="#b8d9f8"'
+
+node_style: dict[str, str] = {
+    "__default__": "",
+    "EmptyLine": _syntax_style,
+    "IndentedBlock": _syntax_style,
+    "SimpleStatementLine": _syntax_style,
+    "SimpleWhitespace": _syntax_style,
+    "TrailingWhitespace": _syntax_style,
+    "Newline": _syntax_style,
+    "Comma": _syntax_style,
+    "LeftParen": _syntax_style,
+    "RightParen": _syntax_style,
+    "LeftSquareBracket": _syntax_style,
+    "RightSquareBracket": _syntax_style,
+    "LeftCurlyBrace": _syntax_style,
+    "RightCurlyBrace": _syntax_style,
+    "BaseSmallStatement": _syntax_style,
+    "BaseCompoundStatement": _syntax_style,
+    "SimpleStatementSuite": _syntax_style,
+    "Colon": _syntax_style,
+    "Dot": _syntax_style,
+    "Semicolon": _syntax_style,
+    "ParenthesizedWhitespace": _syntax_style,
+    "BaseParenthesizableWhitespace": _syntax_style,
+    "Comment": _syntax_style,
+    "Name": _value_style,
+    "Integer": _value_style,
+    "Float": _value_style,
+    "Imaginary": _value_style,
+    "SimpleString": _value_style,
+    "FormattedStringText": _value_style,
+}
+"""Graphviz style for specific CST nodes"""
+
+
+def _create_node_graphviz(node: CSTNode) -> str:
+    """Creates the graphviz representation of a CST node."""
+    node_name = node.__class__.__qualname__
+
+    if node_name in node_style:
+        style = node_style[node_name]
+    else:
+        style = node_style["__default__"]
+
+    # pyre-ignore[16]: the existence of node.value is checked before usage
+    if hasattr(node, "value") and isinstance(node.value, str):
+        line_break = r"\n"
+        quote = '"'
+        escaped_quote = r"\""
+        value = f"{line_break}<{node.value.replace(quote, escaped_quote)}>"
+        style = style + ', shape="box"'
+    else:
+        value = ""
+
+    return f'{id(node)} [label="{node_name}{value}"{style}]'
+
+
+def _node_repr_recursive(
+    node: object,
+    *,
+    show_defaults: bool,
+    show_syntax: bool,
+    show_whitespace: bool,
+) -> list[str]:
+    """Creates the graphviz representation of a CST node,
+    and of its child nodes."""
+    if not isinstance(node, CSTNode):
+        return []
+
+    fields = filter_node_fields(
+        node,
+        show_defaults=show_defaults,
+        show_syntax=show_syntax,
+        show_whitespace=show_whitespace,
+    )
+
+    graphviz_lines: list[str] = [_create_node_graphviz(node)]
+
+    for field in fields:
+        value = getattr(node, field.name)
+        if isinstance(value, CSTNode):
+            # Display a single node
+            graphviz_lines.append(f'{id(node)} -> {id(value)} [label="{field.name}"]')
+            graphviz_lines.extend(
+                _node_repr_recursive(
+                    value,
+                    show_defaults=show_defaults,
+                    show_syntax=show_syntax,
+                    show_whitespace=show_whitespace,
+                )
+            )
+            continue
+
+        if isinstance(value, Sequence):
+            # Display a sequence of nodes
+            for index, child in enumerate(value):
+                if isinstance(child, CSTNode):
+                    graphviz_lines.append(
+                        rf'{id(node)} -> {id(child)} [label="{field.name}[{index}]"]'
+                    )
+                    graphviz_lines.extend(
+                        _node_repr_recursive(
+                            child,
+                            show_defaults=show_defaults,
+                            show_syntax=show_syntax,
+                            show_whitespace=show_whitespace,
+                        )
+                    )
+
+    return graphviz_lines
+
+
+def dump_graphviz(
+    node: object,
+    *,
+    show_defaults: bool = False,
+    show_syntax: bool = False,
+    show_whitespace: bool = False,
+) -> str:
+    """
+    Returns a string representation (in graphviz .dot style) of a CST node,
+    and its child nodes.
+
+    Setting ``show_defaults`` to ``True`` will add fields regardless if their
+    value is different from the default value.
+
+    Setting ``show_whitespace`` will add whitespace fields and setting
+    ``show_syntax`` will add syntax fields while respecting the value of
+    ``show_defaults``.
+    """
+
+    graphviz_settings = textwrap.dedent(
+        r"""
+        layout=dot;
+        rankdir=TB;
+        splines=line;
+        ranksep=0.5;
+        nodesep=1.0;
+        dpi=300;
+        bgcolor=transparent;
+        node [
+            style=filled,
+            color="#fb8d3f",
+            fontcolor="#4b4f54",
+            fillcolor="#fdd2b3",
+            fontname="Source Code Pro Semibold",
+            penwidth="2",
+            group=main,
+        ];
+        edge [
+            color="#999999",
+            fontcolor="#4b4f54",
+            fontname="Source Code Pro Semibold",
+            fontsize=12,
+            penwidth=2,
+        ];
+        """[
+            1:
+        ]
+    )
+
+    return "\n".join(
+        ["digraph {", graphviz_settings]
+        + _node_repr_recursive(
+            node,
+            show_defaults=show_defaults,
+            show_syntax=show_syntax,
+            show_whitespace=show_whitespace,
+        )
+        + ["}"]
+    )

libcst/display/tests/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.

libcst/display/tests/test_dump_graphviz.py

@@ -0,0 +1,83 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+from textwrap import dedent
+from typing import TYPE_CHECKING
+
+from libcst import parse_module
+from libcst.display import dump_graphviz
+from libcst.testing.utils import UnitTest
+
+if TYPE_CHECKING:
+    from libcst import Module
+
+
+class CSTDumpGraphvizTest(UnitTest):
+    """Check dump_graphviz contains CST nodes."""
+
+    source_code: str = dedent(
+        r"""
+        def foo(a: str) -> None:
+            pass ;
+            pass
+            return
+        """[
+            1:
+        ]
+    )
+    cst: Module
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.cst = parse_module(cls.source_code)
+
+    def _assert_node(self, node_name: str, graphviz_str: str) -> None:
+        self.assertIn(
+            node_name, graphviz_str, f"No node {node_name} found in graphviz_dump"
+        )
+
+    def _check_essential_nodes_in_tree(self, graphviz_str: str) -> None:
+        # Check CST nodes are present in graphviz string
+        self._assert_node("Module", graphviz_str)
+        self._assert_node("FunctionDef", graphviz_str)
+        self._assert_node("Name", graphviz_str)
+        self._assert_node("Parameters", graphviz_str)
+        self._assert_node("Param", graphviz_str)
+        self._assert_node("Annotation", graphviz_str)
+        self._assert_node("IndentedBlock", graphviz_str)
+        self._assert_node("SimpleStatementLine", graphviz_str)
+        self._assert_node("Pass", graphviz_str)
+        self._assert_node("Return", graphviz_str)
+
+        # Check CST values are present in graphviz string
+        self._assert_node("<foo>", graphviz_str)
+        self._assert_node("<a>", graphviz_str)
+        self._assert_node("<str>", graphviz_str)
+        self._assert_node("<None>", graphviz_str)
+
+    def test_essential_tree(self) -> None:
+        """Check essential nodes are present in the CST graphviz dump."""
+        graphviz_str = dump_graphviz(self.cst)
+        self._check_essential_nodes_in_tree(graphviz_str)
+
+    def test_full_tree(self) -> None:
+        """Check all nodes are present in the CST graphviz dump."""
+        graphviz_str = dump_graphviz(
+            self.cst,
+            show_whitespace=True,
+            show_defaults=True,
+            show_syntax=True,
+        )
+        self._check_essential_nodes_in_tree(graphviz_str)
+
+        self._assert_node("Semicolon", graphviz_str)
+        self._assert_node("SimpleWhitespace", graphviz_str)
+        self._assert_node("Newline", graphviz_str)
+        self._assert_node("TrailingWhitespace", graphviz_str)
+
+        self._assert_node("<>", graphviz_str)
+        self._assert_node("< >", graphviz_str)

libcst/display/tests/test_dump_text.py

@@ -10,7 +10,7 @@ from libcst.testing.utils import UnitTest
 from libcst.tool import dump
 
 
-class PrettyPrintNodesTest(UnitTest):
+class CSTDumpTextTest(UnitTest):
     def test_full_tree(self) -> None:
         module = r"""
             Module(

libcst/display/text.py

@@ -0,0 +1,133 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+import dataclasses
+from typing import List, Sequence
+
+from libcst import CSTLogicError, CSTNode
+from libcst.helpers import filter_node_fields
+
+_DEFAULT_INDENT: str = "  "
+
+
+def _node_repr_recursive(  # noqa: C901
+    node: object,
+    *,
+    indent: str = _DEFAULT_INDENT,
+    show_defaults: bool = False,
+    show_syntax: bool = False,
+    show_whitespace: bool = False,
+) -> List[str]:
+    if isinstance(node, CSTNode):
+        # This is a CSTNode, we must pretty-print it.
+        fields: Sequence[dataclasses.Field[CSTNode]] = filter_node_fields(
+            node=node,
+            show_defaults=show_defaults,
+            show_syntax=show_syntax,
+            show_whitespace=show_whitespace,
+        )
+
+        tokens: List[str] = [node.__class__.__name__]
+
+        if len(fields) == 0:
+            tokens.append("()")
+        else:
+            tokens.append("(\n")
+
+            for field in fields:
+                child_tokens: List[str] = [field.name, "="]
+                value = getattr(node, field.name)
+
+                if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
+                    # Render out the node contents
+                    child_tokens.extend(
+                        _node_repr_recursive(
+                            value,
+                            indent=indent,
+                            show_whitespace=show_whitespace,
+                            show_defaults=show_defaults,
+                            show_syntax=show_syntax,
+                        )
+                    )
+                elif isinstance(value, Sequence):
+                    # Render out a list of individual nodes
+                    if len(value) > 0:
+                        child_tokens.append("[\n")
+                        list_tokens: List[str] = []
+
+                        last_value = len(value) - 1
+                        for j, v in enumerate(value):
+                            list_tokens.extend(
+                                _node_repr_recursive(
+                                    v,
+                                    indent=indent,
+                                    show_whitespace=show_whitespace,
+                                    show_defaults=show_defaults,
+                                    show_syntax=show_syntax,
+                                )
+                            )
+                            if j != last_value:
+                                list_tokens.append(",\n")
+                            else:
+                                list_tokens.append(",")
+
+                        split_by_line = "".join(list_tokens).split("\n")
+                        child_tokens.append(
+                            "\n".join(f"{indent}{t}" for t in split_by_line)
+                        )
+
+                        child_tokens.append("\n]")
+                    else:
+                        child_tokens.append("[]")
+                else:
+                    raise CSTLogicError("Logic error!")
+
+                # Handle indentation and trailing comma.
+                split_by_line = "".join(child_tokens).split("\n")
+                tokens.append("\n".join(f"{indent}{t}" for t in split_by_line))
+                tokens.append(",\n")
+
+            tokens.append(")")
+
+        return tokens
+    else:
+        # This is a python value, just return the repr
+        return [repr(node)]
+
+
+def dump(
+    node: CSTNode,
+    *,
+    indent: str = _DEFAULT_INDENT,
+    show_defaults: bool = False,
+    show_syntax: bool = False,
+    show_whitespace: bool = False,
+) -> str:
+    """
+    Returns a string representation of the node that contains minimal differences
+    from the default contruction of the node while also hiding whitespace and
+    syntax fields.
+
+    Setting ``show_defaults`` to ``True`` will add fields regardless if their
+    value is different from the default value.
+
+    Setting ``show_whitespace`` will add whitespace fields and setting
+    ``show_syntax`` will add syntax fields while respecting the value of
+    ``show_defaults``.
+
+    When all keyword args are set to true, the output of this function is
+    indentical to the __repr__ method of the node.
+    """
+    return "".join(
+        _node_repr_recursive(
+            node,
+            indent=indent,
+            show_defaults=show_defaults,
+            show_syntax=show_syntax,
+            show_whitespace=show_whitespace,
+        )
+    )

libcst/helpers/__init__.py

@@ -25,6 +25,14 @@ from libcst.helpers.module import (
     insert_header_comments,
     ModuleNameAndPackage,
 )
+from libcst.helpers.node_fields import (
+    filter_node_fields,
+    get_field_default_value,
+    get_node_fields,
+    is_default_node_field,
+    is_syntax_node_field,
+    is_whitespace_node_field,
+)
 
 __all__ = [
     "calculate_module_and_package",
@@ -42,4 +50,10 @@ __all__ = [
     "parse_template_statement",
     "parse_template_expression",
     "ModuleNameAndPackage",
+    "get_node_fields",
+    "get_field_default_value",
+    "is_whitespace_node_field",
+    "is_syntax_node_field",
+    "is_default_node_field",
+    "filter_node_fields",
 ]

libcst/helpers/_template.py

@@ -45,12 +45,12 @@ def unmangled_name(var: str) -> Optional[str]:
 
 def mangle_template(template: str, template_vars: Set[str]) -> str:
     if TEMPLATE_PREFIX in template or TEMPLATE_SUFFIX in template:
-        raise Exception("Cannot parse a template containing reserved strings")
+        raise ValueError("Cannot parse a template containing reserved strings")
 
     for var in template_vars:
         original = f"{{{var}}}"
         if original not in template:
-            raise Exception(
+            raise ValueError(
                 f'Template string is missing a reference to "{var}" referred to in kwargs'
             )
         template = template.replace(original, mangled_name(var))
@@ -142,7 +142,7 @@ class TemplateTransformer(cst.CSTTransformer):
             name for name in template_replacements if name not in supported_vars
         }
         if unsupported_vars:
-            raise Exception(
+            raise ValueError(
                 f'Template replacement for "{next(iter(unsupported_vars))}" is unsupported'
             )
 
@@ -350,7 +350,7 @@ class TemplateChecker(cst.CSTVisitor):
     def visit_Name(self, node: cst.Name) -> None:
         for var in self.template_vars:
             if node.value == mangled_name(var):
-                raise Exception(f'Template variable "{var}" was not replaced properly')
+                raise ValueError(f'Template variable "{var}" was not replaced properly')
 
 
 def unmangle_nodes(
@@ -424,8 +424,8 @@ def parse_template_statement(
     if not isinstance(
         new_statement, (cst.SimpleStatementLine, cst.BaseCompoundStatement)
     ):
-        raise Exception(
-            f"Expected a statement but got a {new_statement.__class__.__name__}!"
+        raise TypeError(
+            f"Expected a statement but got a {new_statement.__class__.__qualname__}!"
         )
     new_statement.visit(TemplateChecker({name for name in template_replacements}))
     return new_statement

libcst/helpers/common.py

@@ -19,7 +19,7 @@ def ensure_type(node: object, nodetype: Type[T]) -> T:
     """
 
     if not isinstance(node, nodetype):
-        raise Exception(
-            f"Expected a {nodetype.__name__} but got a {node.__class__.__name__}!"
+        raise ValueError(
+            f"Expected a {nodetype.__name__} but got a {node.__class__.__qualname__}!"
         )
     return node

libcst/helpers/expression.py

@@ -38,5 +38,5 @@ def get_full_name_for_node_or_raise(node: Union[str, cst.CSTNode]) -> str:
     """
     full_name = get_full_name_for_node(node)
     if full_name is None:
-        raise Exception(f"Not able to parse full name for: {node}")
+        raise ValueError(f"Not able to parse full name for: {node}")
     return full_name

libcst/helpers/matchers.py

@@ -0,0 +1,45 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+from dataclasses import fields, is_dataclass, MISSING
+
+from libcst import matchers
+from libcst._nodes.base import CSTNode
+
+
+def node_to_matcher(
+    node: CSTNode, *, match_syntactic_trivia: bool = False
+) -> matchers.BaseMatcherNode:
+    """Convert a concrete node to a matcher."""
+    if not is_dataclass(node):
+        raise ValueError(f"{node} is not a CSTNode")
+
+    attrs = {}
+    for field in fields(node):
+        name = field.name
+        child = getattr(node, name)
+        if not match_syntactic_trivia and field.name.startswith("whitespace"):
+            # Not all nodes have whitespace fields, some have multiple, but they all
+            # start with whitespace*
+            child = matchers.DoNotCare()
+        elif field.default is not MISSING and child == field.default:
+            child = matchers.DoNotCare()
+        # pyre-ignore[29]: Union[MISSING_TYPE, ...] is not a function.
+        elif field.default_factory is not MISSING and child == field.default_factory():
+            child = matchers.DoNotCare()
+        elif isinstance(child, (list, tuple)):
+            child = type(child)(
+                node_to_matcher(item, match_syntactic_trivia=match_syntactic_trivia)
+                for item in child
+            )
+        elif hasattr(matchers, type(child).__name__):
+            child = node_to_matcher(
+                child, match_syntactic_trivia=match_syntactic_trivia
+            )
+        attrs[name] = child
+
+    matcher = getattr(matchers, type(node).__name__)
+    return matcher(**attrs)

libcst/helpers/module.py

@@ -5,7 +5,7 @@
 #
 from dataclasses import dataclass
 from itertools import islice
-from pathlib import PurePath
+from pathlib import Path, PurePath
 from typing import List, Optional
 
 from libcst import Comment, EmptyLine, ImportFrom, Module
@@ -80,7 +80,7 @@ def get_absolute_module_for_import_or_raise(
 ) -> str:
     module = get_absolute_module_for_import(current_module, import_node)
     if module is None:
-        raise Exception(f"Unable to compute absolute module for {import_node}")
+        raise ValueError(f"Unable to compute absolute module for {import_node}")
     return module
 
 
@@ -121,7 +121,7 @@ def get_absolute_module_from_package_for_import_or_raise(
 ) -> str:
     module = get_absolute_module_from_package_for_import(current_package, import_node)
     if module is None:
-        raise Exception(f"Unable to compute absolute module for {import_node}")
+        raise ValueError(f"Unable to compute absolute module for {import_node}")
     return module
 
 
@@ -132,11 +132,25 @@ class ModuleNameAndPackage:
 
 
 def calculate_module_and_package(
-    repo_root: StrPath, filename: StrPath
+    repo_root: StrPath, filename: StrPath, use_pyproject_toml: bool = False
 ) -> ModuleNameAndPackage:
     # Given an absolute repo_root and an absolute filename, calculate the
     # python module name for the file.
-    relative_filename = PurePath(filename).relative_to(repo_root)
+    if use_pyproject_toml:
+        # But also look for pyproject.toml files, indicating nested packages in the repo.
+        abs_repo_root = Path(repo_root).resolve()
+        abs_filename = Path(filename).resolve()
+        package_root = abs_filename.parent
+        while package_root != abs_repo_root:
+            if (package_root / "pyproject.toml").exists():
+                break
+            if package_root == package_root.parent:
+                break
+            package_root = package_root.parent
+
+        relative_filename = abs_filename.relative_to(package_root)
+    else:
+        relative_filename = PurePath(filename).relative_to(repo_root)
     relative_filename = relative_filename.with_suffix("")
 
     # handle special cases

libcst/helpers/node_fields.py

@@ -0,0 +1,128 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING
+
+from libcst import IndentedBlock, Module
+from libcst._nodes.deep_equals import deep_equals
+
+if TYPE_CHECKING:
+    from typing import Sequence
+
+    from libcst import CSTNode
+
+
+def get_node_fields(node: CSTNode) -> Sequence[dataclasses.Field[CSTNode]]:
+    """
+    Returns the sequence of a given CST-node's fields.
+    """
+    return dataclasses.fields(node)
+
+
+def is_whitespace_node_field(node: CSTNode, field: dataclasses.Field[CSTNode]) -> bool:
+    """
+    Returns True if a given CST-node's field is a whitespace-related field
+    (whitespace, indent, header, footer, etc.).
+    """
+    if "whitespace" in field.name:
+        return True
+    if "leading_lines" in field.name:
+        return True
+    if "lines_after_decorators" in field.name:
+        return True
+    if isinstance(node, (IndentedBlock, Module)) and field.name in [
+        "header",
+        "footer",
+    ]:
+        return True
+    if isinstance(node, IndentedBlock) and field.name == "indent":
+        return True
+    return False
+
+
+def is_syntax_node_field(node: CSTNode, field: dataclasses.Field[CSTNode]) -> bool:
+    """
+    Returns True if a given CST-node's field is a syntax-related field
+    (colon, semicolon, dot, encoding, etc.).
+    """
+    if isinstance(node, Module) and field.name in [
+        "encoding",
+        "default_indent",
+        "default_newline",
+        "has_trailing_newline",
+    ]:
+        return True
+    type_str = repr(field.type)
+    if (
+        "Sentinel" in type_str
+        and field.name not in ["star_arg", "star", "posonly_ind"]
+        and "whitespace" not in field.name
+    ):
+        # This is a value that can optionally be specified, so its
+        # definitely syntax.
+        return True
+
+    for name in ["Semicolon", "Colon", "Comma", "Dot", "AssignEqual"]:
+        # These are all nodes that exist for separation syntax
+        if name in type_str:
+            return True
+
+    return False
+
+
+def get_field_default_value(field: dataclasses.Field[CSTNode]) -> object:
+    """
+    Returns the default value of a CST-node's field.
+    """
+    if field.default_factory is not dataclasses.MISSING:
+        # pyre-fixme[29]: `Union[dataclasses._MISSING_TYPE,
+        #  dataclasses._DefaultFactory[object]]` is not a function.
+        return field.default_factory()
+    return field.default
+
+
+def is_default_node_field(node: CSTNode, field: dataclasses.Field[CSTNode]) -> bool:
+    """
+    Returns True if a given CST-node's field has its default value.
+    """
+    return deep_equals(getattr(node, field.name), get_field_default_value(field))
+
+
+def filter_node_fields(
+    node: CSTNode,
+    *,
+    show_defaults: bool,
+    show_syntax: bool,
+    show_whitespace: bool,
+) -> Sequence[dataclasses.Field[CSTNode]]:
+    """
+    Returns a filtered sequence of a CST-node's fields.
+
+    Setting ``show_whitespace`` to ``False`` will filter whitespace fields.
+
+    Setting ``show_defaults`` to ``False`` will filter fields if their value is equal to
+    the default value ;  while respecting  the value of ``show_whitespace``.
+
+    Setting ``show_syntax``  to ``False`` will filter syntax fields ; while respecting
+    the value of ``show_whitespace`` & ``show_defaults``.
+    """
+
+    fields: Sequence[dataclasses.Field[CSTNode]] = dataclasses.fields(node)
+    # Hide all fields prefixed with "_"
+    fields = [f for f in fields if f.name[0] != "_"]
+    # Filter whitespace nodes if needed
+    if not show_whitespace:
+        fields = [f for f in fields if not is_whitespace_node_field(node, f)]
+    # Filter values which aren't changed from their defaults
+    if not show_defaults:
+        fields = [f for f in fields if not is_default_node_field(node, f)]
+    # Filter out values which aren't interesting if needed
+    if not show_syntax:
+        fields = [f for f in fields if not is_syntax_node_field(node, f)]
+
+    return fields