Run ecosystem checks with preview mode enabled (#8358)

Until https://github.com/astral-sh/ruff/issues/8076 is ready, it seems
beneficial to get feedback on preview mode changes.

Tested locally, updated logs to output the flags passed to `ruff` and
verified `--preview` is used.

.github/workflows/ci.yaml

@@ -218,7 +218,7 @@ jobs:
           # Set pipefail to avoid hiding errors with tee
           set -eo pipefail
 
-          ruff-ecosystem check ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown | tee ecosystem-result-check
+          ruff-ecosystem check ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown --force-preview | tee ecosystem-result-check
 
           cat ecosystem-result-check > $GITHUB_STEP_SUMMARY
           cat ecosystem-result-check > ecosystem-result
@@ -233,7 +233,7 @@ jobs:
           # Set pipefail to avoid hiding errors with tee
           set -eo pipefail
 
-          ruff-ecosystem format ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown | tee ecosystem-result-format
+          ruff-ecosystem format ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown --force-preview | tee ecosystem-result-format
 
           cat ecosystem-result-format > $GITHUB_STEP_SUMMARY
           cat ecosystem-result-format >> ecosystem-result

python/ruff-ecosystem/ruff_ecosystem/check.py

@@ -25,11 +25,10 @@ from ruff_ecosystem.types import (
     Diff,
     Result,
     RuffError,
-    Serializable,
 )
 
 if TYPE_CHECKING:
-    from ruff_ecosystem.projects import ClonedRepository, Project
+    from ruff_ecosystem.projects import CheckOptions, ClonedRepository, Project
 
 
 # Matches lines that are summaries rather than diagnostics
@@ -501,8 +500,8 @@ async def ruff_check(
     *, executable: Path, path: Path, name: str, options: CheckOptions
 ) -> Sequence[str]:
     """Run the given ruff binary against the specified path."""
-    logger.debug(f"Checking {name} with {executable}")
     ruff_args = options.to_cli_args()
+    logger.debug(f"Checking {name} with {executable} " + " ".join(ruff_args))
 
     start = time.time()
     proc = await create_subprocess_exec(
@@ -529,35 +528,3 @@ async def ruff_check(
     ]
 
     return lines
-
-
-@dataclass(frozen=True)
-class CheckOptions(Serializable):
-    """
-    Ruff check options
-    """
-
-    select: str = ""
-    ignore: str = ""
-    exclude: str = ""
-
-    # Generating fixes is slow and verbose
-    show_fixes: bool = False
-
-    # Limit the number of reported lines per rule
-    max_lines_per_rule: int | None = 50
-
-    def markdown(self) -> str:
-        return f"select {self.select} ignore {self.ignore} exclude {self.exclude}"
-
-    def to_cli_args(self) -> list[str]:
-        args = ["check", "--no-cache", "--exit-zero"]
-        if self.select:
-            args.extend(["--select", self.select])
-        if self.ignore:
-            args.extend(["--ignore", self.ignore])
-        if self.exclude:
-            args.extend(["--exclude", self.exclude])
-        if self.show_fixes:
-            args.extend(["--show-fixes", "--ecosystem-ci"])
-        return args

python/ruff-ecosystem/ruff_ecosystem/cli.py

@@ -73,6 +73,10 @@ def entrypoint():
             ruff_comparison,
         )
 
+    targets = DEFAULT_TARGETS
+    if args.force_preview:
+        targets = [target.with_preview_enabled() for target in targets]
+
     with cache_context as cache:
         loop = asyncio.get_event_loop()
         main_task = asyncio.ensure_future(
@@ -80,7 +84,7 @@ def entrypoint():
                 command=RuffCommand(args.ruff_command),
                 ruff_baseline_executable=ruff_baseline,
                 ruff_comparison_executable=ruff_comparison,
-                targets=DEFAULT_TARGETS,
+                targets=targets,
                 format=OutputFormat(args.output_format),
                 project_dir=Path(cache),
                 raise_on_failure=args.pdb,
@@ -131,6 +135,11 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable debugging on failure",
     )
+    parser.add_argument(
+        "--force-preview",
+        action="store_true",
+        help="Force preview mode to be enabled for all projects",
+    )
     parser.add_argument(
         "ruff_command",
         choices=[option.name for option in RuffCommand],

python/ruff-ecosystem/ruff_ecosystem/format.py

@@ -6,7 +6,6 @@ from __future__ import annotations
 
 import time
 from asyncio import create_subprocess_exec
-from dataclasses import dataclass
 from pathlib import Path
 from subprocess import PIPE
 from typing import TYPE_CHECKING, Sequence
@@ -18,7 +17,7 @@ from ruff_ecosystem.markdown import markdown_project_section
 from ruff_ecosystem.types import Comparison, Diff, Result, RuffError
 
 if TYPE_CHECKING:
-    from ruff_ecosystem.projects import ClonedRepository
+    from ruff_ecosystem.projects import ClonedRepository, FormatOptions
 
 
 def markdown_format_result(result: Result) -> str:
@@ -153,8 +152,8 @@ async def ruff_format(
     diff: bool = False,
 ) -> Sequence[str]:
     """Run the given ruff binary against the specified path."""
-    logger.debug(f"Formatting {name} with {executable}")
     ruff_args = options.to_cli_args()
+    logger.debug(f"Formatting {name} with {executable} " + " ".join(ruff_args))
 
     if diff:
         ruff_args.append("--diff")
@@ -178,18 +177,3 @@ async def ruff_format(
 
     lines = result.decode("utf8").splitlines()
     return lines
-
-
-@dataclass(frozen=True)
-class FormatOptions:
-    """
-    Ruff format options.
-    """
-
-    exclude: str = ""
-
-    def to_cli_args(self) -> list[str]:
-        args = ["format"]
-        if self.exclude:
-            args.extend(["--exclude", self.exclude])
-        return args

python/ruff-ecosystem/ruff_ecosystem/projects.py

@@ -4,6 +4,8 @@ Abstractions and utilities for working with projects to run ecosystem checks on.
 
 from __future__ import annotations
 
+import abc
+import dataclasses
 from asyncio import create_subprocess_exec
 from dataclasses import dataclass, field
 from enum import Enum
@@ -12,8 +14,6 @@ from subprocess import PIPE
 from typing import Self
 
 from ruff_ecosystem import logger
-from ruff_ecosystem.check import CheckOptions
-from ruff_ecosystem.format import FormatOptions
 from ruff_ecosystem.types import Serializable
 
 
@@ -27,12 +27,82 @@ class Project(Serializable):
     check_options: CheckOptions = field(default_factory=lambda: CheckOptions())
     format_options: FormatOptions = field(default_factory=lambda: FormatOptions())
 
+    def with_preview_enabled(self: Self) -> Self:
+        return type(self)(
+            repo=self.repo,
+            check_options=self.check_options.with_options(preview=True),
+            format_options=self.format_options.with_options(preview=True),
+        )
+
 
 class RuffCommand(Enum):
     check = "check"
     format = "format"
 
 
+@dataclass(frozen=True)
+class CommandOptions(Serializable, abc.ABC):
+    def with_options(self: Self, **kwargs) -> Self:
+        """
+        Return a copy of self with the given options set.
+        """
+        return type(self)(**{**dataclasses.asdict(self), **kwargs})
+
+    @abc.abstractmethod
+    def to_cli_args(self) -> list[str]:
+        pass
+
+
+@dataclass(frozen=True)
+class CheckOptions(CommandOptions):
+    """
+    Ruff check options
+    """
+
+    select: str = ""
+    ignore: str = ""
+    exclude: str = ""
+    preview: bool = False
+
+    # Generating fixes is slow and verbose
+    show_fixes: bool = False
+
+    # Limit the number of reported lines per rule
+    max_lines_per_rule: int | None = 50
+
+    def to_cli_args(self) -> list[str]:
+        args = ["check", "--no-cache", "--exit-zero"]
+        if self.select:
+            args.extend(["--select", self.select])
+        if self.ignore:
+            args.extend(["--ignore", self.ignore])
+        if self.exclude:
+            args.extend(["--exclude", self.exclude])
+        if self.show_fixes:
+            args.extend(["--show-fixes", "--ecosystem-ci"])
+        if self.preview:
+            args.append("--preview")
+        return args
+
+
+@dataclass(frozen=True)
+class FormatOptions(CommandOptions):
+    """
+    Ruff format options.
+    """
+
+    preview: bool = False
+    exclude: str = ""
+
+    def to_cli_args(self) -> list[str]:
+        args = ["format"]
+        if self.exclude:
+            args.extend(["--exclude", self.exclude])
+        if self.preview:
+            args.append("--preview")
+        return args
+
+
 class ProjectSetupError(Exception):
     """An error setting up a project."""