Enable benchmarking of uv tool and pipx (#5531)

## Summary

Closes https://github.com/astral-sh/uv/issues/5263.

BENCHMARKS.md

@@ -67,7 +67,7 @@ The benchmark script itself has a several requirements:
 To benchmark resolution against pip-compile, Poetry, and PDM:
 
 ```shell
-uv run benchmark \
+uv run resolver \
     --uv-pip \
     --poetry \
     --pdm \
@@ -80,7 +80,7 @@ uv run benchmark \
 To benchmark installation against pip-sync, Poetry, and PDM:
 
 ```shell
-uv run benchmark \
+uv run resolver \
     --uv-pip \
     --poetry \
     --pdm \

CONTRIBUTING.md

@@ -86,7 +86,7 @@ We provide diverse sets of requirements for testing and benchmarking the resolve
 You can use `scripts/benchmark` to benchmark predefined workloads between uv versions and with other tools, e.g., from the `scripts/benchmark` directory:
 
 ```shell
-uv run benchmark \
+uv run resolver \
     --uv-pip \
     --poetry \
     --benchmark \

scripts/benchmark/README.md

@@ -7,7 +7,7 @@ Benchmarking scripts for uv and other package management tools.
 From the `scripts/benchmark` directory:
 
 ```shell
-uv run benchmark \
+uv run resolver \
     --uv-pip \
     --poetry \
     --benchmark \

scripts/benchmark/pyproject.toml

@@ -10,10 +10,12 @@ dependencies = [
   "tomli",
   "tomli_w",
   "virtualenv",
+  "pipx",
 ]
 
 [project.scripts]
-benchmark = "benchmark:main"
+resolver = "benchmark.resolver:main"
+tools = "benchmark.tools:main"
 
 [build-system]
 requires = ["hatchling"]

scripts/benchmark/src/benchmark/tools.py

@@ -0,0 +1,338 @@
+"""Benchmark the uv `tool` interface against other packaging tools.
+
+For example, to benchmark uv against pipx, run the following from the
+`scripts/benchmark` directory:
+
+    uv run tools --uv --pipx
+"""
+
+import abc
+import argparse
+import enum
+import logging
+import os.path
+import tempfile
+
+from benchmark import Command, Hyperfine
+
+TOOL = "flask"
+
+
+class Benchmark(enum.Enum):
+    """Enumeration of the benchmarks to run."""
+
+    INSTALL_COLD = "install-cold"
+    INSTALL_WARM = "install-warm"
+    RUN = "run"
+
+
+class Suite(abc.ABC):
+    """Abstract base class for packaging tools."""
+
+    def command(self, benchmark: Benchmark, *, cwd: str) -> Command | None:
+        """Generate a command to benchmark a given tool."""
+        match benchmark:
+            case Benchmark.INSTALL_COLD:
+                return self.install_cold(cwd=cwd)
+            case Benchmark.INSTALL_WARM:
+                return self.install_warm(cwd=cwd)
+            case Benchmark.RUN:
+                return self.run(cwd=cwd)
+            case _:
+                raise ValueError(f"Invalid benchmark: {benchmark}")
+
+    @abc.abstractmethod
+    def install_cold(self, *, cwd: str) -> Command | None:
+        """Resolve a set of dependencies using pip-tools, from a cold cache.
+
+        The resolution is performed from scratch, i.e., without an existing lockfile,
+        and the cache directory is cleared between runs.
+        """
+
+    @abc.abstractmethod
+    def install_warm(self, *, cwd: str) -> Command | None:
+        """Resolve a set of dependencies using pip-tools, from a warm cache.
+
+        The resolution is performed from scratch, i.e., without an existing lockfile;
+        however, the cache directory is _not_ cleared between runs.
+        """
+
+    @abc.abstractmethod
+    def run(self, *, cwd: str) -> Command | None:
+        """Resolve a modified lockfile using pip-tools, from a warm cache.
+
+        The resolution is performed with an existing lockfile, and the cache directory
+        is _not_ cleared between runs. However, a new dependency is added to the set
+        of input requirements, which does not appear in the lockfile.
+        """
+
+
+class Pipx(Suite):
+    def __init__(self, path: str | None = None) -> None:
+        self.name = path or "pipx"
+        self.path = path or "pipx"
+
+    def install_cold(self, *, cwd: str) -> Command | None:
+        home_dir = os.path.join(cwd, "home")
+        bin_dir = os.path.join(cwd, "bin")
+        man_dir = os.path.join(cwd, "man")
+
+        # pipx uses a shared virtualenv directory in `${PIPX_HOME}/shared`, which
+        # contains pip. If we remove `${PIPX_HOME}/shared`, we're simulating the _first_
+        # pipx invocation on a machine, rather than `pipx run` with a cold cache. So,
+        # instead, we only remove the installed tools, rather than the shared
+        # dependencies.
+        venvs_dir = os.path.join(home_dir, "venvs")
+
+        return Command(
+            name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
+            prepare=f"rm -rf {venvs_dir} && rm -rf {bin_dir} && rm -rf {man_dir}",
+            command=[
+                f"PIPX_HOME={home_dir}",
+                f"PIPX_BIN_DIR={bin_dir}",
+                f"PIPX_MAN_DIR={man_dir}",
+                self.path,
+                "install",
+                "--pip-args=--no-cache-dir",
+                TOOL,
+            ],
+        )
+
+    def install_warm(self, *, cwd: str) -> Command | None:
+        home_dir = os.path.join(cwd, "home")
+        bin_dir = os.path.join(cwd, "bin")
+        man_dir = os.path.join(cwd, "man")
+
+        # pipx uses a shared virtualenv directory in `${PIPX_HOME}/shared`, which
+        # contains pip. If we remove `${PIPX_HOME}/shared`, we're simulating the _first_
+        # pipx invocation on a machine, rather than `pipx run` with a cold cache. So,
+        # instead, we only remove the installed tools, rather than the shared
+        # dependencies.
+        venvs_dir = os.path.join(home_dir, "venvs")
+
+        return Command(
+            name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
+            prepare=f"rm -rf {venvs_dir} && rm -rf {bin_dir} && rm -rf {man_dir}",
+            command=[
+                f"PIPX_HOME={home_dir}",
+                f"PIPX_BIN_DIR={bin_dir}",
+                f"PIPX_MAN_DIR={man_dir}",
+                self.path,
+                "install",
+                TOOL,
+            ],
+        )
+
+    def run(self, *, cwd: str) -> Command | None:
+        home_dir = os.path.join(cwd, "home")
+        bin_dir = os.path.join(cwd, "bin")
+        man_dir = os.path.join(cwd, "man")
+
+        return Command(
+            name=f"{self.name} ({Benchmark.RUN.value})",
+            prepare="",
+            command=[
+                f"PIPX_HOME={home_dir}",
+                f"PIPX_BIN_DIR={bin_dir}",
+                f"PIPX_MAN_DIR={man_dir}",
+                self.path,
+                "install",
+                TOOL,
+            ],
+        )
+
+
+class Uv(Suite):
+    def __init__(self, *, path: str | None = None) -> Command | None:
+        """Initialize a uv benchmark."""
+        self.name = path or "uv"
+        self.path = path or os.path.join(
+            os.path.dirname(
+                os.path.dirname(
+                    os.path.dirname(
+                        os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+                    )
+                )
+            ),
+            "target",
+            "release",
+            "uv",
+        )
+
+    def install_cold(self, *, cwd: str) -> Command | None:
+        bin_dir = os.path.join(cwd, "bin")
+        tool_dir = os.path.join(cwd, "tool")
+        cache_dir = os.path.join(cwd, ".cache")
+
+        return Command(
+            name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
+            prepare=f"rm -rf {bin_dir} && rm -rf {tool_dir} && rm -rf {cache_dir}",
+            command=[
+                f"XDG_BIN_HOME={bin_dir}",
+                f"UV_TOOL_DIR={tool_dir}",
+                self.path,
+                "tool",
+                "install",
+                "--cache-dir",
+                cache_dir,
+                "--",
+                TOOL,
+            ],
+        )
+
+    def install_warm(self, *, cwd: str) -> Command | None:
+        bin_dir = os.path.join(cwd, "bin")
+        tool_dir = os.path.join(cwd, "tool")
+        cache_dir = os.path.join(cwd, ".cache")
+
+        return Command(
+            name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
+            prepare=f"rm -rf {bin_dir} && rm -rf {tool_dir}",
+            command=[
+                f"XDG_BIN_HOME={bin_dir}",
+                f"UV_TOOL_DIR={tool_dir}",
+                self.path,
+                "tool",
+                "install",
+                "--cache-dir",
+                cache_dir,
+                "--",
+                TOOL,
+            ],
+        )
+
+    def run(self, *, cwd: str) -> Command | None:
+        bin_dir = os.path.join(cwd, "bin")
+        tool_dir = os.path.join(cwd, "tool")
+        cache_dir = os.path.join(cwd, ".cache")
+
+        return Command(
+            name=f"{self.name} ({Benchmark.RUN.value})",
+            prepare="",
+            command=[
+                f"XDG_BIN_HOME={bin_dir}",
+                f"UV_TOOL_DIR={tool_dir}",
+                self.path,
+                "tool",
+                "run",
+                "--cache-dir",
+                cache_dir,
+                "--",
+                TOOL,
+                "--version",
+            ],
+        )
+
+
+def main():
+    """Run the benchmark."""
+    parser = argparse.ArgumentParser(
+        description="Benchmark uv against other packaging tools."
+    )
+    parser.add_argument(
+        "--verbose", "-v", action="store_true", help="Print verbose output."
+    )
+    parser.add_argument("--json", action="store_true", help="Export results to JSON.")
+    parser.add_argument(
+        "--warmup",
+        type=int,
+        help="The number of warmup runs to perform.",
+        default=3,
+    )
+    parser.add_argument(
+        "--min-runs",
+        type=int,
+        help="The minimum number of runs to perform.",
+        default=10,
+    )
+    parser.add_argument(
+        "--benchmark",
+        "-b",
+        type=str,
+        help="The benchmark(s) to run.",
+        choices=[benchmark.value for benchmark in Benchmark],
+        action="append",
+    )
+    parser.add_argument(
+        "--pipx",
+        help="Whether to benchmark `pipx`.",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--uv",
+        help="Whether to benchmark uv (assumes a uv binary exists at `./target/release/uv`).",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--pipx-path",
+        type=str,
+        help="Path(s) to the `pipx` binary to benchmark.",
+        action="append",
+    )
+    parser.add_argument(
+        "--uv-path",
+        type=str,
+        help="Path(s) to the uv binary to benchmark.",
+        action="append",
+    )
+
+    args = parser.parse_args()
+    logging.basicConfig(
+        level=logging.INFO if args.verbose else logging.WARN,
+        format="%(asctime)s %(levelname)s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    verbose = args.verbose
+    json = args.json
+    warmup = args.warmup
+    min_runs = args.min_runs
+
+    # Determine the tools to benchmark, based on the user-provided arguments.
+    suites = []
+    if args.pipx:
+        suites.append(Pipx())
+    if args.uv:
+        suites.append(Uv())
+    for path in args.pipx_path or []:
+        suites.append(Pipx(path=path))
+    for path in args.uv_path or []:
+        suites.append(Uv(path=path))
+
+    # If no tools were specified, benchmark all tools.
+    if not suites:
+        suites = [
+            Pipx(),
+            Uv(),
+        ]
+
+    # Determine the benchmarks to run, based on user input.
+    benchmarks = (
+        [Benchmark(benchmark) for benchmark in args.benchmark]
+        if args.benchmark is not None
+        else list(Benchmark)
+    )
+
+    with tempfile.TemporaryDirectory() as cwd:
+        for benchmark in benchmarks:
+            # Generate the benchmark command for each tool.
+            commands = [
+                command
+                for suite in suites
+                if (command := suite.command(benchmark, cwd=tempfile.mkdtemp(dir=cwd)))
+            ]
+
+            if commands:
+                hyperfine = Hyperfine(
+                    name=str(benchmark.value),
+                    commands=commands,
+                    warmup=warmup,
+                    min_runs=min_runs,
+                    verbose=verbose,
+                    json=json,
+                )
+                hyperfine.run()
+
+
+if __name__ == "__main__":
+    main()

scripts/benchmark/uv.lock

@@ -14,6 +14,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7b/a2/10639a79341f6c019dedc95bd48a4928eed9f1d1197f4c04f546fc7ae0ff/anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7", size = 86780 },
 ]
 
+[[distribution]]
+name = "argcomplete"
+version = "3.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/db/ca/45176b8362eb06b68f946c2bf1184b92fc98d739a3f8c790999a257db91f/argcomplete-3.4.0.tar.gz", hash = "sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f", size = 82275 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0b/29/cba741f3abc1700dda883c4a1dd83f4ae89e4e8654067929d89143df2c58/argcomplete-3.4.0-py3-none-any.whl", hash = "sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5", size = 42641 },
+]
+
 [[distribution]]
 name = "benchmark"
 version = "0.0.1"
@@ -21,6 +30,7 @@ source = { editable = "." }
 dependencies = [
     { name = "pdm" },
     { name = "pip-tools" },
+    { name = "pipx" },
     { name = "poetry" },
     { name = "tomli" },
     { name = "tomli-w" },
@@ -511,6 +521,22 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", hash = "sha256:4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", size = 61235 },
 ]
 
+[[distribution]]
+name = "pipx"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "argcomplete" },
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "packaging" },
+    { name = "platformdirs" },
+    { name = "userpath" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2d/f3/c04c5cd0a5795fe6bb09d56c4892384e53cb75813fc08e5cbfa4d080664a/pipx-1.6.0.tar.gz", hash = "sha256:840610e00103e3d49ae24b6b51804b60988851a5dd65468adb71e5a97e2699b2", size = 307321 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/db/2f/ec2454e6784168837dfe0ffad5080c1c2bdf37ee052999cabfd849a56338/pipx-1.6.0-py3-none-any.whl", hash = "sha256:760889dc3aeed7bf4024973bf22ca0c2a891003f52389159ab5cb0c57d9ebff4", size = 77756 },
+]
+
 [[distribution]]
 name = "pkginfo"
 version = "1.11.1"
@@ -835,6 +861,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ca/1c/89ffc63a9605b583d5df2be791a27bc1a42b7c32bab68d3c8f2f73a98cd4/urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", size = 121444 },
 ]
 
+[[distribution]]
+name = "userpath"
+version = "1.9.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d5/b7/30753098208505d7ff9be5b3a32112fb8a4cb3ddfccbbb7ba9973f2e29ff/userpath-1.9.2.tar.gz", hash = "sha256:6c52288dab069257cc831846d15d48133522455d4677ee69a9781f11dbefd815", size = 11140 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/43/99/3ec6335ded5b88c2f7ed25c56ffd952546f7ed007ffb1e1539dc3b57015a/userpath-1.9.2-py3-none-any.whl", hash = "sha256:2cbf01a23d655a1ff8fc166dfb78da1b641d1ceabf0fe5f970767d380b14e89d", size = 9065 },
+]
+
 [[distribution]]
 name = "virtualenv"
 version = "20.26.3"