mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 21:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Use Python 3.12 in benchmarks (#1215)

Browse source 
I originally used Python 3.10, since 3.10 and 3.11 are by far the most
common (at least for [Ruff](https://pypistats.org/packages/ruff)). But
3.12 should give Python tools the most favorable benchmarks.
This commit is contained in:
Charlie Marsh 2024-01-31 12:51:13 -08:00 • committed by GitHub
parent ee69fb51ea
commit 51e8609ee8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 27 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

11
BENCHMARKS.md
View file

@ -1,6 +1,7 @@
# Benchmarks
All benchmarks were computed on macOS, and come with a few important caveats:
All benchmarks were computed on macOS using Python 3.12 (for non-Puffin tools), and come with a few
important caveats:
- Benchmark performance may vary dramatically across different operating systems and filesystems.
In particular, Puffin uses different installation strategies based on the underlying filesystem's
@ -25,7 +26,7 @@ Benchmarking package installation (e.g., `puffin pip sync`) with a warm cache. T
to removing and recreating a virtual environment, and then populating it with dependencies that
you've installed previously on the same machine.
![](https://github.com/astral-sh/ruff/assets/1309177/6ceea7aa-4813-4ea8-8c95-b8013d702cf4)
![install-warm](https://github.com/astral-sh/ruff/assets/1309177/b6cb8d48-52e0-45c2-ae15-0a3f69ec3263)
## Cold Installation
@ -33,7 +34,7 @@ Benchmarking package installation (e.g., `puffin pip sync`) with a cold cache. T
to running `puffin pip sync` on a new machine or in CI (assuming that the package manager cache is
not shared across runs).
![](https://github.com/astral-sh/ruff/assets/1309177/c960d6fd-ec34-467e-9aa2-d4e6713abed0)
![install-cold](https://github.com/astral-sh/ruff/assets/1309177/ed86c193-582f-4163-b369-f12ec3905c3c)
## Warm Resolution
@ -41,7 +42,7 @@ Benchmarking dependency resolution (e.g., `puffin pip compile`) with a warm cach
lockfile. This is equivalent to blowing away an existing `requirements.txt` file to regenerate it
from a `requirements.in` file.
![](https://github.com/astral-sh/ruff/assets/1309177/aab99181-e54e-4bdb-9ce6-15b018ef8466)
![resolve-warm](https://github.com/astral-sh/ruff/assets/1309177/a4ca9d23-1148-4103-abe7-a35fa488409d)
## Cold Resolution
@ -49,7 +50,7 @@ Benchmarking dependency resolution (e.g., `puffin pip compile`) with a cold cach
equivalent to running `puffin pip compile` on a new machine or in CI (assuming that the package
manager cache is not shared across runs).
![](https://github.com/astral-sh/ruff/assets/1309177/a6075ebc-bb8f-46db-a3b4-14ee5f713565)
![resolve-cold](https://github.com/astral-sh/ruff/assets/1309177/556ac7aa-0a6a-4f94-b0d9-90b25461de7b)
## Reproduction

6
crates/puffin-dev/src/render_benchmarks.rs
View file

@ -23,13 +23,15 @@ pub(crate) fn render_benchmarks(args: &RenderBenchmarksArgs) -> Result<()> {
// but we assume we're running over a single benchmark here.)
for result in &mut results.results {
if result.command.starts_with("puffin") {
result.command = "puffin".into();
result.command = "Puffin".into();
} else if result.command.starts_with("pip-compile") {
result.command = "pip-compile".into();
} else if result.command.starts_with("pip-sync") {
result.command = "pip-sync".into();
} else if result.command.starts_with("poetry") {
result.command = "poetry".into();
result.command = "Poetry".into();
} else if result.command.starts_with("pdm") {
result.command = "PDM".into();
} else {
return Err(anyhow!("unknown command: {}", result.command));
}

30
scripts/bench/__main__.py
View file

@ -4,7 +4,7 @@ This script assumes that `pip`, `pip-tools`, `virtualenv`, `poetry` and `hyperfi
installed, and that a Puffin release builds exists at `./target/release/puffin`
(relative to the repository root).
This script assumes that Python 3.10 is installed.
This script assumes that Python 3.12 is installed.
To set up the required environment, run:
@ -300,7 +300,7 @@ class PipSync(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.10 {venv_dir}",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
command=[
self.path,
os.path.abspath(requirements_file),
@ -317,7 +317,7 @@ class PipSync(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.10 {venv_dir}",
prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
command=[
self.path,
os.path.abspath(requirements_file),
@ -357,9 +357,11 @@ class Poetry(Suite):
"bench",
"--no-interaction",
"--python",
">=3.10",
"3.12",
],
cwd=cwd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
# Parse the pyproject.toml.
@ -518,7 +520,7 @@ class Poetry(Suite):
f"rm -rf {config_dir} && "
f"rm -rf {cache_dir} && "
f"rm -rf {data_dir} &&"
f"virtualenv --clear -p 3.10 {venv_dir} --no-seed"
f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
),
command=[
f"POETRY_CONFIG_DIR={config_dir}",
@ -558,7 +560,7 @@ class Poetry(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.10 {venv_dir}",
prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
command=[
f"POETRY_CONFIG_DIR={config_dir}",
f"POETRY_CACHE_DIR={cache_dir}",
@ -595,8 +597,10 @@ class Pdm(Suite):
# Create a PDM project.
subprocess.check_call(
[self.path, "init", "--non-interactive", "--python", "3.10"],
[self.path, "init", "--non-interactive", "--python", "3.12"],
cwd=cwd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
# Parse the pyproject.toml.
@ -635,7 +639,7 @@ class Pdm(Suite):
cache_dir = os.path.join(cwd, "cache", "pdm")
return Command(
name=f"{self.name} ({Benchmark.RESOLVE_COLD.value})",
name=f"{self.name} ({Benchmark.RESOLVE_WARM.value})",
prepare=f"rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}",
command=[
self.path,
@ -719,7 +723,7 @@ class Pdm(Suite):
prepare=(
f"rm -rf {cache_dir} && "
f"{self.path} config cache_dir {cache_dir} && "
f"virtualenv --clear -p 3.10 {venv_dir} --no-seed"
f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
),
command=[
f"VIRTUAL_ENV={venv_dir}",
@ -750,10 +754,10 @@ class Pdm(Suite):
cache_dir = os.path.join(cwd, "cache", "pdm")
return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=(
f"{self.path} config cache_dir {cache_dir} && "
f"virtualenv --clear -p 3.10 {venv_dir} --no-seed"
f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
),
command=[
f"VIRTUAL_ENV={venv_dir}",
@ -871,7 +875,7 @@ class Puffin(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.10 {venv_dir}",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
command=[
f"VIRTUAL_ENV={venv_dir}",
self.path,
@ -889,7 +893,7 @@ class Puffin(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.10 {venv_dir}",
prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
command=[
f"VIRTUAL_ENV={venv_dir}",
self.path,