mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-12-23 09:19:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge bfa8ec0d86 into 7865672918

Browse source
This commit is contained in:
Tomasz Kramkowski 2025-12-22 16:43:49 -05:00 • committed by GitHub
commit 2b130079a3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 535 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

36
crates/uv-cli/src/lib.rs
View file

@ -6114,6 +6114,36 @@ pub struct PythonDirArgs {
pub bin: bool,
}
#[derive(Args)]
pub struct PythonInstallCompileBytecodeArgs {
/// Compile Python's standard library to bytecode after installation.
///
/// By default, uv does not compile Python (`.py`) files to bytecode (`__pycache__/*.pyc`);
/// instead, compilation is performed lazily the first time a module is imported. For use-cases
/// in which start time is important, such as CLI applications and Docker containers, this
/// option can be enabled to trade longer installation times and some additional disk space for
/// faster start times.
///
/// When enabled, uv will process the Python version's `stdlib` directory. It will ignore any
/// compilation errors.
#[arg(
long,
alias = "compile",
overrides_with("no_compile_bytecode"),
env = EnvVars::UV_COMPILE_BYTECODE,
value_parser = clap::builder::BoolishValueParser::new(),
)]
pub compile_bytecode: bool,
#[arg(
long,
alias = "no-compile",
overrides_with("compile_bytecode"),
hide = true
)]
pub no_compile_bytecode: bool,
}
#[derive(Args)]
pub struct PythonInstallArgs {
/// The directory to store the Python installation in.
@ -6233,6 +6263,9 @@ pub struct PythonInstallArgs {
/// If multiple Python versions are requested, uv will exit with an error.
#[arg(long, conflicts_with("no_bin"))]
pub default: bool,
#[command(flatten)]
pub compile_bytecode: PythonInstallCompileBytecodeArgs,
}
impl PythonInstallArgs {
@ -6293,6 +6326,9 @@ pub struct PythonUpgradeArgs {
/// URL pointing to JSON of custom Python installations.
#[arg(long, value_hint = ValueHint::Other)]
pub python_downloads_json_url: Option<String>,
#[command(flatten)]
pub compile_bytecode: PythonInstallCompileBytecodeArgs,
}
impl PythonUpgradeArgs {

192
crates/uv/src/commands/python/install.rs
View file

@ -5,16 +5,18 @@ use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use anyhow::{Error, Result};
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use anyhow::{Context, Error, Result};
use futures::{StreamExt, join};
use indexmap::IndexSet;
use itertools::{Either, Itertools};
use owo_colors::{AnsiColors, OwoColorize};
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::{debug, trace};
use tokio::sync::mpsc;
use tracing::{debug, trace, warn};
use uv_cache::Cache;
use uv_client::BaseClientBuilder;
use uv_configuration::Concurrency;
use uv_fs::Simplified;
use uv_platform::{Arch, Libc};
use uv_preview::{Preview, PreviewFeatures};
@ -27,8 +29,9 @@ use uv_python::managed::{
create_link_to_executable, python_executable_dir,
};
use uv_python::{
PythonDownloads, PythonInstallationKey, PythonInstallationMinorVersionKey, PythonRequest,
PythonVersionFile, VersionFileDiscoveryOptions, VersionFilePreference, VersionRequest,
ImplementationName, Interpreter, PythonDownloads, PythonInstallationKey,
PythonInstallationMinorVersionKey, PythonRequest, PythonVersionFile,
VersionFileDiscoveryOptions, VersionFilePreference, VersionRequest,
};
use uv_shell::Shell;
use uv_trampoline_builder::{Launcher, LauncherKind};
@ -191,6 +194,117 @@ pub(crate) async fn install(
default: bool,
python_downloads: PythonDownloads,
no_config: bool,
compile_bytecode: bool,
concurrency: &Concurrency,
cache: &Cache,
preview: Preview,
printer: Printer,
) -> Result<ExitStatus> {
let (sender, mut receiver) = mpsc::unbounded_channel();
let compiler = async {
let mut did_compile = false;
let mut total_files = 0;
let mut total_elapsed = std::time::Duration::default();
let mut total_skipped = 0;
while let Some(installation) = receiver.recv().await {
did_compile = true;
if let Some((files, elapsed)) =
compile_stdlib_bytecode(&installation, concurrency, cache)
.await
.with_context(|| {
format!(
"Failed to bytecode-compile Python standard library for: {}",
installation.key()
)
})?
{
total_files += files;
total_elapsed += elapsed;
} else {
total_skipped += 1;
}
}
Ok::<_, anyhow::Error>(did_compile.then_some((total_files, total_elapsed, total_skipped)))
};
let installer = perform_install(
project_dir,
install_dir,
targets,
reinstall,
upgrade,
bin,
registry,
force,
python_install_mirror,
pypy_install_mirror,
python_downloads_json_url,
client_builder,
default,
python_downloads,
no_config,
compile_bytecode.then_some(sender),
concurrency,
preview,
printer,
);
let (installer_result, compiler_result) = join!(installer, compiler);
if let Some((total_files, total_elapsed, total_skipped)) = compiler_result? {
if total_files > 0 {
let s = if total_files == 1 { "" } else { "s" };
writeln!(
printer.stderr(),
"{}",
format!(
"Bytecode compiled {} {}{}",
format!("{total_files} file{s}").bold(),
format!("in {}", elapsed(total_elapsed)).dimmed(),
if total_skipped > 0 {
format!(
" (skipped {total_skipped} incompatible version{})",
if total_skipped == 1 { "" } else { "s" }
)
} else {
String::new()
}
.dimmed()
)
.dimmed()
)?;
} else if total_skipped > 0 {
writeln!(
printer.stderr(),
"{}",
format!("No compatible versions to bytecode compile (skipped {total_skipped})")
.dimmed()
)?;
}
}
installer_result
}
#[allow(clippy::fn_params_excessive_bools)]
async fn perform_install(
project_dir: &Path,
install_dir: Option<PathBuf>,
targets: Vec<String>,
reinstall: bool,
upgrade: PythonUpgrade,
bin: Option<bool>,
registry: Option<bool>,
force: bool,
python_install_mirror: Option<String>,
pypy_install_mirror: Option<String>,
python_downloads_json_url: Option<String>,
client_builder: BaseClientBuilder<'_>,
default: bool,
python_downloads: PythonDownloads,
no_config: bool,
bytecode_compilation_sender: Option<mpsc::UnboundedSender<ManagedPythonInstallation>>,
concurrency: &Concurrency,
preview: Preview,
printer: Printer,
) -> Result<ExitStatus> {
@ -433,6 +547,16 @@ pub(crate) async fn install(
})
};
// For all satisfied installs, bytecode compile them now before any future
// early return.
if let Some(ref sender) = bytecode_compilation_sender {
satisfied
.iter()
.copied()
.cloned()
.try_for_each(|installation| sender.send(installation))?;
}
// Check if Python downloads are banned
if matches!(python_downloads, PythonDownloads::Never) && !unsatisfied.is_empty() {
writeln!(
@ -458,10 +582,9 @@ pub(crate) async fn install(
// Download and unpack the Python versions concurrently
let reporter = PythonDownloadReporter::new(printer, Some(downloads.len() as u64));
let mut tasks = FuturesUnordered::new();
for download in &downloads {
tasks.push(async {
let mut tasks = futures::stream::iter(&downloads)
.map(async |download| {
(
*download,
download
@ -477,8 +600,8 @@ pub(crate) async fn install(
)
.await,
)
});
}
})
.buffer_unordered(concurrency.downloads);
let mut errors = vec![];
let mut downloaded = Vec::with_capacity(downloads.len());
@ -493,6 +616,9 @@ pub(crate) async fn install(
};
let installation = ManagedPythonInstallation::new(path, download);
if let Some(ref sender) = bytecode_compilation_sender {
sender.send(installation.clone())?;
}
changelog.installed.insert(installation.key().clone());
for request in &requests {
// Take note of which installations satisfied which requests
@ -1071,6 +1197,50 @@ fn create_bin_links(
}
}
/// Attempt to compile the bytecode for a [`ManagedPythonInstallation`]'s stdlib
async fn compile_stdlib_bytecode(
installation: &ManagedPythonInstallation,
concurrency: &Concurrency,
cache: &Cache,
) -> Result<Option<(usize, std::time::Duration)>> {
let start = std::time::Instant::now();
// Explicit matching so this heuristic is updated for future additions
match installation.implementation() {
ImplementationName::Pyodide => return Ok(None),
ImplementationName::GraalPy | ImplementationName::PyPy | ImplementationName::CPython => (),
}
let interpreter = Interpreter::query(installation.executable(false), cache)
.context("Couldn't locate the interpreter")?;
// Ensure the bytecode compilation occurs in the correct place, in case the installed
// interpreter reports a weird stdlib path.
let interpreter_path = installation.path().canonicalize()?;
let stdlib_path = match interpreter.stdlib().canonicalize() {
Ok(path) if path.starts_with(&interpreter_path) => path,
_ => {
warn!(
"The stdlib path for {} ({}) is not a subdirectory of its installation path ({}).",
installation.key(),
interpreter.stdlib().display(),
interpreter_path.display()
);
return Ok(None);
}
};
let files = uv_installer::compile_tree(
&stdlib_path,
&installation.executable(false),
concurrency,
cache.root(),
)
.await
.with_context(|| format!("Error compiling bytecode in: {}", stdlib_path.display()))?;
Ok(Some((files, start.elapsed())))
}
pub(crate) fn format_executables(
event: &ChangeEvent,
executables: &FxHashMap<PythonInstallationKey, FxHashSet<PathBuf>>,

12
crates/uv/src/lib.rs
View file

@ -1604,6 +1604,9 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
let args = settings::PythonInstallSettings::resolve(args, filesystem, environment);
show_settings!(args);
// Initialize the cache.
let cache = cache.init().await?;
commands::python_install(
&project_dir,
args.install_dir,
@ -1620,6 +1623,9 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
args.default,
globals.python_downloads,
cli.top_level.no_config,
args.compile_bytecode,
&globals.concurrency,
&cache,
globals.preview,
printer,
)
@ -1633,6 +1639,9 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
show_settings!(args);
let upgrade = commands::PythonUpgrade::Enabled(commands::PythonUpgradeSource::Upgrade);
// Initialize the cache.
let cache = cache.init().await?;
commands::python_install(
&project_dir,
args.install_dir,
@ -1649,6 +1658,9 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
args.default,
globals.python_downloads,
cli.top_level.no_config,
args.compile_bytecode,
&globals.concurrency,
&cache,
globals.preview,
printer,
)

16
crates/uv/src/settings.rs
View file

@ -1113,6 +1113,7 @@ pub(crate) struct PythonInstallSettings {
pub(crate) pypy_install_mirror: Option<String>,
pub(crate) python_downloads_json_url: Option<String>,
pub(crate) default: bool,
pub(crate) compile_bytecode: bool,
}
impl PythonInstallSettings {
@ -1152,6 +1153,7 @@ impl PythonInstallSettings {
pypy_mirror: _,
python_downloads_json_url: _,
default,
compile_bytecode,
} = args;
Self {
@ -1171,6 +1173,12 @@ impl PythonInstallSettings {
pypy_install_mirror,
python_downloads_json_url,
default,
compile_bytecode: flag(
compile_bytecode.compile_bytecode,
compile_bytecode.no_compile_bytecode,
"compile-bytecode",
)
.unwrap_or_default(),
}
}
}
@ -1189,6 +1197,7 @@ pub(crate) struct PythonUpgradeSettings {
pub(crate) python_downloads_json_url: Option<String>,
pub(crate) default: bool,
pub(crate) bin: Option<bool>,
pub(crate) compile_bytecode: bool,
}
impl PythonUpgradeSettings {
@ -1226,6 +1235,7 @@ impl PythonUpgradeSettings {
pypy_mirror: _,
reinstall,
python_downloads_json_url: _,
compile_bytecode,
} = args;
Self {
@ -1239,6 +1249,12 @@ impl PythonUpgradeSettings {
python_downloads_json_url,
default,
bin,
compile_bytecode: flag(
compile_bytecode.compile_bytecode,
compile_bytecode.no_compile_bytecode,
"compile-bytecode",
)
.unwrap_or_default(),
}
}
}

10
crates/uv/tests/it/common/mod.rs
View file

@ -481,6 +481,16 @@ impl TestContext {
self
}
/// Add a filter for (bytecode) compilation file counts
#[must_use]
pub fn with_filtered_compiled_file_count(mut self) -> Self {
self.filters.push((
r"compiled \d+ files".to_string(),
"compiled [COUNT] files".to_string(),
));
self
}
/// Adds filters for non-deterministic `CycloneDX` data
pub fn with_cyclonedx_filters(mut self) -> Self {
self.filters.push((

17
crates/uv/tests/it/help.rs
View file

@ -580,6 +580,20 @@ fn help_subsubcommand() {
If multiple Python versions are requested, uv will exit with an error.
--compile-bytecode
Compile Python's standard library to bytecode after installation.
By default, uv does not compile Python (`.py`) files to bytecode (`__pycache__/*.pyc`);
instead, compilation is performed lazily the first time a module is imported. For
use-cases in which start time is important, such as CLI applications and Docker
containers, this option can be enabled to trade longer installation times and some
additional disk space for faster start times.
When enabled, uv will process the Python version's `stdlib` directory. It will ignore any
compilation errors.
[env: UV_COMPILE_BYTECODE=]
Cache options:
-n, --no-cache
Avoid reading from or writing to the cache, instead using a temporary directory for the
@ -833,6 +847,9 @@ fn help_flag_subsubcommand() {
Upgrade existing Python installations to the latest patch version
--default
Use as the default Python version
--compile-bytecode
Compile Python's standard library to bytecode after installation [env:
UV_COMPILE_BYTECODE=]
Cache options:
-n, --no-cache Avoid reading from or writing to the cache, instead using a temporary

254
crates/uv/tests/it/python_install.rs
View file

@ -4,6 +4,7 @@ use std::path::PathBuf;
use std::{env, path::Path, process::Command};
use crate::common::{TestContext, uv_snapshot};
use anyhow::Context;
use assert_cmd::assert::OutputAssertExt;
use assert_fs::{
assert::PathAssert,
@ -15,6 +16,7 @@ use tracing::debug;
use uv_fs::Simplified;
use uv_static::EnvVars;
use walkdir::WalkDir;
#[test]
fn python_install() {
@ -3951,3 +3953,255 @@ fn python_install_upgrade_version_file() {
hint: The version request came from a `.python-version` file; change the patch version in the file to upgrade instead
");
}
#[test]
fn python_install_compile_bytecode() -> anyhow::Result<()> {
fn count_files_by_ext(dir: &Path, extension: &str) -> anyhow::Result<usize> {
let mut count = 0;
let walker = WalkDir::new(dir).into_iter();
for entry in walker {
let entry = entry?;
let path = entry.path();
if entry.metadata()?.is_file() && path.extension().is_some_and(|ext| ext == extension) {
count += 1;
}
}
Ok(count)
}
let context: TestContext = TestContext::new_with_versions(&[])
.with_filtered_python_keys()
.with_filtered_exe_suffix()
.with_filtered_compiled_file_count()
.with_managed_python_dirs()
.with_empty_python_install_mirror()
.with_python_download_cache();
// Install 3.14 and compile its bytecode
uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("3.14"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed Python 3.14.2 in [TIME]
+ cpython-3.14.2-[PLATFORM] (python3.14)
Bytecode compiled [COUNT] files in [TIME]
");
// Find the stdlib path for cpython 3.14
let bin_path = context
.bin_dir
.child(format!("python3.14{}", std::env::consts::EXE_SUFFIX));
#[cfg(unix)]
let stdlib = fs_err::read_link(bin_path)?
.parent()
.context("Python binary should be a child of `bin`")?
.parent()
.context("`bin` directory should be a child of the installation path")?
.join("lib")
.join("python3.14");
#[cfg(windows)]
let stdlib = launcher_path(&bin_path)
.parent()
.context("Python binary should be a child of the installation path")?
.join("Lib");
// And the count should match
let pyc_count = count_files_by_ext(&stdlib, "pyc")?;
let py_count = count_files_by_ext(&stdlib, "py")?;
assert_eq!(pyc_count, py_count);
// Attempting to install with --compile-bytecode should (currently)
// unconditionally re-run the bytecode compiler
uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("3.14"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Python 3.14 is already installed
Bytecode compiled [COUNT] files in [TIME]
");
// Reinstalling with --compile-bytecode should compile bytecode.
uv_snapshot!(context.filters(), context.python_install().arg("--reinstall").arg("--compile-bytecode").arg("3.14"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed Python 3.14.2 in [TIME]
~ cpython-3.14.2-[PLATFORM] (python3.14)
Bytecode compiled [COUNT] files in [TIME]
");
Ok(())
}
#[test]
fn python_install_compile_bytecode_existing() {
let context: TestContext = TestContext::new_with_versions(&[])
.with_filtered_python_keys()
.with_filtered_exe_suffix()
.with_filtered_compiled_file_count()
.with_managed_python_dirs()
.with_empty_python_install_mirror()
.with_python_download_cache();
// A fresh install should be able to be compiled later
uv_snapshot!(context.filters(), context.python_install().arg("3.14"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed Python 3.14.2 in [TIME]
+ cpython-3.14.2-[PLATFORM] (python3.14)
");
uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("3.14"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Python 3.14 is already installed
Bytecode compiled [COUNT] files in [TIME]
");
}
#[test]
fn python_install_compile_bytecode_upgrade() {
let context: TestContext = TestContext::new_with_versions(&[])
.with_filtered_python_keys()
.with_filtered_exe_suffix()
.with_filtered_compiled_file_count()
.with_managed_python_dirs()
.with_empty_python_install_mirror()
.with_python_download_cache();
// An upgrade should also compile bytecode
uv_snapshot!(context.filters(), context.python_install().arg("3.14.0"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed Python 3.14.0 in [TIME]
+ cpython-3.14.0-[PLATFORM] (python3.14)
");
uv_snapshot!(context.filters(), context.python_install().arg("--upgrade").arg("--compile-bytecode").arg("3.14"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed Python 3.14.2 in [TIME]
+ cpython-3.14.2-[PLATFORM] (python3.14)
Bytecode compiled [COUNT] files in [TIME]
");
}
#[test]
fn python_install_compile_bytecode_multiple() {
let context: TestContext = TestContext::new_with_versions(&[])
.with_filtered_python_keys()
.with_filtered_exe_suffix()
.with_filtered_compiled_file_count()
.with_managed_python_dirs()
.with_empty_python_install_mirror()
.with_python_download_cache();
// Should handle installing and compiling multiple versions correctly
uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("3.14").arg("3.12"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed 2 versions in [TIME]
+ cpython-3.12.12-[PLATFORM] (python3.12)
+ cpython-3.14.2-[PLATFORM] (python3.14)
Bytecode compiled [COUNT] files in [TIME]
");
}
#[cfg(unix)] // Pyodide cannot be used on Windows
#[test]
fn python_install_compile_bytecode_pyodide() {
let context: TestContext = TestContext::new_with_versions(&[])
.with_filtered_python_keys()
.with_filtered_exe_suffix()
.with_filtered_compiled_file_count()
.with_managed_python_dirs()
.with_empty_python_install_mirror()
.with_python_download_cache();
// Should warn on explicit pyodide installation
uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("cpython-3.13.2-emscripten-wasm32-musl"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed Python 3.13.2 in [TIME]
+ pyodide-3.13.2-emscripten-wasm32-musl (python3.13)
No compatible versions to bytecode compile (skipped 1)
");
// TODO(tk) There's a bug with python_upgrade when pyodide is installed which leads to
// `error: No download found for request: pyodide-3.13-emscripten-wasm32-musl`
//// Recompilation where pyodide isn't explicitly specified shouldn't warn
//uv_snapshot!(context.filters(), context.python_upgrade().arg("--compile-bytecode"), @r"TODO");
}
// TODO(tk): Revisit: Currently this reliably hits the 120s timeout on the Windows runners
#[cfg(unix)]
#[test]
fn python_install_compile_bytecode_graalpy() {
let context: TestContext = TestContext::new_with_versions(&[])
.with_filtered_python_keys()
.with_filtered_exe_suffix()
.with_filtered_compiled_file_count()
.with_managed_python_dirs()
.with_empty_python_install_mirror()
.with_python_download_cache();
// Should work for graalpy
uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("graalpy-3.12"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed Python 3.12.0 in [TIME]
+ graalpy-3.12.0-[PLATFORM] (python3.12)
Bytecode compiled [COUNT] files in [TIME]
");
}
#[test]
fn python_install_compile_bytecode_pypy() {
let context: TestContext = TestContext::new_with_versions(&[])
.with_filtered_python_keys()
.with_filtered_exe_suffix()
.with_filtered_compiled_file_count()
.with_managed_python_dirs()
.with_empty_python_install_mirror()
.with_python_download_cache();
// Should work for pypy
uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("pypy-3.11"), @r"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed Python 3.11.13 in [TIME]
+ pypy-3.11.13-[PLATFORM] (python3.11)
Bytecode compiled [COUNT] files in [TIME]
");
}

10
docs/guides/integration/docker.md
View file

@ -326,11 +326,12 @@ See a complete example in the
### Compiling bytecode
Compiling Python source files to bytecode is typically desirable for production images as it tends
to improve startup time (at the cost of increased installation time).
to improve startup time (at the cost of increased installation time and image size).
To enable bytecode compilation, use the `--compile-bytecode` flag:
```dockerfile title="Dockerfile"
RUN uv python install --compile-bytecode
RUN uv sync --compile-bytecode
```
@ -341,6 +342,13 @@ commands within the Dockerfile compile bytecode:
ENV UV_COMPILE_BYTECODE=1
```
!!! note
uv will only compile the standard library of _managed_ Python versions during
`uv python install`. The distributor of unmanaged Python versions decides if the
standard library is pre-compiled. For example, the official `python` image will not
have a compiled standard library.
### Caching
A [cache mount](https://docs.docker.com/build/guide/mounts/#add-a-cache-mount) can be used to