Add PEP 723 support to uv run (#4656)

Closes #3096 

## Summary

Enables `uv run foo.py` to execute PEP 723-compatible scripts.

For example, given:

```python
# /// script
# requires-python = ">=3.11"
# dependencies = [
#   "requests<3",
#   "rich",
# ]
# ///

import requests
from rich.pretty import pprint

resp = requests.get("https://peps.python.org/api/peps.json")
data = resp.json()
pprint([(k, v["title"]) for k, v in data.items()][:10])
```

![Screenshot 2024-06-29 at 7 23
52 PM](c60f2415-4874-4b15-b9f5-dd8c8c35382e)
This commit is contained in:
Charlie Marsh 2024-07-01 08:20:24 -04:00 committed by GitHub
parent bbd2deb64f
commit bfadadefaf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 552 additions and 39 deletions

21
Cargo.lock generated
View file

@ -2074,9 +2074,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.7.2"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "memmap2"
@ -4486,6 +4486,7 @@ dependencies = [
"uv-normalize",
"uv-requirements",
"uv-resolver",
"uv-scripts",
"uv-settings",
"uv-tool",
"uv-toolchain",
@ -4979,6 +4980,22 @@ dependencies = [
"uv-warnings",
]
[[package]]
name = "uv-scripts"
version = "0.0.1"
dependencies = [
"fs-err",
"indoc",
"memchr",
"once_cell",
"pep440_rs",
"pep508_rs",
"pypi-types",
"serde",
"thiserror",
"toml",
]
[[package]]
name = "uv-settings"
version = "0.0.1"

View file

@ -43,6 +43,7 @@ uv-macros = { path = "crates/uv-macros" }
uv-normalize = { path = "crates/uv-normalize" }
uv-requirements = { path = "crates/uv-requirements" }
uv-resolver = { path = "crates/uv-resolver" }
uv-scripts = { path = "crates/uv-scripts" }
uv-settings = { path = "crates/uv-settings" }
uv-state = { path = "crates/uv-state" }
uv-tool = { path = "crates/uv-tool" }
@ -94,6 +95,7 @@ itertools = { version = "0.13.0" }
junction = { version = "1.0.0" }
mailparse = { version = "0.15.0" }
md-5 = { version = "0.10.6" }
memchr = { version = "2.7.4" }
miette = { version = "7.2.0" }
nanoid = { version = "0.4.0" }
once_cell = { version = "1.19.0" }

View file

@ -321,4 +321,15 @@ impl RequirementsSpecification {
) -> Result<Self> {
Self::from_sources(requirements, &[], &[], client_builder).await
}
/// Initialize a [`RequirementsSpecification`] from a list of [`Requirement`].
pub fn from_requirements(requirements: Vec<Requirement>) -> Self {
Self {
requirements: requirements
.into_iter()
.map(UnresolvedRequirementSpecification::from)
.collect(),
..Self::default()
}
}
}

View file

@ -0,0 +1,23 @@
[package]
name = "uv-scripts"
version = "0.0.1"
edition = "2021"
description = "Parse PEP 723-style Python scripts."
[lints]
workspace = true
[dependencies]
pep440_rs = { workspace = true }
pep508_rs = { workspace = true }
pypi-types = { workspace = true }
fs-err = { workspace = true, features = ["tokio"] }
memchr = { workspace = true }
once_cell = { workspace = true }
serde = { workspace = true, features = ["derive"] }
thiserror = { workspace = true }
toml = { workspace = true }
[dev-dependencies]
indoc = { workspace = true }

View file

@ -0,0 +1,283 @@
use std::io;
use std::path::Path;
use memchr::memmem::Finder;
use once_cell::sync::Lazy;
use pypi_types::VerbatimParsedUrl;
use serde::{Deserialize, Serialize};
use thiserror::Error;
static FINDER: Lazy<Finder> = Lazy::new(|| Finder::new(b"# /// script"));
/// PEP 723 metadata as parsed from a `script` comment block.
///
/// See: <https://peps.python.org/pep-0723/>
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct Pep723Metadata {
pub dependencies: Vec<pep508_rs::Requirement<VerbatimParsedUrl>>,
pub requires_python: Option<pep440_rs::VersionSpecifiers>,
}
#[derive(Debug, Error)]
pub enum Pep723Error {
#[error(transparent)]
Io(#[from] io::Error),
#[error(transparent)]
Utf8(#[from] std::str::Utf8Error),
#[error(transparent)]
Toml(#[from] toml::de::Error),
}
/// Read the PEP 723 `script` metadata from a Python file, if it exists.
///
/// See: <https://peps.python.org/pep-0723/>
pub async fn read_pep723_metadata(
file: impl AsRef<Path>,
) -> Result<Option<Pep723Metadata>, Pep723Error> {
let contents = match fs_err::tokio::read(file).await {
Ok(contents) => contents,
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(err.into()),
};
// Extract the `script` tag.
let Some(contents) = extract_script_tag(&contents)? else {
return Ok(None);
};
// Parse the metadata.
let metadata = toml::from_str(&contents)?;
Ok(Some(metadata))
}
/// Given the contents of a Python file, extract the `script` metadata block, with leading comment
/// hashes removed.
///
/// See: <https://peps.python.org/pep-0723/>
fn extract_script_tag(contents: &[u8]) -> Result<Option<String>, Pep723Error> {
// Identify the opening pragma.
let Some(index) = FINDER.find(contents) else {
return Ok(None);
};
// The opening pragma must be the first line, or immediately preceded by a newline.
if !(index == 0 || matches!(contents[index - 1], b'\r' | b'\n')) {
return Ok(None);
}
// Decode as UTF-8.
let contents = &contents[index..];
let contents = std::str::from_utf8(contents)?;
let mut lines = contents.lines();
// Ensure that the first line is exactly `# /// script`.
if !lines.next().is_some_and(|line| line == "# /// script") {
return Ok(None);
}
// > Every line between these two lines (# /// TYPE and # ///) MUST be a comment starting
// > with #. If there are characters after the # then the first character MUST be a space. The
// > embedded content is formed by taking away the first two characters of each line if the
// > second character is a space, otherwise just the first character (which means the line
// > consists of only a single #).
let mut toml = vec![];
for line in lines {
// Remove the leading `#`.
let Some(line) = line.strip_prefix('#') else {
break;
};
// If the line is empty, continue.
if line.is_empty() {
toml.push("");
continue;
}
// Otherwise, the line _must_ start with ` `.
let Some(line) = line.strip_prefix(' ') else {
break;
};
toml.push(line);
}
// Find the closing `# ///`. The precedence is such that we need to identify the _last_ such
// line.
//
// For example, given:
// ```python
// # /// script
// #
// # ///
// #
// # ///
// ```
//
// The latter `///` is the closing pragma
let Some(index) = toml.iter().rev().position(|line| *line == "///") else {
return Ok(None);
};
let index = toml.len() - index;
// Discard any lines after the closing `# ///`.
//
// For example, given:
// ```python
// # /// script
// #
// # ///
// #
// #
// ```
//
// We need to discard the last two lines.
toml.truncate(index - 1);
// Join the lines into a single string.
let toml = toml.join("\n") + "\n";
Ok(Some(toml))
}
#[cfg(test)]
mod tests {
#[test]
fn missing_space() {
let contents = indoc::indoc! {r"
# /// script
#requires-python = '>=3.11'
# ///
"};
assert_eq!(
super::extract_script_tag(contents.as_bytes()).unwrap(),
None
);
}
#[test]
fn no_closing_pragma() {
let contents = indoc::indoc! {r"
# /// script
# requires-python = '>=3.11'
# dependencies = [
# 'requests<3',
# 'rich',
# ]
"};
assert_eq!(
super::extract_script_tag(contents.as_bytes()).unwrap(),
None
);
}
#[test]
fn leading_content() {
let contents = indoc::indoc! {r"
pass # /// script
# requires-python = '>=3.11'
# dependencies = [
# 'requests<3',
# 'rich',
# ]
# ///
#
#
"};
assert_eq!(
super::extract_script_tag(contents.as_bytes()).unwrap(),
None
);
}
#[test]
fn simple() {
let contents = indoc::indoc! {r"
# /// script
# requires-python = '>=3.11'
# dependencies = [
# 'requests<3',
# 'rich',
# ]
# ///
"};
let expected = indoc::indoc! {r"
requires-python = '>=3.11'
dependencies = [
'requests<3',
'rich',
]
"};
let actual = super::extract_script_tag(contents.as_bytes())
.unwrap()
.unwrap();
assert_eq!(actual, expected);
}
#[test]
fn embedded_comment() {
let contents = indoc::indoc! {r"
# /// script
# embedded-csharp = '''
# /// <summary>
# /// text
# ///
# /// </summary>
# public class MyClass { }
# '''
# ///
"};
let expected = indoc::indoc! {r"
embedded-csharp = '''
/// <summary>
/// text
///
/// </summary>
public class MyClass { }
'''
"};
let actual = super::extract_script_tag(contents.as_bytes())
.unwrap()
.unwrap();
assert_eq!(actual, expected);
}
#[test]
fn trailing_lines() {
let contents = indoc::indoc! {r"
# /// script
# requires-python = '>=3.11'
# dependencies = [
# 'requests<3',
# 'rich',
# ]
# ///
#
#
"};
let expected = indoc::indoc! {r"
requires-python = '>=3.11'
dependencies = [
'requests<3',
'rich',
]
"};
let actual = super::extract_script_tag(contents.as_bytes())
.unwrap()
.unwrap();
assert_eq!(actual, expected);
}
}

View file

@ -33,9 +33,10 @@ uv-installer = { workspace = true }
uv-normalize = { workspace = true }
uv-requirements = { workspace = true }
uv-resolver = { workspace = true }
uv-scripts = { workspace = true }
uv-settings = { workspace = true, features = ["schemars"] }
uv-toolchain = { workspace = true, features = ["schemars"]}
uv-tool = { workspace = true }
uv-toolchain = { workspace = true, features = ["schemars"]}
uv-types = { workspace = true }
uv-virtualenv = { workspace = true }
uv-warnings = { workspace = true }

View file

@ -15,7 +15,7 @@ use uv_distribution::Workspace;
use uv_fs::Simplified;
use uv_git::GitResolver;
use uv_installer::{SatisfiesResult, SitePackages};
use uv_requirements::{RequirementsSource, RequirementsSpecification};
use uv_requirements::RequirementsSpecification;
use uv_resolver::{FlatIndex, InMemoryIndex, OptionsBuilder, PythonRequirement, RequiresPython};
use uv_toolchain::{
request_from_version_file, EnvironmentPreference, Interpreter, PythonEnvironment, Toolchain,
@ -280,7 +280,7 @@ pub(crate) async fn init_environment(
/// Update a [`PythonEnvironment`] to satisfy a set of [`RequirementsSource`]s.
pub(crate) async fn update_environment(
venv: PythonEnvironment,
requirements: &[RequirementsSource],
spec: RequirementsSpecification,
settings: &ResolverInstallerSettings,
preview: PreviewMode,
connectivity: Connectivity,
@ -305,17 +305,6 @@ pub(crate) async fn update_environment(
build_options,
} = settings;
let client_builder = BaseClientBuilder::new()
.connectivity(connectivity)
.native_tls(native_tls)
.keyring(*keyring_provider);
// Read all requirements from the provided sources.
// TODO(zanieb): Consider allowing constraints and extras
// TODO(zanieb): Allow specifying extras somehow
let spec =
RequirementsSpecification::from_sources(requirements, &[], &[], &client_builder).await?;
// Check if the current environment satisfies the requirements
let site_packages = SitePackages::from_environment(&venv)?;
if spec.source_trees.is_empty() && reinstall.is_none() {

View file

@ -6,16 +6,17 @@ use anyhow::{Context, Result};
use tokio::process::Command;
use tracing::debug;
use pypi_types::Requirement;
use uv_cache::Cache;
use uv_cli::ExternalCommand;
use uv_client::{BaseClientBuilder, Connectivity};
use uv_configuration::{Concurrency, ExtrasSpecification, PreviewMode};
use uv_distribution::{VirtualProject, Workspace, WorkspaceError};
use uv_normalize::PackageName;
use uv_requirements::RequirementsSource;
use uv_requirements::{RequirementsSource, RequirementsSpecification};
use uv_toolchain::{
EnvironmentPreference, Interpreter, PythonEnvironment, Toolchain, ToolchainPreference,
ToolchainRequest,
request_from_version_file, EnvironmentPreference, Interpreter, PythonEnvironment, Toolchain,
ToolchainPreference, ToolchainRequest, VersionRequest,
};
use uv_warnings::warn_user_once;
@ -49,8 +50,83 @@ pub(crate) async fn run(
// Parse the input command.
let command = RunCommand::from(command);
// Determine whether the command to execute is a PEP 723 script.
let temp_dir;
let script_interpreter = if let RunCommand::Python(target, _) = &command {
if let Some(metadata) = uv_scripts::read_pep723_metadata(&target).await? {
debug!("Found PEP 723 script at: {}", target.display());
let spec = RequirementsSpecification::from_requirements(
metadata
.dependencies
.into_iter()
.map(Requirement::from)
.collect(),
);
// (1) Explicit request from user
let python_request = if let Some(request) = python.as_deref() {
Some(ToolchainRequest::parse(request))
// (2) Request from `.python-version`
} else if let Some(request) = request_from_version_file().await? {
Some(request)
// (3) `Requires-Python` in `pyproject.toml`
} else {
metadata.requires_python.map(|requires_python| {
ToolchainRequest::Version(VersionRequest::Range(requires_python))
})
};
let client_builder = BaseClientBuilder::new()
.connectivity(connectivity)
.native_tls(native_tls);
let interpreter = Toolchain::find_or_fetch(
python_request,
EnvironmentPreference::Any,
toolchain_preference,
client_builder,
cache,
)
.await?
.into_interpreter();
// Create a virtual environment
temp_dir = cache.environment()?;
let venv = uv_virtualenv::create_venv(
temp_dir.path(),
interpreter,
uv_virtualenv::Prompt::None,
false,
false,
)?;
// Install the script requirements.
let environment = project::update_environment(
venv,
spec,
&settings,
preview,
connectivity,
concurrency,
native_tls,
cache,
printer,
)
.await?;
Some(environment.into_interpreter())
} else {
None
}
} else {
None
};
// Discover and sync the base environment.
let base_interpreter = if isolated {
let base_interpreter = if let Some(script_interpreter) = script_interpreter {
Some(script_interpreter)
} else if isolated {
// package is `None`, isolated and package are marked as conflicting in clap.
None
} else {
@ -199,11 +275,18 @@ pub(crate) async fn run(
false,
)?;
let client_builder = BaseClientBuilder::new()
.connectivity(connectivity)
.native_tls(native_tls);
let spec =
RequirementsSpecification::from_simple_sources(&requirements, &client_builder).await?;
// Install the ephemeral requirements.
Some(
project::update_environment(
venv,
&requirements,
spec,
&settings,
preview,
connectivity,
@ -316,8 +399,8 @@ impl std::fmt::Display for RunCommand {
}
Ok(())
}
Self::External(command, args) => {
write!(f, "{}", command.to_string_lossy())?;
Self::External(executable, args) => {
write!(f, "{}", executable.to_string_lossy())?;
for arg in args {
write!(f, " {}", arg.to_string_lossy())?;
}

View file

@ -7,7 +7,6 @@ use anyhow::{bail, Context, Result};
use distribution_types::Name;
use itertools::Itertools;
use pep508_rs::Requirement;
use pypi_types::VerbatimParsedUrl;
use tracing::debug;
use uv_cache::Cache;
@ -17,7 +16,7 @@ use uv_configuration::{Concurrency, PreviewMode, Reinstall};
use uv_fs::replace_symlink;
use uv_fs::Simplified;
use uv_installer::SitePackages;
use uv_requirements::RequirementsSource;
use uv_requirements::RequirementsSpecification;
use uv_tool::{entrypoint_paths, find_executable_directory, InstalledTools, Tool, ToolEntrypoint};
use uv_toolchain::{EnvironmentPreference, Toolchain, ToolchainPreference, ToolchainRequest};
use uv_warnings::warn_user_once;
@ -48,11 +47,12 @@ pub(crate) async fn install(
}
let from = if let Some(from) = from {
let from_requirement = Requirement::<VerbatimParsedUrl>::from_str(&from)?;
let from_requirement = pep508_rs::Requirement::<VerbatimParsedUrl>::from_str(&from)?;
// Check if the user provided more than just a name positionally or if that name conflicts with `--from`
if from_requirement.name.to_string() != package {
// Determine if its an entirely different package or a conflicting specification
let package_requirement = Requirement::<VerbatimParsedUrl>::from_str(&package)?;
let package_requirement =
pep508_rs::Requirement::<VerbatimParsedUrl>::from_str(&package)?;
if from_requirement.name == package_requirement.name {
bail!(
"Package requirement `{}` provided with `--from` conflicts with install request `{}`",
@ -68,7 +68,7 @@ pub(crate) async fn install(
}
from_requirement
} else {
Requirement::<VerbatimParsedUrl>::from_str(&package)?
pep508_rs::Requirement::<VerbatimParsedUrl>::from_str(&package)?
};
let name = from.name.to_string();
@ -102,14 +102,19 @@ pub(crate) async fn install(
let requirements = [Ok(from.clone())]
.into_iter()
.chain(with.iter().map(|name| Requirement::from_str(name)))
.collect::<Result<Vec<Requirement<VerbatimParsedUrl>>, _>>()?;
.chain(
with.iter()
.map(|name| pep508_rs::Requirement::from_str(name)),
)
.collect::<Result<Vec<pep508_rs::Requirement<VerbatimParsedUrl>>, _>>()?;
// TODO(zanieb): Duplicative with the above parsing but needed for `update_environment`
let requirements_sources = [RequirementsSource::from_package(from.to_string())]
.into_iter()
.chain(with.into_iter().map(RequirementsSource::from_package))
.collect::<Vec<_>>();
let spec = RequirementsSpecification::from_requirements(
requirements
.iter()
.cloned()
.map(pypi_types::Requirement::from)
.collect(),
);
let Some(from) = requirements.first().cloned() else {
bail!("Expected at least one requirement")
@ -139,7 +144,7 @@ pub(crate) async fn install(
// Install the ephemeral requirements.
let environment = update_environment(
environment,
&requirements_sources,
spec,
&settings,
preview,
connectivity,

View file

@ -11,9 +11,9 @@ use tracing::debug;
use uv_cache::Cache;
use uv_cli::ExternalCommand;
use uv_client::Connectivity;
use uv_client::{BaseClientBuilder, Connectivity};
use uv_configuration::{Concurrency, PreviewMode};
use uv_requirements::RequirementsSource;
use uv_requirements::{RequirementsSource, RequirementsSpecification};
use uv_toolchain::{
EnvironmentPreference, PythonEnvironment, Toolchain, ToolchainPreference, ToolchainRequest,
};
@ -60,6 +60,13 @@ pub(crate) async fn run(
.chain(with.into_iter().map(RequirementsSource::from_package))
.collect::<Vec<_>>();
let client_builder = BaseClientBuilder::new()
.connectivity(connectivity)
.native_tls(native_tls);
let spec =
RequirementsSpecification::from_simple_sources(&requirements, &client_builder).await?;
// TODO(zanieb): When implementing project-level tools, discover the project and check if it has the tool.
// TODO(zanieb): Determine if we should layer on top of the project environment if it is present.
@ -93,7 +100,7 @@ pub(crate) async fn run(
let ephemeral_env = Some(
update_environment(
venv,
&requirements,
spec,
&settings,
preview,
connectivity,

View file

@ -171,3 +171,95 @@ fn run_args() -> Result<()> {
Ok(())
}
/// Run a PEP 723-compatible script. The script should take precedence over the workspace
/// dependencies.
#[test]
fn run_script() -> Result<()> {
let context = TestContext::new("3.12");
let pyproject_toml = context.temp_dir.child("pyproject.toml");
pyproject_toml.write_str(indoc! { r#"
[project]
name = "foo"
version = "1.0.0"
requires-python = ">=3.8"
dependencies = ["anyio"]
"#
})?;
// If the script contains a PEP 723 tag, we should install its requirements.
let test_script = context.temp_dir.child("main.py");
test_script.write_str(indoc! { r#"
# /// script
# requires-python = ">=3.11"
# dependencies = [
# "iniconfig",
# ]
# ///
import iniconfig
"#
})?;
uv_snapshot!(context.filters(), context.run().arg("--preview").arg("main.py"), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 1 package in [TIME]
Prepared 1 package in [TIME]
Installed 1 package in [TIME]
+ iniconfig==2.0.0
"###);
// Otherwise, the script requirements should _not_ be available, but the project requirements
// should.
let test_non_script = context.temp_dir.child("main.py");
test_non_script.write_str(indoc! { r"
import iniconfig
"
})?;
uv_snapshot!(context.filters(), context.run().arg("--preview").arg("main.py"), @r###"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
Resolved 6 packages in [TIME]
Prepared 4 packages in [TIME]
Installed 4 packages in [TIME]
+ anyio==4.3.0
+ foo==1.0.0 (from file://[TEMP_DIR]/)
+ idna==3.6
+ sniffio==1.3.1
Traceback (most recent call last):
File "[TEMP_DIR]/main.py", line 1, in <module>
import iniconfig
ModuleNotFoundError: No module named 'iniconfig'
"###);
// But the script should be runnable.
let test_non_script = context.temp_dir.child("main.py");
test_non_script.write_str(indoc! { r#"
import idna
print("Hello, world!")
"#
})?;
uv_snapshot!(context.filters(), context.run().arg("--preview").arg("main.py"), @r###"
success: true
exit_code: 0
----- stdout -----
Hello, world!
----- stderr -----
Resolved 6 packages in [TIME]
Audited 4 packages in [TIME]
"###);
Ok(())
}