diff --git a/.env b/.env index e7317790a..39058d8d5 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ PATH=$PWD/bin:$PATH -PUFFIN_PYTHON_PATH=$PWD/bin +PUFFIN_TEST_PYTHON_PATH=$PWD/bin diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4aa9bf5be..602e74247 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,12 +30,14 @@ jobs: cargo-clippy: name: "cargo clippy" - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: "Install Rust toolchain" - run: | - rustup component add clippy + run: rustup component add clippy - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} @@ -45,21 +47,32 @@ jobs: cargo-test: strategy: matrix: - os: [ubuntu-latest] + include: + # We use the large GitHub actions runners for faster testing + # For Ubuntu and Windows, this requires Organization-level configuration + # See: https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners/about-larger-runners#about-ubuntu-and-windows-larger-runners + - { os: "ubuntu", runner: "ubuntu-latest-large" } + - { os: "windows", runner: "windows-latest-large" } + - { os: "macos", runner: "macos-14" } runs-on: - # We use the large GitHub actions runners for faster testing - # For Ubuntu and Windows, this requires Organization-level configuration - # See: https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners/about-larger-runners#about-ubuntu-and-windows-larger-runners - labels: ${{ matrix.os }}-large + labels: ${{ matrix.runner }} name: "cargo test | ${{ matrix.os }}" steps: - uses: actions/checkout@v4 + - if: ${{ matrix.os == 'macos' }} + name: "Install bootstrap dependencies" + run: brew install coreutils + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: "Install pipx" + run: pip install pipx - name: "Install required Python versions" - run: | - scripts/bootstrap/install.sh + run: pipx run scripts/bootstrap/install.py - name: "Install Rust toolchain" run: rustup show - - uses: rui314/setup-mold@v1 + - if: ${{ matrix.os != 'windows' }} + uses: rui314/setup-mold@v1 - name: "Install cargo nextest" uses: taiki-e/install-action@v2 with: @@ -67,59 +80,8 @@ jobs: - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} - - name: "Tests" - run: | - source .env - cargo nextest run --all --status-level skip --failure-output immediate-final --no-fail-fast -j 12 - - macos: - runs-on: - labels: macos-14 - name: "cargo test | macos" - steps: - - uses: actions/checkout@v4 - - name: "Install bootstrap dependencies" - run: | - brew install coreutils - - name: "Install required Python versions" - run: | - scripts/bootstrap/install.sh - - name: "Install Rust toolchain" - run: rustup show - - uses: rui314/setup-mold@v1 - - name: "Install cargo nextest" - uses: taiki-e/install-action@v2 - with: - tool: cargo-nextest - - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: "Tests" - run: | - source .env - cargo nextest run --all --status-level skip --failure-output immediate-final --no-fail-fast -j 12 - - # TODO(konstin): Merge with the cargo-test job once the tests pass - windows: - runs-on: windows-latest-large - name: "cargo check | windows" - steps: - - uses: actions/checkout@v4 - - name: "Install Rust toolchain" - run: rustup component add clippy - - name: "Install Python for bootstrapping" - uses: actions/setup-python@v4 - with: - python-version: 3.12 - - name: "Install Python binaries" - run: | - pip install zstandard==0.22.0 - python scripts/bootstrap/install.py - # ex) The path needs to be updated downstream - $env:Path = "$pwd\bin" + $env:Path - - uses: rui314/setup-mold@v1 - - uses: Swatinem/rust-cache@v2 - - run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + - name: "Cargo Test" + run: cargo nextest run --all --status-level skip --failure-output immediate-final --no-fail-fast -j 12 # Separate job for the nightly crate windows-trampoline: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 437effed9..14b771774 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,8 +4,6 @@ [Rust](https://rustup.rs/), a C compiler, and CMake are required to build Puffin. -Testing Puffin requires multiple specific Python versions. We provide a script to bootstrap development by downloading the required versions. - ### Linux @@ -35,31 +33,22 @@ See the [Python](#python) section for instructions on installing the Python vers You can install CMake from the [installers](https://cmake.org/download/) or with `pipx install cmake` (make sure that the pipx install path is in `PATH`, pipx complains if it isn't). -### Python - -Install required Python versions with the bootstrapping script: - -``` -scripts/bootstrap/install.sh -``` - -The installed Python binaries will be available in `/bin` and must be added to your path to be used. We -provide a `.env` file with the proper environment variables for development. You may activate it with: - -``` -source .env -``` - -Or, if you use `direnv` to manage your environment: - -``` -echo "dotenv" >> .envrc -direnv allow -``` - ## Testing -To run the tests we recommend [nextest](https://nexte.st/). Make sure to run the tests with `--all-features`, otherwise you'll miss most of our integration tests. +Testing Puffin requires multiple specific Python versions. You can install them into +`/bin` via our bootstrapping script: + +```shell +pipx run scripts/bootstrap/install.py +``` + +Alternatively, you can install `zstandard` from PyPI, then run: + +``` +python3.12 scripts/bootstrap/install.py +``` + +For running tests, we recommend [nextest](https://nexte.st/). ## Running inside a docker container diff --git a/crates/puffin-interpreter/src/interpreter.rs b/crates/puffin-interpreter/src/interpreter.rs index a16e19e0a..8c2c12ece 100644 --- a/crates/puffin-interpreter/src/interpreter.rs +++ b/crates/puffin-interpreter/src/interpreter.rs @@ -137,7 +137,7 @@ impl Interpreter { /// - If a python version is given: `pythonx.y` /// - `python3` (unix) or `python.exe` (windows) /// - /// If `PUFFIN_PYTHON_PATH` is set, we will not check for Python versions in the + /// If `PUFFIN_TEST_PYTHON_PATH` is set, we will not check for Python versions in the /// global PATH, instead we will search using the provided path. Virtual environments /// will still be respected. /// @@ -220,7 +220,7 @@ impl Interpreter { pub fn find_executable + Into + Copy>( requested: R, ) -> Result { - if let Some(isolated) = std::env::var_os("PUFFIN_PYTHON_PATH") { + if let Some(isolated) = std::env::var_os("PUFFIN_TEST_PYTHON_PATH") { if let Ok(cwd) = std::env::current_dir() { which::which_in(requested, Some(isolated), cwd) .map_err(|err| Error::from_which_error(requested.into(), err)) diff --git a/crates/puffin-interpreter/src/python_query.rs b/crates/puffin-interpreter/src/python_query.rs index cfaa02170..90a9c2473 100644 --- a/crates/puffin-interpreter/src/python_query.rs +++ b/crates/puffin-interpreter/src/python_query.rs @@ -38,20 +38,30 @@ pub fn find_requested_python(request: &str) -> Result { let formatted = PathBuf::from(format!("python{request}")); Interpreter::find_executable(&formatted) } else if cfg!(windows) { - if let Some(python_overwrite) = env::var_os("PUFFIN_PYTHON_PATH") { - for path in env::split_paths(&python_overwrite) { - if path - .as_os_str() - .to_str() - // Good enough since we control the bootstrap directory - .is_some_and(|path| path.contains(&format!("@{request}"))) - { - return Ok(path); - } - } - } - if let [major, minor] = versions.as_slice() { + if let Some(python_overwrite) = env::var_os("PUFFIN_TEST_PYTHON_PATH") { + let executable_dir = env::split_paths(&python_overwrite).find(|path| { + path.as_os_str() + .to_str() + // Good enough since we control the bootstrap directory + .is_some_and(|path| path.contains(&format!("@{request}"))) + }); + return if let Some(path) = executable_dir { + Ok(path.join(if cfg!(unix) { + "python3" + } else if cfg!(windows) { + "python.exe" + } else { + unimplemented!("Only Windows and Unix are supported") + })) + } else { + Err(Error::NoSuchPython { + major: *major, + minor: *minor, + }) + }; + } + find_python_windows(*major, *minor)?.ok_or(Error::NoSuchPython { major: *major, minor: *minor, @@ -73,21 +83,27 @@ pub fn find_requested_python(request: &str) -> Result { /// Pick a sensible default for the python a user wants when they didn't specify a version. /// -/// We prefer the test overwrite `PUFFIN_PYTHON_PATH` if it is set, otherwise `python3`/`python` or +/// We prefer the test overwrite `PUFFIN_TEST_PYTHON_PATH` if it is set, otherwise `python3`/`python` or /// `python.exe` respectively. #[instrument] pub fn find_default_python() -> Result { let current_dir = env::current_dir()?; let python = if cfg!(unix) { - which::which_in("python3", env::var_os("PUFFIN_PYTHON_PATH"), current_dir) - .or_else(|_| which::which("python")) - .map_err(|_| Error::NoPythonInstalledUnix)? + which::which_in( + "python3", + env::var_os("PUFFIN_TEST_PYTHON_PATH"), + current_dir, + ) + .or_else(|_| which::which("python")) + .map_err(|_| Error::NoPythonInstalledUnix)? } else if cfg!(windows) { // TODO(konstin): Is that the right order, or should we look for `py --list-paths` first? With the current way // it works even if the python launcher is not installed. - if let Ok(python) = - which::which_in("python.exe", env::var_os("PUFFIN_PYTHON_PATH"), current_dir) - { + if let Ok(python) = which::which_in( + "python.exe", + env::var_os("PUFFIN_TEST_PYTHON_PATH"), + current_dir, + ) { python } else { installed_pythons_windows()? @@ -107,7 +123,7 @@ pub fn find_default_python() -> Result { /// The command takes 8ms on my machine. TODO(konstin): Implement to read python /// installations from the registry instead. fn installed_pythons_windows() -> Result, Error> { - // TODO(konstin): We're not checking PUFFIN_PYTHON_PATH here, no test currently depends on it. + // TODO(konstin): We're not checking PUFFIN_TEST_PYTHON_PATH here, no test currently depends on it. // TODO(konstin): Special case the not found error let output = info_span!("py_list_paths") @@ -148,6 +164,24 @@ fn installed_pythons_windows() -> Result, Error> { } pub(crate) fn find_python_windows(major: u8, minor: u8) -> Result, Error> { + if let Some(python_overwrite) = env::var_os("PUFFIN_TEST_PYTHON_PATH") { + let executable_dir = env::split_paths(&python_overwrite).find(|path| { + path.as_os_str() + .to_str() + // Good enough since we control the bootstrap directory + .is_some_and(|path| path.contains(&format!("@{major}.{minor}"))) + }); + return Ok(executable_dir.map(|path| { + path.join(if cfg!(unix) { + "python3" + } else if cfg!(windows) { + "python.exe" + } else { + unimplemented!("Only Windows and Unix are supported") + }) + })); + } + Ok(installed_pythons_windows()? .into_iter() .find(|(major_, minor_, _path)| *major_ == major && *minor_ == minor) @@ -170,23 +204,20 @@ mod tests { .join("\n Caused by: ") } - #[cfg(unix)] #[test] - fn python312() { - assert_eq!( - find_requested_python("3.12").unwrap(), - find_requested_python("python3.12").unwrap() + fn no_such_python_version() { + assert_snapshot!( + format_err(find_requested_python("3.1000")), + @"Couldn't find `3.1000` in PATH. Is this Python version installed?" ); } - #[test] - fn no_such_python_version() { - assert_snapshot!(format_err(find_requested_python("3.1000")), @"Couldn't find `3.1000` in PATH. Is this Python version installed?"); - } - #[test] fn no_such_python_binary() { - assert_display_snapshot!(format_err(find_requested_python("python3.1000")), @"Couldn't find `python3.1000` in PATH. Is this Python version installed?"); + assert_display_snapshot!( + format_err(find_requested_python("python3.1000")), + @"Couldn't find `python3.1000` in PATH. Is this Python version installed?" + ); } #[cfg(unix)] diff --git a/crates/puffin/tests/common/mod.rs b/crates/puffin/tests/common/mod.rs index 548a62743..e9f74c35b 100644 --- a/crates/puffin/tests/common/mod.rs +++ b/crates/puffin/tests/common/mod.rs @@ -1,6 +1,8 @@ // The `unreachable_pub` is to silence false positives in RustRover. #![allow(dead_code, unreachable_pub)] +use std::borrow::BorrowMut; +use std::env; use std::path::{Path, PathBuf}; use std::process::Output; @@ -9,8 +11,14 @@ use assert_cmd::Command; use assert_fs::assert::PathAssert; use assert_fs::fixture::PathChild; use assert_fs::TempDir; +#[cfg(unix)] +use fs_err::os::unix::fs::symlink as symlink_file; +#[cfg(windows)] +use fs_err::os::windows::fs::symlink_file; use regex::Regex; +use puffin_interpreter::find_requested_python; + // Exclude any packages uploaded after this date. pub static EXCLUDE_NEWER: &str = "2023-11-18T12:00:00Z"; @@ -173,16 +181,50 @@ pub fn get_bin() -> PathBuf { PathBuf::from(env!("CARGO_BIN_EXE_puffin")) } +/// Create a directory with the requested Python binaries available. +pub fn create_bin_with_executables( + temp_dir: &assert_fs::TempDir, + python_versions: &[&str], +) -> anyhow::Result { + if let Some(bootstrapped_pythons) = bootstrapped_pythons() { + let selected_pythons = bootstrapped_pythons.into_iter().filter(|path| { + python_versions.iter().any(|python_version| { + // Good enough since we control the directory + path.to_str() + .unwrap() + .contains(&format!("@{python_version}")) + }) + }); + return Ok(env::join_paths(selected_pythons)?.into()); + } + + let bin = temp_dir.child("bin"); + fs_err::create_dir(&bin)?; + for request in python_versions { + let executable = find_requested_python(request)?; + let name = executable + .file_name() + .expect("Discovered executable must have a filename"); + symlink_file(&executable, bin.child(name))?; + } + Ok(bin.canonicalize()?) +} + /// Execute the command and format its output status, stdout and stderr into a snapshot string. /// /// This function is derived from `insta_cmd`s `spawn_with_info`. pub fn run_and_format<'a>( - command: &mut std::process::Command, + mut command: impl BorrowMut, filters: impl AsRef<[(&'a str, &'a str)]>, windows_filters: bool, ) -> (String, Output) { - let program = command.get_program().to_string_lossy().to_string(); + let program = command + .borrow_mut() + .get_program() + .to_string_lossy() + .to_string(); let output = command + .borrow_mut() .output() .unwrap_or_else(|_| panic!("Failed to spawn {program}")); diff --git a/crates/puffin/tests/pip_compile_scenarios.rs b/crates/puffin/tests/pip_compile_scenarios.rs index 0e86026c9..d60cd83a3 100644 --- a/crates/puffin/tests/pip_compile_scenarios.rs +++ b/crates/puffin/tests/pip_compile_scenarios.rs @@ -6,52 +6,17 @@ #![cfg(all(feature = "python", feature = "pypi"))] use std::env; -use std::path::PathBuf; use std::process::Command; use anyhow::Result; use assert_cmd::assert::OutputAssertExt; use assert_fs::fixture::{FileWriteStr, PathChild}; -#[cfg(unix)] -use fs_err::os::unix::fs::symlink as symlink_file; -#[cfg(windows)] -use fs_err::os::windows::fs::symlink_file; use predicates::prelude::predicate; -use common::{bootstrapped_pythons, get_bin, puffin_snapshot, TestContext, INSTA_FILTERS}; -use puffin_interpreter::find_requested_python; +use common::{create_bin_with_executables, get_bin, puffin_snapshot, TestContext, INSTA_FILTERS}; mod common; -/// Create a directory with the requested Python binaries available. -pub(crate) fn create_bin_with_executables( - temp_dir: &assert_fs::TempDir, - python_versions: &[&str], -) -> Result { - if let Some(bootstrapped_pythons) = bootstrapped_pythons() { - let selected_pythons = bootstrapped_pythons.into_iter().filter(|path| { - python_versions.iter().any(|python_version| { - // Good enough since we control the directory - path.to_str() - .unwrap() - .contains(&format!("@{python_version}")) - }) - }); - return Ok(env::join_paths(selected_pythons)?.into()); - } - - let bin = temp_dir.child("bin"); - fs_err::create_dir(&bin)?; - for request in python_versions { - let executable = find_requested_python(request)?; - let name = executable - .file_name() - .expect("Discovered executable must have a filename"); - symlink_file(&executable, bin.child(name))?; - } - Ok(bin.canonicalize()?) -} - /// Provision python binaries and return a `pip compile` command with options shared across all scenarios. fn command(context: &TestContext, python_versions: &[&str]) -> Command { let bin = create_bin_with_executables(&context.temp_dir, python_versions) @@ -67,7 +32,7 @@ fn command(context: &TestContext, python_versions: &[&str]) -> Command { .arg(context.cache_dir.path()) .env("VIRTUAL_ENV", context.venv.as_os_str()) .env("PUFFIN_NO_WRAP", "1") - .env("PUFFIN_PYTHON_PATH", bin) + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&context.temp_dir); command } diff --git a/crates/puffin/tests/pip_install.rs b/crates/puffin/tests/pip_install.rs index 2fa340296..33a81bca0 100644 --- a/crates/puffin/tests/pip_install.rs +++ b/crates/puffin/tests/pip_install.rs @@ -664,14 +664,17 @@ fn install_no_index_version() { /// Install a package without using pre-built wheels. #[test] -#[cfg(not(all(windows, debug_assertions)))] // Stack overflow on debug on windows -.- fn install_no_binary() { let context = TestContext::new("3.12"); - puffin_snapshot!(command(&context) - .arg("Flask") - .arg("--no-binary") - .arg("--strict"), @r###" + let mut command = command(&context); + command.arg("Flask").arg("--no-binary").arg("--strict"); + if cfg!(all(windows, debug_assertions)) { + // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the + // default windows stack of 1MB + command.env("PUFFIN_STACK_SIZE", (2 * 1024 * 1024).to_string()); + } + puffin_snapshot!(command, @r###" success: true exit_code: 0 ----- stdout ----- @@ -728,14 +731,18 @@ fn install_no_binary_subset() { /// Install a package without using pre-built wheels. #[test] -#[cfg(not(all(windows, debug_assertions)))] // Stack overflow on debug on windows -.- fn reinstall_no_binary() { let context = TestContext::new("3.12"); // The first installation should use a pre-built wheel - puffin_snapshot!(command(&context) - .arg("Flask") - .arg("--strict"), @r###" + let mut command = command(&context); + command.arg("Flask").arg("--strict"); + if cfg!(all(windows, debug_assertions)) { + // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the + // default windows stack of 1MB + command.env("PUFFIN_STACK_SIZE", (2 * 1024 * 1024).to_string()); + } + puffin_snapshot!(command, @r###" success: true exit_code: 0 ----- stdout ----- @@ -757,10 +764,15 @@ fn reinstall_no_binary() { context.assert_command("import flask").success(); // Running installation again with `--no-binary` should be a no-op - puffin_snapshot!(command(&context) - .arg("Flask") - .arg("--no-binary") - .arg("--strict"), @r###" + // The first installation should use a pre-built wheel + let mut command = crate::command(&context); + command.arg("Flask").arg("--no-binary").arg("--strict"); + if cfg!(all(windows, debug_assertions)) { + // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the + // default windows stack of 1MB + command.env("PUFFIN_STACK_SIZE", (2 * 1024 * 1024).to_string()); + } + puffin_snapshot!(command, @r###" success: true exit_code: 0 ----- stdout ----- @@ -773,12 +785,29 @@ fn reinstall_no_binary() { context.assert_command("import flask").success(); // With `--reinstall`, `--no-binary` should have an affect - puffin_snapshot!(command(&context) + let filters = if cfg!(windows) { + // Remove the colorama count on windows + INSTA_FILTERS + .iter() + .copied() + .chain([("Resolved 8 packages", "Resolved 7 packages")]) + .collect() + } else { + INSTA_FILTERS.to_vec() + }; + let mut command = crate::command(&context); + command .arg("Flask") .arg("--no-binary") .arg("--reinstall-package") .arg("Flask") - .arg("--strict"), @r###" + .arg("--strict"); + if cfg!(all(windows, debug_assertions)) { + // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the + // default windows stack of 1MB + command.env("PUFFIN_STACK_SIZE", (2 * 1024 * 1024).to_string()); + } + puffin_snapshot!(filters, command, @r###" success: true exit_code: 0 ----- stdout ----- diff --git a/crates/puffin/tests/pip_sync.rs b/crates/puffin/tests/pip_sync.rs index be0664998..80255ad11 100644 --- a/crates/puffin/tests/pip_sync.rs +++ b/crates/puffin/tests/pip_sync.rs @@ -10,7 +10,9 @@ use assert_fs::prelude::*; use indoc::indoc; use url::Url; -use common::{create_venv, puffin_snapshot, venv_to_interpreter, INSTA_FILTERS}; +use common::{ + create_bin_with_executables, create_venv, puffin_snapshot, venv_to_interpreter, INSTA_FILTERS, +}; use puffin_fs::NormalizedDisplay; use crate::common::{get_bin, TestContext}; @@ -318,6 +320,8 @@ fn link() -> Result<()> { .success(); let venv2 = context.temp_dir.child(".venv2"); + let bin = create_bin_with_executables(&context.temp_dir, &["3.12"]) + .expect("Failed to create bin dir"); Command::new(get_bin()) .arg("venv") .arg(venv2.as_os_str()) @@ -325,6 +329,7 @@ fn link() -> Result<()> { .arg(context.cache_dir.path()) .arg("--python") .arg("3.12") + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&context.temp_dir) .assert() .success(); @@ -805,7 +810,6 @@ fn install_numpy_py38() -> Result<()> { /// Install a package without using pre-built wheels. #[test] -#[cfg(not(all(windows, debug_assertions)))] // Stack overflow on debug on windows -.- fn install_no_binary() -> Result<()> { let context = TestContext::new("3.12"); @@ -813,10 +817,17 @@ fn install_no_binary() -> Result<()> { requirements_txt.touch()?; requirements_txt.write_str("MarkupSafe==2.1.3")?; - puffin_snapshot!(command(&context) + let mut command = command(&context); + command .arg("requirements.txt") .arg("--no-binary") - .arg("--strict"), @r###" + .arg("--strict"); + if cfg!(all(windows, debug_assertions)) { + // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the + // default windows stack of 1MB + command.env("PUFFIN_STACK_SIZE", (2 * 1024 * 1024).to_string()); + } + puffin_snapshot!(command, @r###" success: true exit_code: 0 ----- stdout ----- diff --git a/crates/puffin/tests/venv.rs b/crates/puffin/tests/venv.rs index c1e885ae6..a84dc3afa 100644 --- a/crates/puffin/tests/venv.rs +++ b/crates/puffin/tests/venv.rs @@ -7,7 +7,7 @@ use assert_fs::prelude::*; use puffin_fs::NormalizedDisplay; -use crate::common::{get_bin, puffin_snapshot}; +use crate::common::{create_bin_with_executables, get_bin, puffin_snapshot}; mod common; @@ -15,6 +15,7 @@ mod common; fn create_venv() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let cache_dir = assert_fs::TempDir::new()?; + let bin = create_bin_with_executables(&temp_dir, &["3.12"]).expect("Failed to create bin dir"); let venv = temp_dir.child(".venv"); let filter_venv = regex::escape(&venv.normalized_display().to_string()); @@ -32,6 +33,7 @@ fn create_venv() -> Result<()> { .arg("3.12") .arg("--cache-dir") .arg(cache_dir.path()) + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&temp_dir), @r###" success: true exit_code: 0 @@ -52,6 +54,7 @@ fn create_venv() -> Result<()> { fn create_venv_defaults_to_cwd() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let cache_dir = assert_fs::TempDir::new()?; + let bin = create_bin_with_executables(&temp_dir, &["3.12"]).expect("Failed to create bin dir"); let venv = temp_dir.child(".venv"); let filter_venv = regex::escape(&venv.normalized_display().to_string()); @@ -68,6 +71,7 @@ fn create_venv_defaults_to_cwd() -> Result<()> { .arg("3.12") .arg("--cache-dir") .arg(cache_dir.path()) + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&temp_dir), @r###" success: true exit_code: 0 @@ -88,6 +92,7 @@ fn create_venv_defaults_to_cwd() -> Result<()> { fn seed() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let cache_dir = assert_fs::TempDir::new()?; + let bin = create_bin_with_executables(&temp_dir, &["3.12"]).expect("Failed to create bin dir"); let venv = temp_dir.child(".venv"); let filter_venv = regex::escape(&venv.normalized_display().to_string()); @@ -106,6 +111,7 @@ fn seed() -> Result<()> { .arg("3.12") .arg("--cache-dir") .arg(cache_dir.path()) + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&temp_dir), @r###" success: true exit_code: 0 @@ -129,6 +135,7 @@ fn seed() -> Result<()> { fn create_venv_unknown_python_minor() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let cache_dir = assert_fs::TempDir::new()?; + let bin = create_bin_with_executables(&temp_dir, &["3.12"]).expect("Failed to create bin dir"); let venv = temp_dir.child(".venv"); let mut command = Command::new(get_bin()); @@ -139,6 +146,7 @@ fn create_venv_unknown_python_minor() -> Result<()> { .arg("3.15") .arg("--cache-dir") .arg(cache_dir.path()) + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&temp_dir); if cfg!(windows) { puffin_snapshot!(&mut command, @r###" @@ -172,6 +180,7 @@ fn create_venv_unknown_python_minor() -> Result<()> { fn create_venv_unknown_python_patch() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let cache_dir = assert_fs::TempDir::new()?; + let bin = create_bin_with_executables(&temp_dir, &["3.12"]).expect("Failed to create bin dir"); let venv = temp_dir.child(".venv"); let filter_venv = regex::escape(&venv.normalized_display().to_string()); @@ -189,6 +198,7 @@ fn create_venv_unknown_python_patch() -> Result<()> { .arg("3.8.0") .arg("--cache-dir") .arg(cache_dir.path()) + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&temp_dir), @r###" success: false exit_code: 1 @@ -205,10 +215,13 @@ fn create_venv_unknown_python_patch() -> Result<()> { } #[test] +#[ignore] // TODO(konstin): Switch patch version strategy #[cfg(unix)] // TODO(konstin): Support patch versions on Windows fn create_venv_python_patch() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let cache_dir = assert_fs::TempDir::new()?; + let bin = + create_bin_with_executables(&temp_dir, &["3.12.1"]).expect("Failed to create bin dir"); let venv = temp_dir.child(".venv"); let filter_venv = regex::escape(&venv.normalized_display().to_string()); @@ -223,6 +236,7 @@ fn create_venv_python_patch() -> Result<()> { .arg("3.12.1") .arg("--cache-dir") .arg(cache_dir.path()) + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&temp_dir), @r###" success: true exit_code: 0 diff --git a/scripts/bootstrap/install.sh b/scripts/bootstrap/install.sh deleted file mode 100755 index 4751ec160..000000000 --- a/scripts/bootstrap/install.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash -# -# Download required Python versions and install to `bin` -# Uses prebuilt Python distributions from indygreg/python-build-standalone -# -# Requirements -# -# macOS -# -# brew install zstd jq coreutils -# -# Ubuntu -# -# apt install zstd jq -# -# Arch Linux -# -# pacman -S zstd jq libxcrypt-compat -# -# Windows -# -# winget install jqlang.jq -# -# Usage -# -# ./scripts/bootstrap/install.sh -# -# The Python versions are installed from `.python_versions`. -# Python versions are linked in-order such that the _last_ defined version will be the default. -# -# Version metadata can be updated with `fetch-version-metadata.py` which requires Python 3.12 - -set -euo pipefail - -# Convenience function for displaying URLs -function urldecode() { : "${*//+/ }"; echo -e "${_//%/\\x}"; } - -# Convenience function for checking that a command exists. -requires() { - cmd="$1" - if ! command -v "$cmd" > /dev/null 2>&1; then - echo "DEPENDENCY MISSING: $(basename $0) requires $cmd to be installed" >&2 - exit 1 - fi -} - -requires jq -requires zstd - -# Setup some file paths -this_dir=$(realpath "$(dirname "$0")") -root_dir=$(dirname "$(dirname "$this_dir")") -bin_dir="$root_dir/bin" -install_dir="$bin_dir/versions" -versions_file="$root_dir/.python-versions" -versions_metadata="$this_dir/versions.json" - -# Determine system metadata -os=$(uname -s | tr '[:upper:]' '[:lower:]') -arch=$(uname -m) -interpreter='cpython' - -# On macOS, we need a newer version of `realpath` for `--relative-to` support -realpath="$(which grealpath || which realpath)" - - -# Utility for linking executables for the version being installed -# We need to update executables even if the version is already downloaded and extracted -# to ensure that changes to the precedence of versions are respected -link_executables() { - # Use relative paths for links so if the bin is moved they don't break - local link=$($realpath --relative-to="$bin_dir" "$install_key/install/bin/python3") - local minor=$(jq --arg key "$key" '.[$key] | .minor' -r < "$versions_metadata") - - # Link as all version tuples, later versions in the file will take precedence - ln -sf "./$link" "$bin_dir/python$version" - ln -sf "./$link" "$bin_dir/python3.$minor" - ln -sf "./$link" "$bin_dir/python3" - ln -sf "./$link" "$bin_dir/python" -} - -# Read requested versions into an array -versions=() -while IFS= read -r version; do - versions+=("$version") -done < "$versions_file" - -# Install each version -for version in "${versions[@]}"; do - key="$interpreter-$version-$os-$arch" - install_key="$install_dir/$interpreter@$version" - echo "Installing $key" - - if [ -d "$install_key" ]; then - echo "Already available, skipping download" - link_executables - echo "Updated executables for python$version" - continue - fi - - url=$(jq --arg key "$key" '.[$key] | .url' -r < "$versions_metadata") - - if [ "$url" == 'null' ]; then - echo "No matching download for $key" - exit 1 - fi - - filename=$(basename "$url") - echo "Downloading $(urldecode "$filename")" - curl -L --progress-bar -o "$filename" "$url" --output-dir "$this_dir" - - expected_sha=$(jq --arg key "$key" '.[$key] | .sha256' -r < "$versions_metadata") - if [ "$expected_sha" == 'null' ]; then - echo "WARNING: no checksum for $key" - else - echo -n "Verifying checksum..." - echo "$expected_sha $this_dir/$filename" | sha256sum -c --quiet - echo " OK" - fi - - rm -rf "$install_key" - echo "Extracting to $($realpath --relative-to="$root_dir" "$install_key")" - mkdir -p "$install_key" - zstd -d "$this_dir/$filename" --stdout | tar -x -C "$install_key" - - # Setup the installation - mv "$install_key/python/"* "$install_key" - - link_executables - echo "Installed executables for python$version" - - # Cleanup - rmdir "$install_key/python/" - rm "$this_dir/$filename" -done - -echo "Done!" diff --git a/scripts/scenarios/templates/compile.mustache b/scripts/scenarios/templates/compile.mustache index 1515e87c4..eef8bf578 100644 --- a/scripts/scenarios/templates/compile.mustache +++ b/scripts/scenarios/templates/compile.mustache @@ -6,52 +6,17 @@ #![cfg(all(feature = "python", feature = "pypi"))] use std::env; -use std::path::PathBuf; use std::process::Command; use anyhow::Result; use assert_cmd::assert::OutputAssertExt; use assert_fs::fixture::{FileWriteStr, PathChild}; -#[cfg(unix)] -use fs_err::os::unix::fs::symlink as symlink_file; -#[cfg(windows)] -use fs_err::os::windows::fs::symlink_file; use predicates::prelude::predicate; -use common::{bootstrapped_pythons, get_bin, puffin_snapshot, TestContext, INSTA_FILTERS}; -use puffin_interpreter::find_requested_python; +use common::{create_bin_with_executables, get_bin, puffin_snapshot, TestContext, INSTA_FILTERS}; mod common; -/// Create a directory with the requested Python binaries available. -pub(crate) fn create_bin_with_executables( - temp_dir: &assert_fs::TempDir, - python_versions: &[&str], -) -> Result { - if let Some(bootstrapped_pythons) = bootstrapped_pythons() { - let selected_pythons = bootstrapped_pythons.into_iter().filter(|path| { - python_versions.iter().any(|python_version| { - // Good enough since we control the directory - path.to_str() - .unwrap() - .contains(&format!("@{python_version}")) - }) - }); - return Ok(env::join_paths(selected_pythons)?.into()); - } - - let bin = temp_dir.child("bin"); - fs_err::create_dir(&bin)?; - for request in python_versions { - let executable = find_requested_python(request)?; - let name = executable - .file_name() - .expect("Discovered executable must have a filename"); - symlink_file(&executable, bin.child(name))?; - } - Ok(bin.canonicalize()?) -} - /// Provision python binaries and return a `pip compile` command with options shared across all scenarios. fn command(context: &TestContext, python_versions: &[&str]) -> Command { let bin = create_bin_with_executables(&context.temp_dir, python_versions) @@ -67,7 +32,7 @@ fn command(context: &TestContext, python_versions: &[&str]) -> Command { .arg(context.cache_dir.path()) .env("VIRTUAL_ENV", context.venv.as_os_str()) .env("PUFFIN_NO_WRAP", "1") - .env("PUFFIN_PYTHON_PATH", bin) + .env("PUFFIN_TEST_PYTHON_PATH", bin) .current_dir(&context.temp_dir); command }