diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a4a153a11..a119b10ce6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,8 +5,7 @@ exclude: | .github/workflows/release.yml| crates/ty_vendored/vendor/.*| crates/ty_project/resources/.*| - crates/ty/docs/configuration.md| - crates/ty/docs/rules.md| + crates/ty/docs/(configuration|rules|cli).md| crates/ruff_benchmark/resources/.*| crates/ruff_linter/resources/.*| crates/ruff_linter/src/rules/.*/snapshots/.*| diff --git a/Cargo.lock b/Cargo.lock index 419dedfd43..b29f973506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -478,7 +478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -487,7 +487,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -1771,6 +1771,15 @@ dependencies = [ "url", ] +[[package]] +name = "markdown" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5cab8f2cadc416a82d2e783a1946388b31654d391d1c7d92cc1f03e295b1deb" +dependencies = [ + "unicode-id", +] + [[package]] name = "matchers" version = "0.1.0" @@ -2702,6 +2711,7 @@ dependencies = [ "indoc", "itertools 0.14.0", "libcst", + "markdown", "pretty_assertions", "rayon", "regex", @@ -2727,6 +2737,7 @@ dependencies = [ "tracing", "tracing-indicatif", "tracing-subscriber", + "ty", "ty_project", "url", ] @@ -4232,6 +4243,12 @@ dependencies = [ "unic-common", ] +[[package]] +name = "unicode-id" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10103c57044730945224467c09f71a4db0071c123a0648cc3e818913bde6b561" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -4608,7 +4625,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e6961ec456..3ac7fe0da5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ ruff_source_file = { path = "crates/ruff_source_file" } ruff_text_size = { path = "crates/ruff_text_size" } ruff_workspace = { path = "crates/ruff_workspace" } +ty = { path = "crates/ty" } ty_ide = { path = "crates/ty_ide" } ty_project = { path = "crates/ty_project", default-features = false } ty_python_semantic = { path = "crates/ty_python_semantic" } diff --git a/crates/ruff_dev/Cargo.toml b/crates/ruff_dev/Cargo.toml index 0bd71ca65c..1e44acd573 100644 --- a/crates/ruff_dev/Cargo.toml +++ b/crates/ruff_dev/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } license = { workspace = true } [dependencies] +ty = { workspace = true } ty_project = { workspace = true, features = ["schemars"] } ruff = { workspace = true } ruff_diagnostics = { workspace = true } @@ -32,6 +33,7 @@ imara-diff = { workspace = true } indicatif = { workspace = true } itertools = { workspace = true } libcst = { workspace = true } +markdown = { version = "1.0.0" } pretty_assertions = { workspace = true } rayon = { workspace = true } regex = { workspace = true } diff --git a/crates/ruff_dev/src/generate_all.rs b/crates/ruff_dev/src/generate_all.rs index e04cdc891f..4d3fb9577e 100644 --- a/crates/ruff_dev/src/generate_all.rs +++ b/crates/ruff_dev/src/generate_all.rs @@ -3,8 +3,8 @@ use anyhow::Result; use crate::{ - generate_cli_help, generate_docs, generate_json_schema, generate_ty_options, generate_ty_rules, - generate_ty_schema, + generate_cli_help, generate_docs, generate_json_schema, generate_ty_cli_reference, + generate_ty_options, generate_ty_rules, generate_ty_schema, }; pub(crate) const REGENERATE_ALL_COMMAND: &str = "cargo dev generate-all"; @@ -43,5 +43,6 @@ pub(crate) fn main(args: &Args) -> Result<()> { })?; generate_ty_options::main(&generate_ty_options::Args { mode: args.mode })?; generate_ty_rules::main(&generate_ty_rules::Args { mode: args.mode })?; + generate_ty_cli_reference::main(&generate_ty_cli_reference::Args { mode: args.mode })?; Ok(()) } diff --git a/crates/ruff_dev/src/generate_cli_help.rs b/crates/ruff_dev/src/generate_cli_help.rs index 20eb41bb2f..6951f46663 100644 --- a/crates/ruff_dev/src/generate_cli_help.rs +++ b/crates/ruff_dev/src/generate_cli_help.rs @@ -1,5 +1,4 @@ //! Generate CLI help. -#![allow(clippy::print_stdout)] use std::path::PathBuf; use std::{fs, str}; diff --git a/crates/ruff_dev/src/generate_docs.rs b/crates/ruff_dev/src/generate_docs.rs index e9b036144a..e422f498ee 100644 --- a/crates/ruff_dev/src/generate_docs.rs +++ b/crates/ruff_dev/src/generate_docs.rs @@ -1,5 +1,4 @@ //! Generate Markdown documentation for applicable rules. -#![allow(clippy::print_stdout, clippy::print_stderr)] use std::collections::HashSet; use std::fmt::Write as _; diff --git a/crates/ruff_dev/src/generate_json_schema.rs b/crates/ruff_dev/src/generate_json_schema.rs index c82843eef2..af82c3c9b9 100644 --- a/crates/ruff_dev/src/generate_json_schema.rs +++ b/crates/ruff_dev/src/generate_json_schema.rs @@ -1,5 +1,3 @@ -#![allow(clippy::print_stdout, clippy::print_stderr)] - use std::fs; use std::path::PathBuf; diff --git a/crates/ruff_dev/src/generate_ty_cli_reference.rs b/crates/ruff_dev/src/generate_ty_cli_reference.rs new file mode 100644 index 0000000000..8a8c6e4bb8 --- /dev/null +++ b/crates/ruff_dev/src/generate_ty_cli_reference.rs @@ -0,0 +1,334 @@ +//! Generate a Markdown-compatible reference for the ty command-line interface. +use std::cmp::max; +use std::path::PathBuf; + +use anyhow::{bail, Result}; +use clap::{Command, CommandFactory}; +use itertools::Itertools; +use pretty_assertions::StrComparison; + +use crate::generate_all::{Mode, REGENERATE_ALL_COMMAND}; +use crate::ROOT_DIR; + +use ty::Cli; + +const SHOW_HIDDEN_COMMANDS: &[&str] = &["generate-shell-completion"]; + +#[derive(clap::Args)] +pub(crate) struct Args { + #[arg(long, default_value_t, value_enum)] + pub(crate) mode: Mode, +} + +pub(crate) fn main(args: &Args) -> Result<()> { + let reference_string = generate(); + let filename = "crates/ty/docs/cli.md"; + let reference_path = PathBuf::from(ROOT_DIR).join(filename); + + match args.mode { + Mode::DryRun => { + println!("{reference_string}"); + } + Mode::Check => { + match std::fs::read_to_string(reference_path) { + Ok(current) => { + if current == reference_string { + println!("Up-to-date: {filename}"); + } else { + let comparison = StrComparison::new(¤t, &reference_string); + bail!("{filename} changed, please run `{REGENERATE_ALL_COMMAND}`:\n{comparison}"); + } + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + bail!("{filename} not found, please run `{REGENERATE_ALL_COMMAND}`"); + } + Err(err) => { + bail!("{filename} changed, please run `{REGENERATE_ALL_COMMAND}`:\n{err}"); + } + } + } + Mode::Write => match std::fs::read_to_string(&reference_path) { + Ok(current) => { + if current == reference_string { + println!("Up-to-date: {filename}"); + } else { + println!("Updating: {filename}"); + std::fs::write(reference_path, reference_string.as_bytes())?; + } + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + println!("Updating: {filename}"); + std::fs::write(reference_path, reference_string.as_bytes())?; + } + Err(err) => { + bail!("{filename} changed, please run `cargo dev generate-cli-reference`:\n{err}"); + } + }, + } + + Ok(()) +} + +fn generate() -> String { + let mut output = String::new(); + + let mut ty = Cli::command(); + + // It is very important to build the command before beginning inspection or subcommands + // will be missing all of the propagated options. + ty.build(); + + let mut parents = Vec::new(); + + output.push_str("# CLI Reference\n\n"); + generate_command(&mut output, &ty, &mut parents); + + output +} + +#[allow(clippy::format_push_string)] +fn generate_command<'a>(output: &mut String, command: &'a Command, parents: &mut Vec<&'a Command>) { + if command.is_hide_set() && !SHOW_HIDDEN_COMMANDS.contains(&command.get_name()) { + return; + } + + // Generate the command header. + let name = if parents.is_empty() { + command.get_name().to_string() + } else { + format!( + "{} {}", + parents.iter().map(|cmd| cmd.get_name()).join(" "), + command.get_name() + ) + }; + + // Display the top-level `ty` command at the same level as its children + let level = max(2, parents.len() + 1); + output.push_str(&format!("{} {name}\n\n", "#".repeat(level))); + + // Display the command description. + if let Some(about) = command.get_long_about().or_else(|| command.get_about()) { + output.push_str(&about.to_string()); + output.push_str("\n\n"); + } + + // Display the usage + { + // This appears to be the simplest way to get rendered usage from Clap, + // it is complicated to render it manually. It's annoying that it + // requires a mutable reference but it doesn't really matter. + let mut command = command.clone(); + output.push_str("

Usage

\n\n"); + output.push_str(&format!( + "```\n{}\n```", + command + .render_usage() + .to_string() + .trim_start_matches("Usage: "), + )); + output.push_str("\n\n"); + } + + if command.get_name() == "help" { + return; + } + + // Display a list of child commands + let mut subcommands = command.get_subcommands().peekable(); + let has_subcommands = subcommands.peek().is_some(); + if has_subcommands { + output.push_str("

Commands

\n\n"); + output.push_str("
"); + + for subcommand in subcommands { + if subcommand.is_hide_set() { + continue; + } + let subcommand_name = format!("{name} {}", subcommand.get_name()); + output.push_str(&format!( + "
{subcommand_name}
", + subcommand_name.replace(' ', "-") + )); + if let Some(about) = subcommand.get_about() { + output.push_str(&format!( + "
{}
\n", + markdown::to_html(&about.to_string()) + )); + } + } + + output.push_str("
\n\n"); + } + + // Do not display options for commands with children + if !has_subcommands { + let name_key = name.replace(' ', "-"); + + // Display positional arguments + let mut arguments = command + .get_positionals() + .filter(|arg| !arg.is_hide_set()) + .peekable(); + + if arguments.peek().is_some() { + output.push_str("

Arguments

\n\n"); + output.push_str("
"); + + for arg in arguments { + let id = format!("{name_key}--{}", arg.get_id()); + output.push_str(&format!("
")); + output.push_str(&format!( + "{}", + arg.get_id().to_string().to_uppercase(), + )); + output.push_str("
"); + if let Some(help) = arg.get_long_help().or_else(|| arg.get_help()) { + output.push_str("
"); + output.push_str(&format!("{}\n", markdown::to_html(&help.to_string()))); + output.push_str("
"); + } + } + + output.push_str("
\n\n"); + } + + // Display options and flags + let mut options = command + .get_arguments() + .filter(|arg| !arg.is_positional()) + .filter(|arg| !arg.is_hide_set()) + .sorted_by_key(|arg| arg.get_id()) + .peekable(); + + if options.peek().is_some() { + output.push_str("

Options

\n\n"); + output.push_str("
"); + for opt in options { + let Some(long) = opt.get_long() else { continue }; + let id = format!("{name_key}--{long}"); + + output.push_str(&format!("
")); + output.push_str(&format!("--{long}")); + for long_alias in opt.get_all_aliases().into_iter().flatten() { + output.push_str(&format!(", --{long_alias}")); + } + if let Some(short) = opt.get_short() { + output.push_str(&format!(", -{short}")); + } + for short_alias in opt.get_all_short_aliases().into_iter().flatten() { + output.push_str(&format!(", -{short_alias}")); + } + + // Re-implements private `Arg::is_takes_value_set` used in `Command::get_opts` + if opt + .get_num_args() + .unwrap_or_else(|| 1.into()) + .takes_values() + { + if let Some(values) = opt.get_value_names() { + for value in values { + output.push_str(&format!( + " {}", + value.to_lowercase().replace('_', "-") + )); + } + } + } + output.push_str("
"); + if let Some(help) = opt.get_long_help().or_else(|| opt.get_help()) { + output.push_str("
"); + output.push_str(&format!("{}\n", markdown::to_html(&help.to_string()))); + emit_env_option(opt, output); + emit_default_option(opt, output); + emit_possible_options(opt, output); + output.push_str("
"); + } + } + + output.push_str("
"); + } + + output.push_str("\n\n"); + } + + parents.push(command); + + // Recurse to all of the subcommands. + for subcommand in command.get_subcommands() { + generate_command(output, subcommand, parents); + } + + parents.pop(); +} + +fn emit_env_option(opt: &clap::Arg, output: &mut String) { + if opt.is_hide_env_set() { + return; + } + if let Some(env) = opt.get_env() { + output.push_str(&markdown::to_html(&format!( + "May also be set with the `{}` environment variable.", + env.to_string_lossy() + ))); + } +} + +fn emit_default_option(opt: &clap::Arg, output: &mut String) { + if opt.is_hide_default_value_set() || !opt.get_num_args().expect("built").takes_values() { + return; + } + + let values = opt.get_default_values(); + if !values.is_empty() { + let value = format!( + "\n[default: {}]", + opt.get_default_values() + .iter() + .map(|s| s.to_string_lossy()) + .join(",") + ); + output.push_str(&markdown::to_html(&value)); + } +} + +fn emit_possible_options(opt: &clap::Arg, output: &mut String) { + if opt.is_hide_possible_values_set() { + return; + } + + let values = opt.get_possible_values(); + if !values.is_empty() { + let value = format!( + "\nPossible values:\n{}", + values + .into_iter() + .filter(|value| !value.is_hide_set()) + .map(|value| { + let name = value.get_name(); + value.get_help().map_or_else( + || format!(" - `{name}`"), + |help| format!(" - `{name}`: {help}"), + ) + }) + .collect_vec() + .join("\n"), + ); + output.push_str(&markdown::to_html(&value)); + } +} + +#[cfg(test)] +mod tests { + + use anyhow::Result; + + use crate::generate_all::Mode; + + use super::{main, Args}; + + #[test] + fn ty_cli_reference_is_up_to_date() -> Result<()> { + main(&Args { mode: Mode::Check }) + } +} diff --git a/crates/ruff_dev/src/generate_ty_options.rs b/crates/ruff_dev/src/generate_ty_options.rs index 1dc02d6e36..c323fc88d9 100644 --- a/crates/ruff_dev/src/generate_ty_options.rs +++ b/crates/ruff_dev/src/generate_ty_options.rs @@ -1,5 +1,4 @@ //! Generate a Markdown-compatible listing of configuration options for `pyproject.toml`. -#![allow(clippy::print_stdout, clippy::print_stderr)] use anyhow::bail; use itertools::Itertools; diff --git a/crates/ruff_dev/src/generate_ty_rules.rs b/crates/ruff_dev/src/generate_ty_rules.rs index 6586afc738..dd894ff0fc 100644 --- a/crates/ruff_dev/src/generate_ty_rules.rs +++ b/crates/ruff_dev/src/generate_ty_rules.rs @@ -1,5 +1,4 @@ //! Generates the rules table for ty -#![allow(clippy::print_stdout, clippy::print_stderr)] use std::borrow::Cow; use std::fmt::Write as _; diff --git a/crates/ruff_dev/src/generate_ty_schema.rs b/crates/ruff_dev/src/generate_ty_schema.rs index 77d8e076ec..b44aa895bb 100644 --- a/crates/ruff_dev/src/generate_ty_schema.rs +++ b/crates/ruff_dev/src/generate_ty_schema.rs @@ -1,5 +1,3 @@ -#![allow(clippy::print_stdout, clippy::print_stderr)] - use std::fs; use std::path::PathBuf; diff --git a/crates/ruff_dev/src/main.rs b/crates/ruff_dev/src/main.rs index 4ef20f8cd2..e598bbc8fe 100644 --- a/crates/ruff_dev/src/main.rs +++ b/crates/ruff_dev/src/main.rs @@ -2,6 +2,8 @@ //! //! Within the ruff repository you can run it with `cargo dev`. +#![allow(clippy::print_stdout, clippy::print_stderr)] + use anyhow::Result; use clap::{Parser, Subcommand}; use ruff::{args::GlobalConfigArgs, check}; @@ -15,6 +17,7 @@ mod generate_docs; mod generate_json_schema; mod generate_options; mod generate_rules_table; +mod generate_ty_cli_reference; mod generate_ty_options; mod generate_ty_rules; mod generate_ty_schema; diff --git a/crates/ruff_dev/src/print_ast.rs b/crates/ruff_dev/src/print_ast.rs index b3682e84f6..57a4ac1185 100644 --- a/crates/ruff_dev/src/print_ast.rs +++ b/crates/ruff_dev/src/print_ast.rs @@ -1,5 +1,4 @@ //! Print the AST for a given Python file. -#![allow(clippy::print_stdout, clippy::print_stderr)] use std::path::PathBuf; diff --git a/crates/ruff_dev/src/print_cst.rs b/crates/ruff_dev/src/print_cst.rs index 166923486e..7af2e46613 100644 --- a/crates/ruff_dev/src/print_cst.rs +++ b/crates/ruff_dev/src/print_cst.rs @@ -1,5 +1,4 @@ //! Print the `LibCST` CST for a given Python file. -#![allow(clippy::print_stdout, clippy::print_stderr)] use std::fs; use std::path::PathBuf; diff --git a/crates/ruff_dev/src/print_tokens.rs b/crates/ruff_dev/src/print_tokens.rs index 2c83affbb5..73c7844cce 100644 --- a/crates/ruff_dev/src/print_tokens.rs +++ b/crates/ruff_dev/src/print_tokens.rs @@ -1,5 +1,4 @@ //! Print the token stream for a given Python file. -#![allow(clippy::print_stdout, clippy::print_stderr)] use std::path::PathBuf; diff --git a/crates/ruff_dev/src/round_trip.rs b/crates/ruff_dev/src/round_trip.rs index 6a070c65c1..7501790b50 100644 --- a/crates/ruff_dev/src/round_trip.rs +++ b/crates/ruff_dev/src/round_trip.rs @@ -1,5 +1,4 @@ //! Run round-trip source code generation on a given Python or Jupyter notebook file. -#![allow(clippy::print_stdout, clippy::print_stderr)] use std::fs; use std::path::PathBuf; diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md new file mode 100644 index 0000000000..e83f0ccc95 --- /dev/null +++ b/crates/ty/docs/cli.md @@ -0,0 +1,140 @@ +# CLI Reference + +## ty + +An extremely fast Python type checker. + +

Usage

+ +``` +ty +``` + +

Commands

+ +
ty check

Check a project for type errors

+
ty server

Start the language server

+
ty version

Display ty's version

+
ty help

Print this message or the help of the given subcommand(s)

+
+ +## ty check + +Check a project for type errors + +

Usage

+ +``` +ty check [OPTIONS] [PATH]... +``` + +

Arguments

+ +
PATHS

List of files or directories to check [default: the project root]

+
+ +

Options

+ +
--color when

Control when colored output is used

+

Possible values:

+
    +
  • auto: Display colors if the output goes to an interactive terminal
    • +
  • always: Always display colors
    • +
  • never: Never display colors
    • +
--config, -c config-option

A TOML <KEY> = <VALUE> pair

+
--error rule

Treat the given rule as having severity 'error'. Can be specified multiple times.

+
--error-on-warning

Use exit code 1 if there are any warning-level diagnostics

+
--exit-zero

Always use exit code 0, even when there are error-level diagnostics

+
--extra-search-path path

Additional path to use as a module-resolution source (can be passed multiple times)

+
--help, -h

Print help (see a summary with '-h')

+
--ignore rule

Disables the rule. Can be specified multiple times.

+
--output-format output-format

The format to use for printing diagnostic messages

+

Possible values:

+
    +
  • full: Print diagnostics verbosely, with context and helpful hints
    • +
  • concise: Print diagnostics concisely, one per line
    • +
--project project

Run the command within the given project directory.

+

All pyproject.toml files will be discovered by walking up the directory tree from the given project directory, as will the project's virtual environment (.venv) unless the venv-path option is set.

+

Other command-line arguments (such as relative paths) will be resolved relative to the current working directory.

+
--python path

Path to the Python installation from which ty resolves type information and third-party dependencies.

+

If not specified, ty will look at the VIRTUAL_ENV environment variable.

+

ty will search in the path's site-packages directories for type information and third-party imports.

+

This option is commonly used to specify the path to a virtual environment.

+
--python-platform, --platform platform

Target platform to assume when resolving types.

+

This is used to specialize the type of sys.platform and will affect the visibility of platform-specific functions and attributes. If the value is set to all, no assumptions are made about the target platform. If unspecified, the current system's platform will be used.

+
--python-version, --target-version version

Python version to assume when resolving types

+

Possible values:

+
    +
  • 3.7
    • +
  • 3.8
    • +
  • 3.9
    • +
  • 3.10
    • +
  • 3.11
    • +
  • 3.12
    • +
  • 3.13
    • +
--respect-ignore-files

Respect file exclusions via .gitignore and other standard ignore files. Use --no-respect-gitignore to disable

+
--typeshed, --custom-typeshed-dir path

Custom directory to use for stdlib typeshed stubs

+
--verbose, -v

Use verbose output (or -vv and -vvv for more verbose output)

+
--warn rule

Treat the given rule as having severity 'warn'. Can be specified multiple times.

+
--watch, -W

Watch files for changes and recheck files related to the changed files

+
+ +## ty server + +Start the language server + +

Usage

+ +``` +ty server +``` + +

Options

+ +
--help, -h

Print help

+
+ +## ty version + +Display ty's version + +

Usage

+ +``` +ty version +``` + +

Options

+ +
--help, -h

Print help

+
+ +## ty generate-shell-completion + +Generate shell completion + +

Usage

+ +``` +ty generate-shell-completion +``` + +

Arguments

+ +
SHELL
+ +

Options

+ +
--help, -h

Print help

+
+ +## ty help + +Print this message or the help of the given subcommand(s) + +

Usage

+ +``` +ty help [COMMAND] +``` + diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 1d42c7a78a..c4e8988e80 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -11,7 +11,7 @@ use ty_python_semantic::lint; #[derive(Debug, Parser)] #[command(author, name = "ty", about = "An extremely fast Python type checker.")] #[command(long_version = crate::version::version())] -pub(crate) struct Args { +pub struct Cli { #[command(subcommand)] pub(crate) command: Command, } diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs new file mode 100644 index 0000000000..4aa4d70a67 --- /dev/null +++ b/crates/ty/src/lib.rs @@ -0,0 +1,396 @@ +mod args; +mod logging; +mod python_version; +mod version; + +pub use args::Cli; + +use std::io::{self, stdout, BufWriter, Write}; +use std::process::{ExitCode, Termination}; + +use anyhow::Result; +use std::sync::Mutex; + +use crate::args::{CheckCommand, Command, TerminalColor}; +use crate::logging::setup_tracing; +use anyhow::{anyhow, Context}; +use clap::{CommandFactory, Parser}; +use crossbeam::channel as crossbeam_channel; +use rayon::ThreadPoolBuilder; +use ruff_db::diagnostic::{Diagnostic, DisplayDiagnosticConfig, Severity}; +use ruff_db::max_parallelism; +use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; +use ruff_db::Upcast; +use salsa::plumbing::ZalsaDatabase; +use ty_project::metadata::options::Options; +use ty_project::watch::ProjectWatcher; +use ty_project::{watch, Db}; +use ty_project::{ProjectDatabase, ProjectMetadata}; +use ty_server::run_server; + +pub fn run() -> anyhow::Result { + setup_rayon(); + + let args = wild::args_os(); + let args = argfile::expand_args_from(args, argfile::parse_fromfile, argfile::PREFIX) + .context("Failed to read CLI arguments from file")?; + let args = Cli::parse_from(args); + + match args.command { + Command::Server => run_server().map(|()| ExitStatus::Success), + Command::Check(check_args) => run_check(check_args), + Command::Version => version().map(|()| ExitStatus::Success), + Command::GenerateShellCompletion { shell } => { + shell.generate(&mut Cli::command(), &mut stdout()); + Ok(ExitStatus::Success) + } + } +} + +pub(crate) fn version() -> Result<()> { + let mut stdout = BufWriter::new(io::stdout().lock()); + let version_info = crate::version::version(); + writeln!(stdout, "ty {}", &version_info)?; + Ok(()) +} + +fn run_check(args: CheckCommand) -> anyhow::Result { + set_colored_override(args.color); + + let verbosity = args.verbosity.level(); + countme::enable(verbosity.is_trace()); + let _guard = setup_tracing(verbosity)?; + + tracing::debug!("Version: {}", version::version()); + + // The base path to which all CLI arguments are relative to. + let cwd = { + let cwd = std::env::current_dir().context("Failed to get the current working directory")?; + SystemPathBuf::from_path_buf(cwd) + .map_err(|path| { + anyhow!( + "The current working directory `{}` contains non-Unicode characters. ty only supports Unicode paths.", + path.display() + ) + })? + }; + + let project_path = args + .project + .as_ref() + .map(|project| { + if project.as_std_path().is_dir() { + Ok(SystemPath::absolute(project, &cwd)) + } else { + Err(anyhow!( + "Provided project path `{project}` is not a directory" + )) + } + }) + .transpose()? + .unwrap_or_else(|| cwd.clone()); + + let check_paths: Vec<_> = args + .paths + .iter() + .map(|path| SystemPath::absolute(path, &cwd)) + .collect(); + + let system = OsSystem::new(cwd); + let watch = args.watch; + let exit_zero = args.exit_zero; + + let cli_options = args.into_options(); + let mut project_metadata = ProjectMetadata::discover(&project_path, &system)?; + project_metadata.apply_cli_options(cli_options.clone()); + project_metadata.apply_configuration_files(&system)?; + + let mut db = ProjectDatabase::new(project_metadata, system)?; + + if !check_paths.is_empty() { + db.project().set_included_paths(&mut db, check_paths); + } + + let (main_loop, main_loop_cancellation_token) = MainLoop::new(cli_options); + + // Listen to Ctrl+C and abort the watch mode. + let main_loop_cancellation_token = Mutex::new(Some(main_loop_cancellation_token)); + ctrlc::set_handler(move || { + let mut lock = main_loop_cancellation_token.lock().unwrap(); + + if let Some(token) = lock.take() { + token.stop(); + } + })?; + + let exit_status = if watch { + main_loop.watch(&mut db)? + } else { + main_loop.run(&mut db)? + }; + + tracing::trace!("Counts for entire CLI run:\n{}", countme::get_all()); + + std::mem::forget(db); + + if exit_zero { + Ok(ExitStatus::Success) + } else { + Ok(exit_status) + } +} + +#[derive(Copy, Clone)] +pub enum ExitStatus { + /// Checking was successful and there were no errors. + Success = 0, + + /// Checking was successful but there were errors. + Failure = 1, + + /// Checking failed due to an invocation error (e.g. the current directory no longer exists, incorrect CLI arguments, ...) + Error = 2, + + /// Internal ty error (panic, or any other error that isn't due to the user using the + /// program incorrectly or transient environment errors). + InternalError = 101, +} + +impl Termination for ExitStatus { + fn report(self) -> ExitCode { + ExitCode::from(self as u8) + } +} + +struct MainLoop { + /// Sender that can be used to send messages to the main loop. + sender: crossbeam_channel::Sender, + + /// Receiver for the messages sent **to** the main loop. + receiver: crossbeam_channel::Receiver, + + /// The file system watcher, if running in watch mode. + watcher: Option, + + cli_options: Options, +} + +impl MainLoop { + fn new(cli_options: Options) -> (Self, MainLoopCancellationToken) { + let (sender, receiver) = crossbeam_channel::bounded(10); + + ( + Self { + sender: sender.clone(), + receiver, + watcher: None, + cli_options, + }, + MainLoopCancellationToken { sender }, + ) + } + + fn watch(mut self, db: &mut ProjectDatabase) -> Result { + tracing::debug!("Starting watch mode"); + let sender = self.sender.clone(); + let watcher = watch::directory_watcher(move |event| { + sender.send(MainLoopMessage::ApplyChanges(event)).unwrap(); + })?; + + self.watcher = Some(ProjectWatcher::new(watcher, db)); + + self.run(db)?; + + Ok(ExitStatus::Success) + } + + fn run(mut self, db: &mut ProjectDatabase) -> Result { + self.sender.send(MainLoopMessage::CheckWorkspace).unwrap(); + + let result = self.main_loop(db); + + tracing::debug!("Exiting main loop"); + + result + } + + fn main_loop(&mut self, db: &mut ProjectDatabase) -> Result { + // Schedule the first check. + tracing::debug!("Starting main loop"); + + let mut revision = 0u64; + + while let Ok(message) = self.receiver.recv() { + match message { + MainLoopMessage::CheckWorkspace => { + let db = db.clone(); + let sender = self.sender.clone(); + + // Spawn a new task that checks the project. This needs to be done in a separate thread + // to prevent blocking the main loop here. + rayon::spawn(move || { + match db.check() { + Ok(result) => { + // Send the result back to the main loop for printing. + sender + .send(MainLoopMessage::CheckCompleted { result, revision }) + .unwrap(); + } + Err(cancelled) => { + tracing::debug!("Check has been cancelled: {cancelled:?}"); + } + } + }); + } + + MainLoopMessage::CheckCompleted { + result, + revision: check_revision, + } => { + let terminal_settings = db.project().settings(db).terminal(); + let display_config = DisplayDiagnosticConfig::default() + .format(terminal_settings.output_format) + .color(colored::control::SHOULD_COLORIZE.should_colorize()); + + if check_revision == revision { + if db.project().files(db).is_empty() { + tracing::warn!("No python files found under the given path(s)"); + } + + let mut stdout = stdout().lock(); + + if result.is_empty() { + writeln!(stdout, "All checks passed!")?; + + if self.watcher.is_none() { + return Ok(ExitStatus::Success); + } + } else { + let mut max_severity = Severity::Info; + let diagnostics_count = result.len(); + + for diagnostic in result { + write!( + stdout, + "{}", + diagnostic.display(&db.upcast(), &display_config) + )?; + + max_severity = max_severity.max(diagnostic.severity()); + } + + writeln!( + stdout, + "Found {} diagnostic{}", + diagnostics_count, + if diagnostics_count > 1 { "s" } else { "" } + )?; + + if max_severity.is_fatal() { + tracing::warn!("A fatal error occurred while checking some files. Not all project files were analyzed. See the diagnostics list above for details."); + } + + if self.watcher.is_none() { + return Ok(match max_severity { + Severity::Info => ExitStatus::Success, + Severity::Warning => { + if terminal_settings.error_on_warning { + ExitStatus::Failure + } else { + ExitStatus::Success + } + } + Severity::Error => ExitStatus::Failure, + Severity::Fatal => ExitStatus::InternalError, + }); + } + } + } else { + tracing::debug!( + "Discarding check result for outdated revision: current: {revision}, result revision: {check_revision}" + ); + } + + tracing::trace!("Counts after last check:\n{}", countme::get_all()); + } + + MainLoopMessage::ApplyChanges(changes) => { + revision += 1; + // Automatically cancels any pending queries and waits for them to complete. + db.apply_changes(changes, Some(&self.cli_options)); + if let Some(watcher) = self.watcher.as_mut() { + watcher.update(db); + } + self.sender.send(MainLoopMessage::CheckWorkspace).unwrap(); + } + MainLoopMessage::Exit => { + // Cancel any pending queries and wait for them to complete. + // TODO: Don't use Salsa internal APIs + // [Zulip-Thread](https://salsa.zulipchat.com/#narrow/stream/333573-salsa-3.2E0/topic/Expose.20an.20API.20to.20cancel.20other.20queries) + let _ = db.zalsa_mut(); + return Ok(ExitStatus::Success); + } + } + + tracing::debug!("Waiting for next main loop message."); + } + + Ok(ExitStatus::Success) + } +} + +#[derive(Debug)] +struct MainLoopCancellationToken { + sender: crossbeam_channel::Sender, +} + +impl MainLoopCancellationToken { + fn stop(self) { + self.sender.send(MainLoopMessage::Exit).unwrap(); + } +} + +/// Message sent from the orchestrator to the main loop. +#[derive(Debug)] +enum MainLoopMessage { + CheckWorkspace, + CheckCompleted { + /// The diagnostics that were found during the check. + result: Vec, + revision: u64, + }, + ApplyChanges(Vec), + Exit, +} + +fn set_colored_override(color: Option) { + let Some(color) = color else { + return; + }; + + match color { + TerminalColor::Auto => { + colored::control::unset_override(); + } + TerminalColor::Always => { + colored::control::set_override(true); + } + TerminalColor::Never => { + colored::control::set_override(false); + } + } +} + +/// Initializes the global rayon thread pool to never use more than `TY_MAX_PARALLELISM` threads. +fn setup_rayon() { + ThreadPoolBuilder::default() + .num_threads(max_parallelism().get()) + // Use a reasonably large stack size to avoid running into stack overflows too easily. The + // size was chosen in such a way as to still be able to handle large expressions involving + // binary operators (x + x + … + x) both during the AST walk in semantic index building as + // well as during type checking. Using this stack size, we can handle handle expressions + // that are several times larger than the corresponding limits in existing type checkers. + .stack_size(16 * 1024 * 1024) + .build_global() + .unwrap(); +} diff --git a/crates/ty/src/main.rs b/crates/ty/src/main.rs index 2f5e31358b..2cc2b48a0a 100644 --- a/crates/ty/src/main.rs +++ b/crates/ty/src/main.rs @@ -1,40 +1,13 @@ -use std::io::{self, stdout, BufWriter, Write}; -use std::process::{ExitCode, Termination}; - -use anyhow::Result; -use std::sync::Mutex; - -use crate::args::{Args, CheckCommand, Command, TerminalColor}; -use crate::logging::setup_tracing; -use anyhow::{anyhow, Context}; -use clap::{CommandFactory, Parser}; use colored::Colorize; -use crossbeam::channel as crossbeam_channel; -use rayon::ThreadPoolBuilder; -use ruff_db::diagnostic::{Diagnostic, DisplayDiagnosticConfig, Severity}; -use ruff_db::max_parallelism; -use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; -use ruff_db::Upcast; -use salsa::plumbing::ZalsaDatabase; -use ty_project::metadata::options::Options; -use ty_project::watch::ProjectWatcher; -use ty_project::{watch, Db}; -use ty_project::{ProjectDatabase, ProjectMetadata}; -use ty_server::run_server; - -mod args; -mod logging; -mod python_version; -mod version; +use std::io; +use ty::{run, ExitStatus}; pub fn main() -> ExitStatus { - setup_rayon(); - run().unwrap_or_else(|error| { - use std::io::Write; + use io::Write; // Use `writeln` instead of `eprintln` to avoid panicking when the stderr pipe is broken. - let mut stderr = std::io::stderr().lock(); + let mut stderr = io::stderr().lock(); // This communicates that this isn't a linter error but ty itself hard-errored for // some reason (e.g. failed to resolve the configuration) @@ -58,368 +31,3 @@ pub fn main() -> ExitStatus { ExitStatus::Error }) } - -fn run() -> anyhow::Result { - let args = wild::args_os(); - let args = argfile::expand_args_from(args, argfile::parse_fromfile, argfile::PREFIX) - .context("Failed to read CLI arguments from file")?; - let args = Args::parse_from(args); - - match args.command { - Command::Server => run_server().map(|()| ExitStatus::Success), - Command::Check(check_args) => run_check(check_args), - Command::Version => version().map(|()| ExitStatus::Success), - Command::GenerateShellCompletion { shell } => { - shell.generate(&mut Args::command(), &mut stdout()); - Ok(ExitStatus::Success) - } - } -} - -pub(crate) fn version() -> Result<()> { - let mut stdout = BufWriter::new(io::stdout().lock()); - let version_info = crate::version::version(); - writeln!(stdout, "ty {}", &version_info)?; - Ok(()) -} - -fn run_check(args: CheckCommand) -> anyhow::Result { - set_colored_override(args.color); - - let verbosity = args.verbosity.level(); - countme::enable(verbosity.is_trace()); - let _guard = setup_tracing(verbosity)?; - - tracing::debug!("Version: {}", version::version()); - - // The base path to which all CLI arguments are relative to. - let cwd = { - let cwd = std::env::current_dir().context("Failed to get the current working directory")?; - SystemPathBuf::from_path_buf(cwd) - .map_err(|path| { - anyhow!( - "The current working directory `{}` contains non-Unicode characters. ty only supports Unicode paths.", - path.display() - ) - })? - }; - - let project_path = args - .project - .as_ref() - .map(|project| { - if project.as_std_path().is_dir() { - Ok(SystemPath::absolute(project, &cwd)) - } else { - Err(anyhow!( - "Provided project path `{project}` is not a directory" - )) - } - }) - .transpose()? - .unwrap_or_else(|| cwd.clone()); - - let check_paths: Vec<_> = args - .paths - .iter() - .map(|path| SystemPath::absolute(path, &cwd)) - .collect(); - - let system = OsSystem::new(cwd); - let watch = args.watch; - let exit_zero = args.exit_zero; - - let cli_options = args.into_options(); - let mut project_metadata = ProjectMetadata::discover(&project_path, &system)?; - project_metadata.apply_cli_options(cli_options.clone()); - project_metadata.apply_configuration_files(&system)?; - - let mut db = ProjectDatabase::new(project_metadata, system)?; - - if !check_paths.is_empty() { - db.project().set_included_paths(&mut db, check_paths); - } - - let (main_loop, main_loop_cancellation_token) = MainLoop::new(cli_options); - - // Listen to Ctrl+C and abort the watch mode. - let main_loop_cancellation_token = Mutex::new(Some(main_loop_cancellation_token)); - ctrlc::set_handler(move || { - let mut lock = main_loop_cancellation_token.lock().unwrap(); - - if let Some(token) = lock.take() { - token.stop(); - } - })?; - - let exit_status = if watch { - main_loop.watch(&mut db)? - } else { - main_loop.run(&mut db)? - }; - - tracing::trace!("Counts for entire CLI run:\n{}", countme::get_all()); - - std::mem::forget(db); - - if exit_zero { - Ok(ExitStatus::Success) - } else { - Ok(exit_status) - } -} - -#[derive(Copy, Clone)] -pub enum ExitStatus { - /// Checking was successful and there were no errors. - Success = 0, - - /// Checking was successful but there were errors. - Failure = 1, - - /// Checking failed due to an invocation error (e.g. the current directory no longer exists, incorrect CLI arguments, ...) - Error = 2, - - /// Internal ty error (panic, or any other error that isn't due to the user using the - /// program incorrectly or transient environment errors). - InternalError = 101, -} - -impl Termination for ExitStatus { - fn report(self) -> ExitCode { - ExitCode::from(self as u8) - } -} - -struct MainLoop { - /// Sender that can be used to send messages to the main loop. - sender: crossbeam_channel::Sender, - - /// Receiver for the messages sent **to** the main loop. - receiver: crossbeam_channel::Receiver, - - /// The file system watcher, if running in watch mode. - watcher: Option, - - cli_options: Options, -} - -impl MainLoop { - fn new(cli_options: Options) -> (Self, MainLoopCancellationToken) { - let (sender, receiver) = crossbeam_channel::bounded(10); - - ( - Self { - sender: sender.clone(), - receiver, - watcher: None, - cli_options, - }, - MainLoopCancellationToken { sender }, - ) - } - - fn watch(mut self, db: &mut ProjectDatabase) -> Result { - tracing::debug!("Starting watch mode"); - let sender = self.sender.clone(); - let watcher = watch::directory_watcher(move |event| { - sender.send(MainLoopMessage::ApplyChanges(event)).unwrap(); - })?; - - self.watcher = Some(ProjectWatcher::new(watcher, db)); - - self.run(db)?; - - Ok(ExitStatus::Success) - } - - fn run(mut self, db: &mut ProjectDatabase) -> Result { - self.sender.send(MainLoopMessage::CheckWorkspace).unwrap(); - - let result = self.main_loop(db); - - tracing::debug!("Exiting main loop"); - - result - } - - fn main_loop(&mut self, db: &mut ProjectDatabase) -> Result { - // Schedule the first check. - tracing::debug!("Starting main loop"); - - let mut revision = 0u64; - - while let Ok(message) = self.receiver.recv() { - match message { - MainLoopMessage::CheckWorkspace => { - let db = db.clone(); - let sender = self.sender.clone(); - - // Spawn a new task that checks the project. This needs to be done in a separate thread - // to prevent blocking the main loop here. - rayon::spawn(move || { - match db.check() { - Ok(result) => { - // Send the result back to the main loop for printing. - sender - .send(MainLoopMessage::CheckCompleted { result, revision }) - .unwrap(); - } - Err(cancelled) => { - tracing::debug!("Check has been cancelled: {cancelled:?}"); - } - } - }); - } - - MainLoopMessage::CheckCompleted { - result, - revision: check_revision, - } => { - let terminal_settings = db.project().settings(db).terminal(); - let display_config = DisplayDiagnosticConfig::default() - .format(terminal_settings.output_format) - .color(colored::control::SHOULD_COLORIZE.should_colorize()); - - if check_revision == revision { - if db.project().files(db).is_empty() { - tracing::warn!("No python files found under the given path(s)"); - } - - let mut stdout = stdout().lock(); - - if result.is_empty() { - writeln!(stdout, "All checks passed!")?; - - if self.watcher.is_none() { - return Ok(ExitStatus::Success); - } - } else { - let mut max_severity = Severity::Info; - let diagnostics_count = result.len(); - - for diagnostic in result { - write!( - stdout, - "{}", - diagnostic.display(&db.upcast(), &display_config) - )?; - - max_severity = max_severity.max(diagnostic.severity()); - } - - writeln!( - stdout, - "Found {} diagnostic{}", - diagnostics_count, - if diagnostics_count > 1 { "s" } else { "" } - )?; - - if max_severity.is_fatal() { - tracing::warn!("A fatal error occurred while checking some files. Not all project files were analyzed. See the diagnostics list above for details."); - } - - if self.watcher.is_none() { - return Ok(match max_severity { - Severity::Info => ExitStatus::Success, - Severity::Warning => { - if terminal_settings.error_on_warning { - ExitStatus::Failure - } else { - ExitStatus::Success - } - } - Severity::Error => ExitStatus::Failure, - Severity::Fatal => ExitStatus::InternalError, - }); - } - } - } else { - tracing::debug!( - "Discarding check result for outdated revision: current: {revision}, result revision: {check_revision}" - ); - } - - tracing::trace!("Counts after last check:\n{}", countme::get_all()); - } - - MainLoopMessage::ApplyChanges(changes) => { - revision += 1; - // Automatically cancels any pending queries and waits for them to complete. - db.apply_changes(changes, Some(&self.cli_options)); - if let Some(watcher) = self.watcher.as_mut() { - watcher.update(db); - } - self.sender.send(MainLoopMessage::CheckWorkspace).unwrap(); - } - MainLoopMessage::Exit => { - // Cancel any pending queries and wait for them to complete. - // TODO: Don't use Salsa internal APIs - // [Zulip-Thread](https://salsa.zulipchat.com/#narrow/stream/333573-salsa-3.2E0/topic/Expose.20an.20API.20to.20cancel.20other.20queries) - let _ = db.zalsa_mut(); - return Ok(ExitStatus::Success); - } - } - - tracing::debug!("Waiting for next main loop message."); - } - - Ok(ExitStatus::Success) - } -} - -#[derive(Debug)] -struct MainLoopCancellationToken { - sender: crossbeam_channel::Sender, -} - -impl MainLoopCancellationToken { - fn stop(self) { - self.sender.send(MainLoopMessage::Exit).unwrap(); - } -} - -/// Message sent from the orchestrator to the main loop. -#[derive(Debug)] -enum MainLoopMessage { - CheckWorkspace, - CheckCompleted { - /// The diagnostics that were found during the check. - result: Vec, - revision: u64, - }, - ApplyChanges(Vec), - Exit, -} - -fn set_colored_override(color: Option) { - let Some(color) = color else { - return; - }; - - match color { - TerminalColor::Auto => { - colored::control::unset_override(); - } - TerminalColor::Always => { - colored::control::set_override(true); - } - TerminalColor::Never => { - colored::control::set_override(false); - } - } -} - -/// Initializes the global rayon thread pool to never use more than `TY_MAX_PARALLELISM` threads. -fn setup_rayon() { - ThreadPoolBuilder::default() - .num_threads(max_parallelism().get()) - // Use a reasonably large stack size to avoid running into stack overflows too easily. The - // size was chosen in such a way as to still be able to handle large expressions involving - // binary operators (x + x + … + x) both during the AST walk in semantic index building as - // well as during type checking. Using this stack size, we can handle handle expressions - // that are several times larger than the corresponding limits in existing type checkers. - .stack_size(16 * 1024 * 1024) - .build_global() - .unwrap(); -}