Add a subcommand to generate dependency graphs (#13402)

## Summary

This PR adds an experimental Ruff subcommand to generate dependency
graphs based on module resolution.

A few highlights:

- You can generate either dependency or dependent graphs via the
`--direction` command-line argument.
- Like Pants, we also provide an option to identify imports from string
literals (`--detect-string-imports`).
- Users can also provide additional dependency data via the
`include-dependencies` key under `[tool.ruff.import-map]`. This map uses
file paths as keys, and lists of strings as values. Those strings can be
file paths or globs.

The dependency resolution uses the red-knot module resolver which is
intended to be fully spec compliant, so it's also a chance to expose the
module resolver in a real-world setting.

The CLI is, e.g., `ruff graph build ../autobot`, which will output a
JSON map from file to files it depends on for the `autobot` project.
This commit is contained in:
Charlie Marsh 2024-09-19 21:06:32 -04:00 committed by GitHub
parent 260c2ecd15
commit 4e935f7d7d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 1339 additions and 45 deletions

View file

@ -3,6 +3,7 @@
//! the various parameters.
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::env::VarError;
use std::num::{NonZeroU16, NonZeroU8};
use std::path::{Path, PathBuf};
@ -19,6 +20,7 @@ use strum::IntoEnumIterator;
use ruff_cache::cache_dir;
use ruff_formatter::IndentStyle;
use ruff_graph::{AnalyzeSettings, Direction};
use ruff_linter::line_width::{IndentWidth, LineLength};
use ruff_linter::registry::RuleNamespace;
use ruff_linter::registry::{Rule, RuleSet, INCOMPATIBLE_CODES};
@ -40,11 +42,11 @@ use ruff_python_formatter::{
};
use crate::options::{
Flake8AnnotationsOptions, Flake8BanditOptions, Flake8BooleanTrapOptions, Flake8BugbearOptions,
Flake8BuiltinsOptions, Flake8ComprehensionsOptions, Flake8CopyrightOptions,
Flake8ErrMsgOptions, Flake8GetTextOptions, Flake8ImplicitStrConcatOptions,
Flake8ImportConventionsOptions, Flake8PytestStyleOptions, Flake8QuotesOptions,
Flake8SelfOptions, Flake8TidyImportsOptions, Flake8TypeCheckingOptions,
AnalyzeOptions, Flake8AnnotationsOptions, Flake8BanditOptions, Flake8BooleanTrapOptions,
Flake8BugbearOptions, Flake8BuiltinsOptions, Flake8ComprehensionsOptions,
Flake8CopyrightOptions, Flake8ErrMsgOptions, Flake8GetTextOptions,
Flake8ImplicitStrConcatOptions, Flake8ImportConventionsOptions, Flake8PytestStyleOptions,
Flake8QuotesOptions, Flake8SelfOptions, Flake8TidyImportsOptions, Flake8TypeCheckingOptions,
Flake8UnusedArgumentsOptions, FormatOptions, IsortOptions, LintCommonOptions, LintOptions,
McCabeOptions, Options, Pep8NamingOptions, PyUpgradeOptions, PycodestyleOptions,
PydocstyleOptions, PyflakesOptions, PylintOptions, RuffOptions,
@ -142,6 +144,7 @@ pub struct Configuration {
pub lint: LintConfiguration,
pub format: FormatConfiguration,
pub analyze: AnalyzeConfiguration,
}
impl Configuration {
@ -207,6 +210,21 @@ impl Configuration {
.unwrap_or(format_defaults.docstring_code_line_width),
};
let analyze = self.analyze;
let analyze_preview = analyze.preview.unwrap_or(global_preview);
let analyze_defaults = AnalyzeSettings::default();
let analyze = AnalyzeSettings {
preview: analyze_preview,
extension: self.extension.clone().unwrap_or_default(),
detect_string_imports: analyze
.detect_string_imports
.unwrap_or(analyze_defaults.detect_string_imports),
include_dependencies: analyze
.include_dependencies
.unwrap_or(analyze_defaults.include_dependencies),
};
let lint = self.lint;
let lint_preview = lint.preview.unwrap_or(global_preview);
@ -401,6 +419,7 @@ impl Configuration {
},
formatter,
analyze,
})
}
@ -534,6 +553,10 @@ impl Configuration {
options.format.unwrap_or_default(),
project_root,
)?,
analyze: AnalyzeConfiguration::from_options(
options.analyze.unwrap_or_default(),
project_root,
)?,
})
}
@ -573,6 +596,7 @@ impl Configuration {
lint: self.lint.combine(config.lint),
format: self.format.combine(config.format),
analyze: self.analyze.combine(config.analyze),
}
}
}
@ -1191,6 +1215,45 @@ impl FormatConfiguration {
}
}
}
#[derive(Clone, Debug, Default)]
pub struct AnalyzeConfiguration {
pub preview: Option<PreviewMode>,
pub direction: Option<Direction>,
pub detect_string_imports: Option<bool>,
pub include_dependencies: Option<BTreeMap<PathBuf, (PathBuf, Vec<String>)>>,
}
impl AnalyzeConfiguration {
#[allow(clippy::needless_pass_by_value)]
pub fn from_options(options: AnalyzeOptions, project_root: &Path) -> Result<Self> {
Ok(Self {
preview: options.preview.map(PreviewMode::from),
direction: options.direction,
detect_string_imports: options.detect_string_imports,
include_dependencies: options.include_dependencies.map(|dependencies| {
dependencies
.into_iter()
.map(|(key, value)| {
(project_root.join(key), (project_root.to_path_buf(), value))
})
.collect::<BTreeMap<_, _>>()
}),
})
}
#[must_use]
#[allow(clippy::needless_pass_by_value)]
pub fn combine(self, config: Self) -> Self {
Self {
preview: self.preview.or(config.preview),
direction: self.direction.or(config.direction),
detect_string_imports: self.detect_string_imports.or(config.detect_string_imports),
include_dependencies: self.include_dependencies.or(config.include_dependencies),
}
}
}
pub(crate) trait CombinePluginOptions {
#[must_use]
fn combine(self, other: Self) -> Self;

View file

@ -1,11 +1,14 @@
use regex::Regex;
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::path::PathBuf;
use strum::IntoEnumIterator;
use crate::options_base::{OptionsMetadata, Visit};
use crate::settings::LineEnding;
use ruff_formatter::IndentStyle;
use ruff_graph::Direction;
use ruff_linter::line_width::{IndentWidth, LineLength};
use ruff_linter::rules::flake8_import_conventions::settings::BannedAliases;
use ruff_linter::rules::flake8_pytest_style::settings::SettingsError;
@ -433,6 +436,10 @@ pub struct Options {
/// Options to configure code formatting.
#[option_group]
pub format: Option<FormatOptions>,
/// Options to configure import map generation.
#[option_group]
pub analyze: Option<AnalyzeOptions>,
}
/// Configures how Ruff checks your code.
@ -3306,6 +3313,59 @@ pub struct FormatOptions {
pub docstring_code_line_length: Option<DocstringCodeLineWidth>,
}
/// Configures Ruff's `analyze` command.
#[derive(
Clone, Debug, PartialEq, Eq, Default, Deserialize, Serialize, OptionsMetadata, CombineOptions,
)]
#[serde(deny_unknown_fields, rename_all = "kebab-case")]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct AnalyzeOptions {
/// Whether to enable preview mode. When preview mode is enabled, Ruff will expose unstable
/// commands.
#[option(
default = "false",
value_type = "bool",
example = r#"
# Enable preview features.
preview = true
"#
)]
pub preview: Option<bool>,
/// Whether to generate a map from file to files that it depends on (dependencies) or files that
/// depend on it (dependents).
#[option(
default = r#"\"dependencies\""#,
value_type = "\"dependents\" | \"dependencies\"",
example = r#"
direction = "dependencies"
"#
)]
pub direction: Option<Direction>,
/// Whether to detect imports from string literals. When enabled, Ruff will search for string
/// literals that "look like" import paths, and include them in the import map, if they resolve
/// to valid Python modules.
#[option(
default = "false",
value_type = "bool",
example = r#"
detect-string-imports = true
"#
)]
pub detect_string_imports: Option<bool>,
/// A map from file path to the list of file paths or globs that should be considered
/// dependencies of that file, regardless of whether relevant imports are detected.
#[option(
default = "{}",
value_type = "dict[str, list[str]]",
example = r#"
include-dependencies = {
"foo/bar.py": ["foo/baz/*.py"],
}
"#
)]
pub include_dependencies: Option<BTreeMap<PathBuf, Vec<String>>>,
}
#[cfg(test)]
mod tests {
use crate::options::Flake8SelfOptions;

View file

@ -395,7 +395,6 @@ pub fn python_files_in_path<'a>(
let walker = builder.build_parallel();
// Run the `WalkParallel` to collect all Python files.
let state = WalkPythonFilesState::new(resolver);
let mut visitor = PythonFilesVisitorBuilder::new(transformer, &state);
walker.visit(&mut visitor);

View file

@ -1,6 +1,7 @@
use path_absolutize::path_dedot;
use ruff_cache::cache_dir;
use ruff_formatter::{FormatOptions, IndentStyle, IndentWidth, LineWidth};
use ruff_graph::AnalyzeSettings;
use ruff_linter::display_settings;
use ruff_linter::settings::types::{
ExtensionMapping, FilePattern, FilePatternSet, OutputFormat, UnsafeFixes,
@ -35,6 +36,7 @@ pub struct Settings {
pub file_resolver: FileResolverSettings,
pub linter: LinterSettings,
pub formatter: FormatterSettings,
pub analyze: AnalyzeSettings,
}
impl Default for Settings {
@ -50,6 +52,7 @@ impl Default for Settings {
linter: LinterSettings::new(project_root),
file_resolver: FileResolverSettings::new(project_root),
formatter: FormatterSettings::default(),
analyze: AnalyzeSettings::default(),
}
}
}
@ -68,7 +71,8 @@ impl fmt::Display for Settings {
self.unsafe_fixes,
self.file_resolver | nested,
self.linter | nested,
self.formatter | nested
self.formatter | nested,
self.analyze | nested,
]
}
Ok(())