Add TOML files to SourceType (#6929)

## Summary

This PR adds a higher-level enum (`SourceType`) around `PySourceType` to
allow us to use the same detection path to handle TOML files. Right now,
we have ad hoc `is_pyproject_toml` checks littered around, and some
codepaths are omitting that logic altogether (like `add_noqa`). Instead,
we should always be required to check the source type and handle TOML
files as appropriate.

This PR will also help with our pre-commit capabilities. If we add
`toml` to pre-commit (to support `pyproject.toml`), pre-commit will
start to pass _other_ files to Ruff (along with `poetry.lock` and
`Pipfile` -- see
[identify](b59996304f/identify/extensions.py (L355))).
By detecting those files and handling those cases, we avoid attempting
to parse them as Python files, which would lead to pre-commit errors.
(We tried to add `toml` to pre-commit here
(https://github.com/astral-sh/ruff-pre-commit/pull/44), but had to
revert here (https://github.com/astral-sh/ruff-pre-commit/pull/45) as it
led to the pre-commit hook attempting to parse `poetry.lock` files as
Python files.)

crates/ruff_cli/src/commands/add_noqa.rs

@@ -8,7 +8,7 @@ use rayon::prelude::*;
 
 use ruff::linter::add_noqa_to_path;
 use ruff::warn_user_once;
-use ruff_python_stdlib::path::{is_jupyter_notebook, is_project_toml};
+use ruff_python_ast::{PySourceType, SourceType};
 use ruff_workspace::resolver::{python_files_in_path, PyprojectConfig};
 
 use crate::args::Overrides;
@@ -46,15 +46,17 @@ pub(crate) fn add_noqa(
         .flatten()
         .filter_map(|entry| {
             let path = entry.path();
-            if is_project_toml(path) || is_jupyter_notebook(path) {
+            let SourceType::Python(source_type @ (PySourceType::Python | PySourceType::Stub)) =
+                SourceType::from(path)
+            else {
                 return None;
-            }
+            };
             let package = path
                 .parent()
                 .and_then(|parent| package_roots.get(parent))
                 .and_then(|package| *package);
             let settings = resolver.resolve(path, pyproject_config);
-            match add_noqa_to_path(path, package, settings) {
+            match add_noqa_to_path(path, package, source_type, settings) {
                 Ok(count) => Some(count),
                 Err(e) => {
                     error!("Failed to add noqa to {}: {e}", path.display());

crates/ruff_cli/src/commands/format.rs

@@ -12,7 +12,7 @@ use tracing::{span, Level};
 use ruff::fs;
 use ruff::warn_user_once;
 use ruff_formatter::LineWidth;
-use ruff_python_ast::PySourceType;
+use ruff_python_ast::{PySourceType, SourceType};
 use ruff_python_formatter::{format_module, FormatModuleError, PyFormatOptions};
 use ruff_workspace::resolver::python_files_in_path;
 
@@ -37,23 +37,21 @@ pub(crate) fn format(cli: &Arguments, overrides: &Overrides) -> Result<ExitStatu
 
     let result = paths
         .into_par_iter()
-        .map(|dir_entry| {
-            let dir_entry = dir_entry?;
-            let path = dir_entry.path();
-            let source_type = PySourceType::from(path);
-            if !(source_type.is_python() || source_type.is_stub())
-                || path
-                    .extension()
-                    .is_some_and(|extension| extension == "toml")
-            {
-                return Ok(());
+        .map(|entry| {
+            let entry = entry?;
+            let path = entry.path();
+            if matches!(
+                SourceType::from(path),
+                SourceType::Python(PySourceType::Python | PySourceType::Stub)
+            ) {
+                let line_length = resolver.resolve(path, &pyproject_config).line_length;
+                let options = PyFormatOptions::from_extension(path)
+                    .with_line_width(LineWidth::from(NonZeroU16::from(line_length)));
+
+                format_path(path, options)
+            } else {
+                Ok(())
             }
-
-            let line_length = resolver.resolve(path, &pyproject_config).line_length;
-            let options = PyFormatOptions::from_extension(path)
-                .with_line_width(LineWidth::from(NonZeroU16::from(line_length)));
-
-            format_path(path, options)
         })
         .map(|result| {
             result.map_err(|err| {

crates/ruff_cli/src/diagnostics.rs

@@ -27,8 +27,7 @@ use ruff::{fs, IOError};
 use ruff_diagnostics::Diagnostic;
 use ruff_macros::CacheKey;
 use ruff_python_ast::imports::ImportMap;
-use ruff_python_ast::PySourceType;
-use ruff_python_stdlib::path::is_project_toml;
+use ruff_python_ast::{PySourceType, SourceType, TomlSourceType};
 use ruff_source_file::{LineIndex, SourceCode, SourceFileBuilder};
 use ruff_text_size::{TextRange, TextSize};
 
@@ -228,36 +227,36 @@ pub(crate) fn lint_path(
 
     debug!("Checking: {}", path.display());
 
-    // We have to special case this here since the Python tokenizer doesn't work with TOML.
-    if is_project_toml(path) {
-        let messages = if settings
-            .lib
-            .rules
-            .iter_enabled()
-            .any(|rule_code| rule_code.lint_source().is_pyproject_toml())
-        {
-            let contents = match std::fs::read_to_string(path) {
-                Ok(contents) => contents,
-                Err(err) => {
-                    return Ok(Diagnostics::from_io_error(&err, path, &settings.lib));
-                }
+    let source_type = match SourceType::from(path) {
+        SourceType::Toml(TomlSourceType::Pyproject) => {
+            let messages = if settings
+                .lib
+                .rules
+                .iter_enabled()
+                .any(|rule_code| rule_code.lint_source().is_pyproject_toml())
+            {
+                let contents = match std::fs::read_to_string(path) {
+                    Ok(contents) => contents,
+                    Err(err) => {
+                        return Ok(Diagnostics::from_io_error(&err, path, &settings.lib));
+                    }
+                };
+                let source_file = SourceFileBuilder::new(path.to_string_lossy(), contents).finish();
+                lint_pyproject_toml(source_file, &settings.lib)
+            } else {
+                vec![]
             };
-            let source_file = SourceFileBuilder::new(path.to_string_lossy(), contents).finish();
-            lint_pyproject_toml(source_file, &settings.lib)
-        } else {
-            vec![]
-        };
-        return Ok(Diagnostics {
-            messages,
-            ..Diagnostics::default()
-        });
-    }
+            return Ok(Diagnostics {
+                messages,
+                ..Diagnostics::default()
+            });
+        }
+        SourceType::Toml(_) => return Ok(Diagnostics::default()),
+        SourceType::Python(source_type) => source_type,
+    };
 
     // Extract the sources from the file.
-    let LintSources {
-        source_type,
-        source_kind,
-    } = match LintSources::try_from_path(path) {
+    let LintSource(source_kind) = match LintSource::try_from_path(path, source_type) {
         Ok(sources) => sources,
         Err(SourceExtractionError::Io(err)) => {
             return Ok(Diagnostics::from_io_error(&err, path, &settings.lib));
@@ -438,21 +437,24 @@ pub(crate) fn lint_stdin(
     noqa: flags::Noqa,
     autofix: flags::FixMode,
 ) -> Result<Diagnostics> {
-    // Extract the sources from the file.
-    let LintSources {
-        source_type,
-        source_kind,
-    } = match LintSources::try_from_source_code(contents, path) {
-        Ok(sources) => sources,
-        Err(SourceExtractionError::Io(err)) => {
-            // SAFETY: An `io::Error` can only occur if we're reading from a path.
-            return Ok(Diagnostics::from_io_error(&err, path.unwrap(), settings));
-        }
-        Err(SourceExtractionError::Diagnostics(diagnostics)) => {
-            return Ok(*diagnostics);
-        }
+    // TODO(charlie): Support `pyproject.toml`.
+    let SourceType::Python(source_type) = path.map(SourceType::from).unwrap_or_default() else {
+        return Ok(Diagnostics::default());
     };
 
+    // Extract the sources from the file.
+    let LintSource(source_kind) =
+        match LintSource::try_from_source_code(contents, path, source_type) {
+            Ok(sources) => sources,
+            Err(SourceExtractionError::Io(err)) => {
+                // SAFETY: An `io::Error` can only occur if we're reading from a path.
+                return Ok(Diagnostics::from_io_error(&err, path.unwrap(), settings));
+            }
+            Err(SourceExtractionError::Diagnostics(diagnostics)) => {
+                return Ok(*diagnostics);
+            }
+        };
+
     // Lint the inputs.
     let (
         LinterResult {
@@ -554,58 +556,40 @@ pub(crate) fn lint_stdin(
 }
 
 #[derive(Debug)]
-struct LintSources {
-    /// The "type" of source code, e.g. `.py`, `.pyi`, `.ipynb`, etc.
-    source_type: PySourceType,
-    /// The "kind" of source, e.g. Python file, Jupyter Notebook, etc.
-    source_kind: SourceKind,
-}
+struct LintSource(SourceKind);
 
-impl LintSources {
-    /// Extract the lint [`LintSources`] from the given file path.
-    fn try_from_path(path: &Path) -> Result<LintSources, SourceExtractionError> {
-        let source_type = PySourceType::from(path);
-
-        // Read the file from disk.
+impl LintSource {
+    /// Extract the lint [`LintSource`] from the given file path.
+    fn try_from_path(
+        path: &Path,
+        source_type: PySourceType,
+    ) -> Result<LintSource, SourceExtractionError> {
         if source_type.is_ipynb() {
             let notebook = notebook_from_path(path).map_err(SourceExtractionError::Diagnostics)?;
             let source_kind = SourceKind::IpyNotebook(notebook);
-            Ok(LintSources {
-                source_type,
-                source_kind,
-            })
+            Ok(LintSource(source_kind))
         } else {
             // This is tested by ruff_cli integration test `unreadable_file`
             let contents = std::fs::read_to_string(path).map_err(SourceExtractionError::Io)?;
-            Ok(LintSources {
-                source_type,
-                source_kind: SourceKind::Python(contents),
-            })
+            Ok(LintSource(SourceKind::Python(contents)))
         }
     }
 
-    /// Extract the lint [`LintSources`] from the raw string contents, optionally accompanied by a
+    /// Extract the lint [`LintSource`] from the raw string contents, optionally accompanied by a
     /// file path indicating the path to the file from which the contents were read. If provided,
     /// the file path should be used for diagnostics, but not for reading the file from disk.
     fn try_from_source_code(
         source_code: String,
         path: Option<&Path>,
-    ) -> Result<LintSources, SourceExtractionError> {
-        let source_type = path.map(PySourceType::from).unwrap_or_default();
-
+        source_type: PySourceType,
+    ) -> Result<LintSource, SourceExtractionError> {
         if source_type.is_ipynb() {
             let notebook = notebook_from_source_code(&source_code, path)
                 .map_err(SourceExtractionError::Diagnostics)?;
             let source_kind = SourceKind::IpyNotebook(notebook);
-            Ok(LintSources {
-                source_type,
-                source_kind,
-            })
+            Ok(LintSource(source_kind))
         } else {
-            Ok(LintSources {
-                source_type,
-                source_kind: SourceKind::Python(source_code),
-            })
+            Ok(LintSource(SourceKind::Python(source_code)))
         }
     }
 }