[pydocstyle] Improve heuristics for detecting Google-style docstrings (#13142)

crates/ruff_linter/resources/test/fixtures/pydocstyle/sections.py

@@ -605,3 +605,17 @@ def test_lowercase_sub_section_header_different_kind(returns: int):
        some value
 
    """
+
+
+# We used to incorrectly infer this as a numpy-style docstring,
+# which caused us to emit D406 and D407 on it;
+# see https://github.com/astral-sh/ruff/issues/13139
+def another_valid_google_style_docstring(a: str) -> str:
+    """Foo bar.
+
+    Examples:
+        Some explanation here.
+        >>> bla bla bla
+
+    """
+    return a

crates/ruff_linter/src/docstrings/styles.rs

@@ -2,7 +2,7 @@ use crate::docstrings::google::GOOGLE_SECTIONS;
 use crate::docstrings::numpy::NUMPY_SECTIONS;
 use crate::docstrings::sections::SectionKind;
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug, is_macro::Is)]
 pub(crate) enum SectionStyle {
     Numpy,
     Google,

crates/ruff_linter/src/rules/pydocstyle/helpers.rs

@@ -1,6 +1,10 @@
+use std::cmp::Ordering;
+
 use ruff_python_ast::helpers::map_callable;
 use ruff_python_semantic::{Definition, SemanticModel};
-use ruff_source_file::UniversalNewlines;
+use ruff_python_trivia::Cursor;
+use ruff_source_file::{Line, UniversalNewlines};
+use ruff_text_size::{TextRange, TextSize};
 
 use crate::docstrings::sections::{SectionContexts, SectionKind};
 use crate::docstrings::styles::SectionStyle;
@@ -112,12 +116,68 @@ pub(crate) fn get_section_contexts<'a>(
                 return google_sections;
             }
 
-            // Otherwise, use whichever convention matched more sections.
+            // Otherwise, If one convention matched more sections, return that...
-            if google_sections.len() > numpy_sections.len() {
+            match google_sections.len().cmp(&numpy_sections.len()) {
-                google_sections
+                Ordering::Greater => return google_sections,
-            } else {
+                Ordering::Less => return numpy_sections,
-                numpy_sections
+                Ordering::Equal => {}
+            };
+
+            // 0 sections of either convention? Default to numpy
+            if google_sections.len() == 0 {
+                return numpy_sections;
             }
+
+            for section in &google_sections {
+                // If any section has something that could be an underline
+                // on the following line, assume Numpy.
+                // If it *doesn't* have an underline and it *does* have a colon
+                // at the end of a section name, assume Google.
+                if let Some(following_line) = section.following_lines().next() {
+                    if find_underline(&following_line, '-').is_some() {
+                        return numpy_sections;
+                    }
+                }
+                if section.summary_after_section_name().starts_with(':') {
+                    return google_sections;
+                }
+            }
+
+            // If all else fails, default to numpy
+            numpy_sections
         }
     }
 }
+
+/// Returns the [`TextRange`] of the underline, if a line consists of only dashes.
+pub(super) fn find_underline(line: &Line, dash: char) -> Option<TextRange> {
+    let mut cursor = Cursor::new(line.as_str());
+
+    // Eat leading whitespace.
+    cursor.eat_while(char::is_whitespace);
+
+    // Determine the start of the dashes.
+    let offset = cursor.token_len();
+
+    // Consume the dashes.
+    cursor.start_token();
+    cursor.eat_while(|c| c == dash);
+
+    // Determine the end of the dashes.
+    let len = cursor.token_len();
+
+    // If there are no dashes, return None.
+    if len == TextSize::new(0) {
+        return None;
+    }
+
+    // Eat trailing whitespace.
+    cursor.eat_while(char::is_whitespace);
+
+    // If there are any characters after the dashes, return None.
+    if !cursor.is_eof() {
+        return None;
+    }
+
+    Some(TextRange::at(offset, len) + line.start())
+}

crates/ruff_linter/src/rules/pydocstyle/rules/sections.rs

@@ -2,7 +2,6 @@ use itertools::Itertools;
 use once_cell::sync::Lazy;
 use regex::Regex;
 use rustc_hash::FxHashSet;
-use std::ops::Add;
 
 use ruff_diagnostics::{AlwaysFixableViolation, Violation};
 use ruff_diagnostics::{Diagnostic, Edit, Fix};
@@ -11,8 +10,8 @@ use ruff_python_ast::docstrings::{clean_space, leading_space};
 use ruff_python_ast::identifier::Identifier;
 use ruff_python_ast::ParameterWithDefault;
 use ruff_python_semantic::analyze::visibility::is_staticmethod;
-use ruff_python_trivia::{textwrap::dedent, Cursor};
+use ruff_python_trivia::textwrap::dedent;
-use ruff_source_file::{Line, NewlineWithTrailingNewline};
+use ruff_source_file::NewlineWithTrailingNewline;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 
 use crate::checkers::ast::Checker;
@@ -20,6 +19,7 @@ use crate::docstrings::sections::{SectionContext, SectionContexts, SectionKind};
 use crate::docstrings::styles::SectionStyle;
 use crate::docstrings::Docstring;
 use crate::registry::Rule;
+use crate::rules::pydocstyle::helpers::find_underline;
 use crate::rules::pydocstyle::settings::Convention;
 
 /// ## What it does
@@ -1341,6 +1341,7 @@ fn blanks_and_section_underline(
     checker: &mut Checker,
     docstring: &Docstring,
     context: &SectionContext,
+    style: SectionStyle,
 ) {
     let mut num_blank_lines_after_header = 0u32;
     let mut blank_lines_end = context.following_range().start();
@@ -1510,7 +1511,7 @@ fn blanks_and_section_underline(
                 }
             }
         } else {
-            if checker.enabled(Rule::DashedUnderlineAfterSection) {
+            if style.is_numpy() && checker.enabled(Rule::DashedUnderlineAfterSection) {
                 if let Some(equal_line) = find_underline(&non_blank_line, '=') {
                     let mut diagnostic = Diagnostic::new(
                         DashedUnderlineAfterSection {
@@ -1608,7 +1609,7 @@ fn blanks_and_section_underline(
         }
     } else {
         // Nothing but blank lines after the section header.
-        if checker.enabled(Rule::DashedUnderlineAfterSection) {
+        if style.is_numpy() && checker.enabled(Rule::DashedUnderlineAfterSection) {
             let mut diagnostic = Diagnostic::new(
                 DashedUnderlineAfterSection {
                     name: context.section_name().to_string(),
@@ -1646,6 +1647,7 @@ fn common_section(
     docstring: &Docstring,
     context: &SectionContext,
     next: Option<&SectionContext>,
+    style: SectionStyle,
 ) {
     if checker.enabled(Rule::CapitalizeSectionName) {
         let capitalized_section_name = context.kind().as_str();
@@ -1776,7 +1778,7 @@ fn common_section(
         }
     }
 
-    blanks_and_section_underline(checker, docstring, context);
+    blanks_and_section_underline(checker, docstring, context, style);
 }
 
 fn missing_args(checker: &mut Checker, docstring: &Docstring, docstrings_args: &FxHashSet<String>) {
@@ -1946,7 +1948,7 @@ fn numpy_section(
     context: &SectionContext,
     next: Option<&SectionContext>,
 ) {
-    common_section(checker, docstring, context, next);
+    common_section(checker, docstring, context, next, SectionStyle::Numpy);
 
     if checker.enabled(Rule::NewLineAfterSectionName) {
         let suffix = context.summary_after_section_name();
@@ -1981,7 +1983,7 @@ fn google_section(
     context: &SectionContext,
     next: Option<&SectionContext>,
 ) {
-    common_section(checker, docstring, context, next);
+    common_section(checker, docstring, context, next, SectionStyle::Google);
 
     if checker.enabled(Rule::SectionNameEndsInColon) {
         let suffix = context.summary_after_section_name();
@@ -2049,36 +2051,3 @@ fn parse_google_sections(
         }
     }
 }
-
-/// Returns the [`TextRange`] of the underline, if a line consists of only dashes.
-fn find_underline(line: &Line, dash: char) -> Option<TextRange> {
-    let mut cursor = Cursor::new(line.as_str());
-
-    // Eat leading whitespace.
-    cursor.eat_while(char::is_whitespace);
-
-    // Determine the start of the dashes.
-    let offset = cursor.token_len();
-
-    // Consume the dashes.
-    cursor.start_token();
-    cursor.eat_while(|c| c == dash);
-
-    // Determine the end of the dashes.
-    let len = cursor.token_len();
-
-    // If there are no dashes, return None.
-    if len == TextSize::new(0) {
-        return None;
-    }
-
-    // Eat trailing whitespace.
-    cursor.eat_while(char::is_whitespace);
-
-    // If there are any characters after the dashes, return None.
-    if !cursor.is_eof() {
-        return None;
-    }
-
-    Some(TextRange::at(offset, len).add(line.start()))
-}

crates/ruff_linter/src/rules/pydocstyle/snapshots/ruff_linter__rules__pydocstyle__tests__D407_sections.py.snap

@@ -79,258 +79,6 @@ sections.py:227:5: D407 [*] Missing dashed underline after section ("Raises")
 229 230 | 
 230 231 |     """
 
-sections.py:263:5: D407 [*] Missing dashed underline after section ("Args")
-    |
-261 |     """Toggle the gizmo.
-262 | 
-263 |     Args:
-    |     ^^^^ D407
-264 |         note: A random string.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-261 261 |     """Toggle the gizmo.
-262 262 | 
-263 263 |     Args:
-    264 |+    ----
-264 265 |         note: A random string.
-265 266 | 
-266 267 |     Returns:
-
-sections.py:266:5: D407 [*] Missing dashed underline after section ("Returns")
-    |
-264 |         note: A random string.
-265 | 
-266 |     Returns:
-    |     ^^^^^^^ D407
-267 | 
-268 |     Raises:
-    |
-    = help: Add dashed line under "Returns"
-
-ℹ Safe fix
-264 264 |         note: A random string.
-265 265 | 
-266 266 |     Returns:
-    267 |+    -------
-267 268 | 
-268 269 |     Raises:
-269 270 |         RandomError: A random error that occurs randomly.
-
-sections.py:268:5: D407 [*] Missing dashed underline after section ("Raises")
-    |
-266 |     Returns:
-267 | 
-268 |     Raises:
-    |     ^^^^^^ D407
-269 |         RandomError: A random error that occurs randomly.
-    |
-    = help: Add dashed line under "Raises"
-
-ℹ Safe fix
-266 266 |     Returns:
-267 267 | 
-268 268 |     Raises:
-    269 |+    ------
-269 270 |         RandomError: A random error that occurs randomly.
-270 271 | 
-271 272 |     """
-
-sections.py:280:5: D407 [*] Missing dashed underline after section ("Args")
-    |
-278 |     """Toggle the gizmo.
-279 | 
-280 |     Args
-    |     ^^^^ D407
-281 |         note: A random string.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-278 278 |     """Toggle the gizmo.
-279 279 | 
-280 280 |     Args
-    281 |+    ----
-281 282 |         note: A random string.
-282 283 | 
-283 284 |     """
-
-sections.py:297:9: D407 [*] Missing dashed underline after section ("Args")
-    |
-295 |         Will this work when referencing x?
-296 | 
-297 |         Args:
-    |         ^^^^ D407
-298 |             x: Test something
-299 | that is broken.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-295 295 |         Will this work when referencing x?
-296 296 | 
-297 297 |         Args:
-    298 |+        ----
-298 299 |             x: Test something
-299 300 | that is broken.
-300 301 | 
-
-sections.py:312:5: D407 [*] Missing dashed underline after section ("Args")
-    |
-310 |     """Toggle the gizmo.
-311 | 
-312 |     Args:
-    |     ^^^^ D407
-313 |         x (int): The greatest integer.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-310 310 |     """Toggle the gizmo.
-311 311 | 
-312 312 |     Args:
-    313 |+    ----
-313 314 |         x (int): The greatest integer.
-314 315 | 
-315 316 |     """
-
-sections.py:324:9: D407 [*] Missing dashed underline after section ("Args")
-    |
-322 |         """Test a valid args section.
-323 | 
-324 |         Args:
-    |         ^^^^ D407
-325 |             test: A parameter.
-326 |             another_test: Another parameter.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-322 322 |         """Test a valid args section.
-323 323 | 
-324 324 |         Args:
-    325 |+        ----
-325 326 |             test: A parameter.
-326 327 |             another_test: Another parameter.
-327 328 | 
-
-sections.py:336:9: D407 [*] Missing dashed underline after section ("Args")
-    |
-334 |         """Test a valid args section.
-335 | 
-336 |         Args:
-    |         ^^^^ D407
-337 |             x: Another parameter.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-334 334 |         """Test a valid args section.
-335 335 | 
-336 336 |         Args:
-    337 |+        ----
-337 338 |             x: Another parameter.
-338 339 | 
-339 340 |         """
-
-sections.py:348:9: D407 [*] Missing dashed underline after section ("Args")
-    |
-346 |         """Test a valid args section.
-347 | 
-348 |         Args:
-    |         ^^^^ D407
-349 |             x: Another parameter. The parameter below is missing description.
-350 |             y:
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-346 346 |         """Test a valid args section.
-347 347 | 
-348 348 |         Args:
-    349 |+        ----
-349 350 |             x: Another parameter. The parameter below is missing description.
-350 351 |             y:
-351 352 | 
-
-sections.py:361:9: D407 [*] Missing dashed underline after section ("Args")
-    |
-359 |         """Test a valid args section.
-360 | 
-361 |         Args:
-    |         ^^^^ D407
-362 |             x: Another parameter.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-359 359 |         """Test a valid args section.
-360 360 | 
-361 361 |         Args:
-    362 |+        ----
-362 363 |             x: Another parameter.
-363 364 | 
-364 365 |         """
-
-sections.py:373:9: D407 [*] Missing dashed underline after section ("Args")
-    |
-371 |         """Test a valid args section.
-372 | 
-373 |         Args:
-    |         ^^^^ D407
-374 |             a:
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-371 371 |         """Test a valid args section.
-372 372 | 
-373 373 |         Args:
-    374 |+        ----
-374 375 |             a:
-375 376 | 
-376 377 |         """
-
-sections.py:382:9: D407 [*] Missing dashed underline after section ("Args")
-    |
-380 |         """Do stuff.
-381 | 
-382 |         Args:
-    |         ^^^^ D407
-383 |             skip (:attr:`.Skip`):
-384 |                 Lorem ipsum dolor sit amet, consectetur adipiscing elit.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-380 380 |         """Do stuff.
-381 381 | 
-382 382 |         Args:
-    383 |+        ----
-383 384 |             skip (:attr:`.Skip`):
-384 385 |                 Lorem ipsum dolor sit amet, consectetur adipiscing elit.
-385 386 |                 Etiam at tellus a tellus faucibus maximus. Curabitur tellus
-
-sections.py:503:9: D407 [*] Missing dashed underline after section ("Args")
-    |
-501 | Testing this incorrectly indented docstring.
-502 | 
-503 |         Args:
-    |         ^^^^ D407
-504 |             x: Test argument.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-501 501 | Testing this incorrectly indented docstring.
-502 502 | 
-503 503 |         Args:
-    504 |+        ----
-504 505 |             x: Test argument.
-505 506 | 
-506 507 |         """
-
 sections.py:522:5: D407 [*] Missing dashed underline after section ("Parameters")
     |
 521 |     Parameters
@@ -369,63 +117,6 @@ sections.py:530:5: D407 [*] Missing dashed underline after section ("Parameters"
 532 532 | 
 533 533 | 
 
-sections.py:550:5: D407 [*] Missing dashed underline after section ("Args")
-    |
-548 |     """Below, `returns:` should _not_ be considered a section header.
-549 | 
-550 |     Args:
-    |     ^^^^ D407
-551 |         Here's a note.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-548 548 |     """Below, `returns:` should _not_ be considered a section header.
-549 549 | 
-550 550 |     Args:
-    551 |+    ----
-551 552 |         Here's a note.
-552 553 | 
-553 554 |         returns:
-
-sections.py:560:5: D407 [*] Missing dashed underline after section ("Args")
-    |
-558 |     """Below, `Returns:` should be considered a section header.
-559 | 
-560 |     Args:
-    |     ^^^^ D407
-561 |         Here's a note.
-    |
-    = help: Add dashed line under "Args"
-
-ℹ Safe fix
-558 558 |     """Below, `Returns:` should be considered a section header.
-559 559 | 
-560 560 |     Args:
-    561 |+    ----
-561 562 |         Here's a note.
-562 563 | 
-563 564 |         Returns:
-
-sections.py:563:9: D407 [*] Missing dashed underline after section ("Returns")
-    |
-561 |         Here's a note.
-562 | 
-563 |         Returns:
-    |         ^^^^^^^ D407
-564 |     """
-    |
-    = help: Add dashed line under "Returns"
-
-ℹ Safe fix
-561 561 |         Here's a note.
-562 562 | 
-563 563 |         Returns:
-    564 |+    -------
-564 565 |     """
-565 566 | 
-566 567 | 
-
 sections.py:602:4: D407 [*] Missing dashed underline after section ("Parameters")
     |
 600 |    """Test that lower case subsection header is valid even if it is of a different kind.