Merge #4683

4683: Implement syntax highlighting for doctests r=ltentrup a=ltentrup

The implementation is more complicated than the previous injection logic as the doctest comments consist of multiple ranges. The implementation extracts the doctests together with an offset-mapping, applies the syntax highlighting, and updates the text ranges.

<img width="478" alt="Bildschirmfoto 2020-06-01 um 15 45 25" src="https://user-images.githubusercontent.com/201808/83415249-1f0b5800-a41f-11ea-8fa6-c282434d6ff7.png">

Part of #4170.

Co-authored-by: Leander Tentrup <leander.tentrup@gmail.com>

crates/ra_ide/src/snapshots/highlight_doctest.html

@@ -0,0 +1,70 @@
+
+<style>
+body                { margin: 0; }
+pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padding: 0.4em; }
+
+.lifetime           { color: #DFAF8F; font-style: italic; }
+.comment            { color: #7F9F7F; }
+.struct, .enum      { color: #7CB8BB; }
+.enum_variant       { color: #BDE0F3; }
+.string_literal     { color: #CC9393; }
+.field              { color: #94BFF3; }
+.function           { color: #93E0E3; }
+.operator.unsafe    { color: #E28C14; }
+.parameter          { color: #94BFF3; }
+.text               { color: #DCDCCC; }
+.type               { color: #7CB8BB; }
+.builtin_type       { color: #8CD0D3; }
+.type_param         { color: #DFAF8F; }
+.attribute          { color: #94BFF3; }
+.numeric_literal    { color: #BFEBBF; }
+.bool_literal       { color: #BFE6EB; }
+.macro              { color: #94BFF3; }
+.module             { color: #AFD8AF; }
+.variable           { color: #DCDCCC; }
+.format_specifier   { color: #CC696B; }
+.mutable            { text-decoration: underline; }
+
+.keyword            { color: #F0DFAF; font-weight: bold; }
+.keyword.unsafe     { color: #BC8383; font-weight: bold; }
+.control            { font-style: italic; }
+</style>
+<pre><code><span class="keyword">impl</span> <span class="unresolved_reference">Foo</span> {
+    <span class="comment">/// Constructs a new `Foo`.</span>
+    <span class="comment">///</span>
+    <span class="comment">/// # Examples</span>
+    <span class="comment">///</span>
+    <span class="comment">/// ```</span>
+    <span class="comment">/// #</span> <span class="attribute">#![</span><span class="function attribute">allow</span><span class="attribute">(unused_mut)]</span>
+    <span class="comment">/// </span><span class="keyword">let</span> <span class="keyword">mut</span> <span class="variable declaration mutable">foo</span>: <span class="unresolved_reference">Foo</span> = <span class="unresolved_reference">Foo</span>::<span class="unresolved_reference">new</span>();
+    <span class="comment">/// ```</span>
+    <span class="keyword">pub</span> <span class="keyword">const</span> <span class="keyword">fn</span> <span class="function declaration">new</span>() -&gt; <span class="unresolved_reference">Foo</span> {
+        <span class="unresolved_reference">Foo</span> { }
+    }
+
+    <span class="comment">/// `bar` method on `Foo`.</span>
+    <span class="comment">///</span>
+    <span class="comment">/// # Examples</span>
+    <span class="comment">///</span>
+    <span class="comment">/// ```</span>
+    <span class="comment">/// </span><span class="keyword">let</span> <span class="variable declaration">foo</span> = <span class="unresolved_reference">Foo</span>::<span class="unresolved_reference">new</span>();
+    <span class="comment">///</span>
+    <span class="comment">/// </span><span class="comment">// calls bar on foo</span>
+    <span class="comment">/// </span><span class="macro">assert!</span>(foo.bar());
+    <span class="comment">///</span>
+    <span class="comment">/// </span><span class="comment">/* multi-line
+    </span><span class="comment">/// </span><span class="comment">       comment */</span>
+    <span class="comment">///</span>
+    <span class="comment">/// </span><span class="keyword">let</span> <span class="variable declaration">multi_line_string</span> = <span class="string_literal">"Foo
+    </span><span class="comment">/// </span><span class="string_literal">  bar
+    </span><span class="comment">/// </span><span class="string_literal">         "</span>;
+    <span class="comment">///</span>
+    <span class="comment">/// ```</span>
+    <span class="comment">///</span>
+    <span class="comment">/// ```</span>
+    <span class="comment">/// </span><span class="keyword">let</span> <span class="variable declaration">foobar</span> = <span class="unresolved_reference">Foo</span>::<span class="unresolved_reference">new</span>().<span class="unresolved_reference">bar</span>();
+    <span class="comment">/// ```</span>
+    <span class="keyword">pub</span> <span class="keyword">fn</span> <span class="function declaration">foo</span>(&<span class="self_keyword">self</span>) -&gt; <span class="builtin_type">bool</span> {
+        <span class="bool_literal">true</span>
+    }
+}</code></pre>

crates/ra_ide/src/syntax_highlighting.rs

@@ -1,5 +1,6 @@
 mod tags;
 mod html;
+mod injection;
 #[cfg(test)]
 mod tests;
 
@@ -10,14 +11,14 @@ use ra_ide_db::{
 };
 use ra_prof::profile;
 use ra_syntax::{
-    ast::{self, HasFormatSpecifier, HasQuotes, HasStringValue},
+    ast::{self, HasFormatSpecifier},
     AstNode, AstToken, Direction, NodeOrToken, SyntaxElement,
     SyntaxKind::*,
-    SyntaxToken, TextRange, WalkEvent, T,
+    TextRange, WalkEvent, T,
 };
 use rustc_hash::FxHashMap;
 
-use crate::{call_info::ActiveParameter, Analysis, FileId};
+use crate::FileId;
 
 use ast::FormatSpecifier;
 pub(crate) use html::highlight_as_html;
@@ -123,6 +124,23 @@ pub(crate) fn highlight(
             _ => (),
         }
 
+        // Check for Rust code in documentation
+        match &event {
+            WalkEvent::Leave(NodeOrToken::Node(node)) => {
+                if let Some((doctest, range_mapping, new_comments)) =
+                    injection::extract_doc_comments(node)
+                {
+                    injection::highlight_doc_comment(
+                        doctest,
+                        range_mapping,
+                        new_comments,
+                        &mut stack,
+                    );
+                }
+            }
+            _ => (),
+        }
+
         let element = match event {
             WalkEvent::Enter(it) => it,
             WalkEvent::Leave(_) => continue,
@@ -173,7 +191,7 @@ pub(crate) fn highlight(
 
         if let Some(token) = element.as_token().cloned().and_then(ast::RawString::cast) {
             let expanded = element_to_highlight.as_token().unwrap().clone();
-            if highlight_injection(&mut stack, &sema, token, expanded).is_some() {
+            if injection::highlight_injection(&mut stack, &sema, token, expanded).is_some() {
                 continue;
             }
         }
@@ -259,9 +277,8 @@ impl HighlightedRangeStack {
             let mut parent = prev.pop().unwrap();
             for ele in children {
                 assert!(parent.range.contains_range(ele.range));
-                let mut cloned = parent.clone();
+
-                parent.range = TextRange::new(parent.range.start(), ele.range.start());
+                let cloned = Self::intersect(&mut parent, &ele);
-                cloned.range = TextRange::new(ele.range.end(), cloned.range.end());
                 if !parent.range.is_empty() {
                     prev.push(parent);
                 }
@@ -274,6 +291,62 @@ impl HighlightedRangeStack {
         }
     }
 
+    /// Intersects the `HighlightedRange` `parent` with `child`.
+    /// `parent` is mutated in place, becoming the range before `child`.
+    /// Returns the range (of the same type as `parent`) *after* `child`.
+    fn intersect(parent: &mut HighlightedRange, child: &HighlightedRange) -> HighlightedRange {
+        assert!(parent.range.contains_range(child.range));
+
+        let mut cloned = parent.clone();
+        parent.range = TextRange::new(parent.range.start(), child.range.start());
+        cloned.range = TextRange::new(child.range.end(), cloned.range.end());
+
+        cloned
+    }
+
+    /// Similar to `pop`, but can modify arbitrary prior ranges (where `pop`)
+    /// can only modify the last range currently on the stack.
+    /// Can be used to do injections that span multiple ranges, like the
+    /// doctest injection below.
+    /// If `delete` is set to true, the parent range is deleted instead of
+    /// intersected.
+    ///
+    /// Note that `pop` can be simulated by `pop_and_inject(false)` but the
+    /// latter is computationally more expensive.
+    fn pop_and_inject(&mut self, delete: bool) {
+        let mut children = self.stack.pop().unwrap();
+        let prev = self.stack.last_mut().unwrap();
+        children.sort_by_key(|range| range.range.start());
+        prev.sort_by_key(|range| range.range.start());
+
+        for child in children {
+            if let Some(idx) =
+                prev.iter().position(|parent| parent.range.contains_range(child.range))
+            {
+                let cloned = Self::intersect(&mut prev[idx], &child);
+                let insert_idx = if delete || prev[idx].range.is_empty() {
+                    prev.remove(idx);
+                    idx
+                } else {
+                    idx + 1
+                };
+                prev.insert(insert_idx, child);
+                if !delete && !cloned.range.is_empty() {
+                    prev.insert(insert_idx + 1, cloned);
+                }
+            } else if let Some(_idx) =
+                prev.iter().position(|parent| parent.range.contains(child.range.start()))
+            {
+                unreachable!("child range should be completely contained in parent range");
+            } else {
+                let idx = prev
+                    .binary_search_by_key(&child.range.start(), |range| range.range.start())
+                    .unwrap_or_else(|x| x);
+                prev.insert(idx, child);
+            }
+        }
+    }
+
     fn add(&mut self, range: HighlightedRange) {
         self.stack
             .last_mut()
@@ -539,42 +612,3 @@ fn highlight_name_by_syntax(name: ast::Name) -> Highlight {
 
     tag.into()
 }
-
-fn highlight_injection(
-    acc: &mut HighlightedRangeStack,
-    sema: &Semantics<RootDatabase>,
-    literal: ast::RawString,
-    expanded: SyntaxToken,
-) -> Option<()> {
-    let active_parameter = ActiveParameter::at_token(&sema, expanded)?;
-    if !active_parameter.name.starts_with("ra_fixture") {
-        return None;
-    }
-    let value = literal.value()?;
-    let (analysis, tmp_file_id) = Analysis::from_single_file(value);
-
-    if let Some(range) = literal.open_quote_text_range() {
-        acc.add(HighlightedRange {
-            range,
-            highlight: HighlightTag::StringLiteral.into(),
-            binding_hash: None,
-        })
-    }
-
-    for mut h in analysis.highlight(tmp_file_id).unwrap() {
-        if let Some(r) = literal.map_range_up(h.range) {
-            h.range = r;
-            acc.add(h)
-        }
-    }
-
-    if let Some(range) = literal.close_quote_text_range() {
-        acc.add(HighlightedRange {
-            range,
-            highlight: HighlightTag::StringLiteral.into(),
-            binding_hash: None,
-        })
-    }
-
-    Some(())
-}

crates/ra_ide/src/syntax_highlighting/injection.rs

@@ -0,0 +1,168 @@
+//! Syntax highlighting injections such as highlighting of documentation tests.
+
+use std::{collections::BTreeMap, convert::TryFrom};
+
+use ast::{HasQuotes, HasStringValue};
+use hir::Semantics;
+use ra_syntax::{ast, AstToken, SyntaxNode, SyntaxToken, TextRange, TextSize};
+use stdx::SepBy;
+
+use crate::{call_info::ActiveParameter, Analysis, HighlightTag, HighlightedRange, RootDatabase};
+
+use super::HighlightedRangeStack;
+
+pub(super) fn highlight_injection(
+    acc: &mut HighlightedRangeStack,
+    sema: &Semantics<RootDatabase>,
+    literal: ast::RawString,
+    expanded: SyntaxToken,
+) -> Option<()> {
+    let active_parameter = ActiveParameter::at_token(&sema, expanded)?;
+    if !active_parameter.name.starts_with("ra_fixture") {
+        return None;
+    }
+    let value = literal.value()?;
+    let (analysis, tmp_file_id) = Analysis::from_single_file(value);
+
+    if let Some(range) = literal.open_quote_text_range() {
+        acc.add(HighlightedRange {
+            range,
+            highlight: HighlightTag::StringLiteral.into(),
+            binding_hash: None,
+        })
+    }
+
+    for mut h in analysis.highlight(tmp_file_id).unwrap() {
+        if let Some(r) = literal.map_range_up(h.range) {
+            h.range = r;
+            acc.add(h)
+        }
+    }
+
+    if let Some(range) = literal.close_quote_text_range() {
+        acc.add(HighlightedRange {
+            range,
+            highlight: HighlightTag::StringLiteral.into(),
+            binding_hash: None,
+        })
+    }
+
+    Some(())
+}
+
+/// Mapping from extracted documentation code to original code
+type RangesMap = BTreeMap<TextSize, TextSize>;
+
+/// Extracts Rust code from documentation comments as well as a mapping from
+/// the extracted source code back to the original source ranges.
+/// Lastly, a vector of new comment highlight ranges (spanning only the
+/// comment prefix) is returned which is used in the syntax highlighting
+/// injection to replace the previous (line-spanning) comment ranges.
+pub(super) fn extract_doc_comments(
+    node: &SyntaxNode,
+) -> Option<(String, RangesMap, Vec<HighlightedRange>)> {
+    // wrap the doctest into function body to get correct syntax highlighting
+    let prefix = "fn doctest() {\n";
+    let suffix = "}\n";
+    // Mapping from extracted documentation code to original code
+    let mut range_mapping: RangesMap = BTreeMap::new();
+    let mut line_start = TextSize::try_from(prefix.len()).unwrap();
+    let mut is_doctest = false;
+    // Replace the original, line-spanning comment ranges by new, only comment-prefix
+    // spanning comment ranges.
+    let mut new_comments = Vec::new();
+    let doctest = node
+        .children_with_tokens()
+        .filter_map(|el| el.into_token().and_then(ast::Comment::cast))
+        .filter(|comment| comment.kind().doc.is_some())
+        .filter(|comment| {
+            if comment.text().contains("```") {
+                is_doctest = !is_doctest;
+                false
+            } else {
+                is_doctest
+            }
+        })
+        .map(|comment| {
+            let prefix_len = comment.prefix().len();
+            let line: &str = comment.text().as_str();
+            let range = comment.syntax().text_range();
+
+            // whitespace after comment is ignored
+            let pos = if let Some(ws) = line.chars().nth(prefix_len).filter(|c| c.is_whitespace()) {
+                prefix_len + ws.len_utf8()
+            } else {
+                prefix_len
+            };
+
+            // lines marked with `#` should be ignored in output, we skip the `#` char
+            let pos = if let Some(ws) = line.chars().nth(pos).filter(|&c| c == '#') {
+                pos + ws.len_utf8()
+            } else {
+                pos
+            };
+
+            range_mapping.insert(line_start, range.start() + TextSize::try_from(pos).unwrap());
+            new_comments.push(HighlightedRange {
+                range: TextRange::new(
+                    range.start(),
+                    range.start() + TextSize::try_from(pos).unwrap(),
+                ),
+                highlight: HighlightTag::Comment.into(),
+                binding_hash: None,
+            });
+            line_start += range.len() - TextSize::try_from(pos).unwrap();
+            line_start += TextSize::try_from('\n'.len_utf8()).unwrap();
+
+            line[pos..].to_owned()
+        })
+        .sep_by("\n")
+        .to_string();
+
+    if doctest.is_empty() {
+        return None;
+    }
+
+    let doctest = format!("{}{}{}", prefix, doctest, suffix);
+    Some((doctest, range_mapping, new_comments))
+}
+
+/// Injection of syntax highlighting of doctests.
+pub(super) fn highlight_doc_comment(
+    text: String,
+    range_mapping: RangesMap,
+    new_comments: Vec<HighlightedRange>,
+    stack: &mut HighlightedRangeStack,
+) {
+    let (analysis, tmp_file_id) = Analysis::from_single_file(text);
+
+    stack.push();
+    for mut h in analysis.highlight(tmp_file_id).unwrap() {
+        // Determine start offset and end offset in case of multi-line ranges
+        let mut start_offset = None;
+        let mut end_offset = None;
+        for (line_start, orig_line_start) in range_mapping.range(..h.range.end()).rev() {
+            if line_start <= &h.range.start() {
+                start_offset.get_or_insert(orig_line_start - line_start);
+                break;
+            } else {
+                end_offset.get_or_insert(orig_line_start - line_start);
+            }
+        }
+        if let Some(start_offset) = start_offset {
+            h.range = TextRange::new(
+                h.range.start() + start_offset,
+                h.range.end() + end_offset.unwrap_or(start_offset),
+            );
+            stack.add(h);
+        }
+    }
+
+    // Inject the comment prefix highlight ranges
+    stack.push();
+    for comment in new_comments {
+        stack.add(comment);
+    }
+    stack.pop_and_inject(false);
+    stack.pop_and_inject(true);
+}

crates/ra_ide/src/syntax_highlighting/tests.rs

@@ -284,3 +284,53 @@ fn main() {
         false,
     );
 }
+
+#[test]
+fn test_highlight_doctest() {
+    check_highlighting(
+        r#"
+impl Foo {
+    /// Constructs a new `Foo`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #![allow(unused_mut)]
+    /// let mut foo: Foo = Foo::new();
+    /// ```
+    pub const fn new() -> Foo {
+        Foo { }
+    }
+
+    /// `bar` method on `Foo`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let foo = Foo::new();
+    ///
+    /// // calls bar on foo
+    /// assert!(foo.bar());
+    ///
+    /// /* multi-line
+    ///        comment */
+    ///
+    /// let multi_line_string = "Foo
+    ///   bar
+    ///          ";
+    ///
+    /// ```
+    ///
+    /// ```
+    /// let foobar = Foo::new().bar();
+    /// ```
+    pub fn foo(&self) -> bool {
+        true
+    }
+}
+"#
+        .trim(),
+        "crates/ra_ide/src/snapshots/highlight_doctest.html",
+        false,
+    )
+}