cleanup

Cargo.lock

@@ -3734,7 +3734,6 @@ dependencies = [
  "criterion 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "encode_unicode",
  "indoc",
- "peg",
  "pretty_assertions",
  "quickcheck",
  "quickcheck_macros",

LEGAL_DETAILS

@@ -203,7 +203,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 * Zig - https://ziglang.org
 
-This source code can be found in compiler/builtins/bitcode/src/hash.zig and compiler/parse/tests/peg_grammar.rs and is licensed under the following terms:
+This source code can be found in compiler/builtins/bitcode/src/hash.zig, highlight/tests/peg_grammar.rs and highlight/src/highlight_parser.rs and is licensed under the following terms:
 
 
 The MIT License (Expat)

code_markup/src/syntax_highlight.rs

@@ -20,9 +20,9 @@ pub enum HighlightStyle {
     Blank,
     Comment,
     DocsComment,
-    UppercaseIdent, // TODO remove other HighlightStyle subtypes of UppercaseIdent?
-    LowercaseIdent, // TODO remove other HighlightStyle subtypes of LowercaseIdent?
-    Keyword,        // if, else, when
+    UppercaseIdent,
+    LowercaseIdent, // TODO we probably don't want all lowercase identifiers to have the same color?
+    Keyword,        // if, else, when...
 }
 
 pub fn default_highlight_map() -> HashMap<HighlightStyle, RgbaTup> {

compiler/parse/Cargo.toml

@@ -22,7 +22,6 @@ indoc = "1.0.3"
 quickcheck = "1.0.3"
 quickcheck_macros = "1.0.0"
 roc_test_utils = { path = "../../test_utils" }
-peg = "0.8.0"
 
 [[bench]]
 name = "bench_parse"

docs/tests/insert_syntax_highlighting.rs

@@ -1,5 +1,6 @@
 #[macro_use]
 extern crate pretty_assertions;
+// Keep this around until the commented out tests can be enabled again.
 /*#[macro_use]
 extern crate indoc;*/
 
@@ -43,6 +44,8 @@ mod insert_doc_syntax_highlighting {
         expect_html_expr("2", r#"<span class="syntax-number">2</span>"#);
     }
 
+    // These tests have been commented out due to introduction of a new syntax highlighting approach.
+    // You can make these tests work by following the instructions at the top of this file here: roc/highlight/src/highlight_parser.rs
     /*#[test]
     fn string_expr() {
         expect_html_expr(r#""abc""#, r#"<span class="syntax-string">"abc"</span>"#);

highlight/src/highlight_parser.rs

@@ -241,38 +241,4 @@ pub mod highlight_tests {
 
         assert_eq!(&str_buffer, "a = 0\n\n");
     }
-
-    /*#[test]
-      fn test_highlight_defs() {
-        let mut mark_node_pool = SlowPool::default();
-
-        let res =
-          highlight_defs(
-            r#"0
-    1"#,
-            &mut mark_node_pool
-          );
-
-        assert!(
-          all_highlight_style(res, HighlightStyle::Number, 2, &mark_node_pool)
-        );
-      }
-
-      fn all_highlight_style(parse_res: Result<Vec<MarkNodeId>, ParseError<usize>>, highlight_style: HighlightStyle, expected_len: usize, mark_node_pool: &SlowPool) -> bool {
-        let node_vec = parse_res
-          .unwrap();
-
-        assert_eq!(node_vec.len(), expected_len);
-
-        node_vec
-        .iter()
-        .all(|m_node| has_highlight_style(mark_node_pool.get(*m_node), highlight_style))
-      }
-
-      fn has_highlight_style(mark_node: &MarkupNode, highlight_style: HighlightStyle) -> bool {
-        match *mark_node {
-          MarkupNode::Text { syn_high_style, .. } => syn_high_style == highlight_style,
-          _ => false,
-        }
-      }*/
 }

highlight/src/tokenizer.rs

@@ -216,9 +216,9 @@ fn add_indents(
                     consumer.token(Token::SameIndent, *curr_byte_ctr, 0);
                 }
                 Ordering::Greater => {
-                    // safe unwrap because we check first
                     while state.indents.last().is_some()
                         && curr_line_indent < *state.indents.last().unwrap()
+                    // safe unwrap because we check first
                     {
                         state.indents.pop();
                         consumer.token(Token::CloseIndent, *curr_byte_ctr, 0);
@@ -237,9 +237,22 @@ fn add_indents(
 
 impl TokenTable {
     pub fn extract_str<'a>(&self, index: usize, content: &'a str) -> &'a str {
-        // TODO remove unwrap
-        let len = *self.lengths.get(index).unwrap();
-        let offset = *self.offsets.get(index).unwrap();
+        // Not returning a result here because weaving it through highlight_parser makes it more difficult to expand and understand.
+        // The only way I think this can panic is by calling position! in highlight_parser after the last element, which does not make sense to begin with.
+        let len = *self.lengths.get(index).unwrap_or_else(|| {
+            panic!(
+                "Index {:?} was out of bounds for TokenTable.lengths with len {:?}",
+                index,
+                self.lengths.len()
+            )
+        });
+        let offset = *self.offsets.get(index).unwrap_or_else(|| {
+            panic!(
+                "Index {:?} was out of bounds for TokenTable.offsets with len {:?}",
+                index,
+                self.lengths.len()
+            )
+        });
 
         &content[offset..(offset + len)]
     }

utils/src/lib.rs

@@ -33,7 +33,7 @@ pub fn index_of<T: ::std::fmt::Debug + std::cmp::Eq>(elt: T, slice: &[T]) -> Uti
     Ok(index)
 }
 
-// replace slice method that return Option with one that return Result and proper Error
+// replaces slice method that return Option with one that return Result and proper Error
 pub fn slice_get<T>(index: usize, slice: &[T]) -> UtilResult<&<usize as SliceIndex<[T]>>::Output> {
     let elt_ref = slice.get(index).context(OutOfBounds {
         index,