Include unicode codepoint in unknown start of token error (#2637)

src/compile_error.rs

@@ -291,7 +291,13 @@ impl Display for CompileError<'_> {
       }
       UnknownFunction { function } => write!(f, "Call to unknown function `{function}`"),
       UnknownSetting { setting } => write!(f, "Unknown setting `{setting}`"),
-      UnknownStartOfToken => write!(f, "Unknown start of token:"),
+      UnknownStartOfToken { start } => {
+        write!(f, "Unknown start of token '{start}'")?;
+        if !start.is_ascii_graphic() {
+          write!(f, " (U+{:04X})", *start as u32)?;
+        }
+        Ok(())
+      }
       UnpairedCarriageReturn => write!(f, "Unpaired carriage return"),
       UnterminatedBacktick => write!(f, "Unterminated backtick"),
       UnterminatedInterpolation => write!(f, "Unterminated interpolation"),

src/compile_error_kind.rs

@@ -153,7 +153,9 @@ pub(crate) enum CompileErrorKind<'src> {
   UnknownSetting {
     setting: &'src str,
   },
-  UnknownStartOfToken,
+  UnknownStartOfToken {
+    start: char,
+  },
   UnpairedCarriageReturn,
   UnterminatedBacktick,
   UnterminatedInterpolation,

src/lexer.rs

@@ -505,7 +505,7 @@ impl<'src> Lexer<'src> {
       _ if Self::is_identifier_start(start) => self.lex_identifier(),
       _ => {
         self.advance()?;
-        Err(self.error(UnknownStartOfToken))
+        Err(self.error(UnknownStartOfToken { start }))
       }
     }
   }
@@ -2121,7 +2121,7 @@ mod tests {
     line:   0,
     column: 0,
     width:  1,
-    kind:   UnknownStartOfToken,
+    kind:   UnknownStartOfToken { start: '%'},
   }
 
   error! {
@@ -2182,7 +2182,7 @@ mod tests {
     line:   0,
     column: 0,
     width:  1,
-    kind:   UnknownStartOfToken,
+    kind:   UnknownStartOfToken{ start: '-'},
   }
 
   error! {
@@ -2192,7 +2192,7 @@ mod tests {
     line:   0,
     column: 0,
     width:  1,
-    kind:   UnknownStartOfToken,
+    kind:   UnknownStartOfToken { start: '0' },
   }
 
   error! {
@@ -2262,7 +2262,7 @@ mod tests {
     line:   0,
     column: 1,
     width:  1,
-    kind:   UnknownStartOfToken,
+    kind:   UnknownStartOfToken { start: '%'},
   }
 
   error! {

tests/misc.rs

@@ -558,9 +558,11 @@ echo `echo command interpolation`
 ",
 }
 
-test! {
-  name:     line_error_spacing,
-  justfile: r"
+#[test]
+fn line_error_spacing() {
+  Test::new()
+    .justfile(
+      r"
 
 
 
@@ -572,14 +574,17 @@ test! {
 
 ^^^
 ",
-  stdout:   "",
-  stderr:   "error: Unknown start of token:
+    )
+    .stderr(
+      "error: Unknown start of token '^'
   ——▶ justfile:10:1
    │
 10 │ ^^^
    │ ^
 ",
-  status:   EXIT_FAILURE,
+    )
+    .status(EXIT_FAILURE)
+    .run();
 }
 
 test! {
@@ -1664,19 +1669,67 @@ test! {
   status:   EXIT_FAILURE,
 }
 
-test! {
-  name:     unknown_start_of_token,
-  justfile: "
+#[test]
+fn unknown_start_of_token() {
+  Test::new()
+    .justfile(
+      "
 assembly_source_files = %(wildcard src/arch/$(arch)/*.s)
-",
-  stderr:   r"
-    error: Unknown start of token:
+      ",
+    )
+    .stderr(
+      r"
+    error: Unknown start of token '%'
      ——▶ justfile:1:25
       │
     1 │ assembly_source_files = %(wildcard src/arch/$(arch)/*.s)
       │                         ^
   ",
-   status:   EXIT_FAILURE,
+    )
+    .status(EXIT_FAILURE)
+    .run();
+}
+
+#[test]
+fn unknown_start_of_token_invisible_unicode() {
+  Test::new()
+    .justfile(
+      "
+\u{200b}foo := 'bar'
+      ",
+    )
+    .stderr(
+      "
+error: Unknown start of token '\u{200b}' (U+200B)
+ ——▶ justfile:1:1
+  │
+1 │ \u{200b}foo := 'bar'
+  │ ^
+",
+    )
+    .status(EXIT_FAILURE)
+    .run();
+}
+
+#[test]
+fn unknown_start_of_token_ascii_control_char() {
+  Test::new()
+    .justfile(
+      "
+\0foo := 'bar'
+",
+    )
+    .stderr(
+      "
+error: Unknown start of token '\0' (U+0000)
+ ——▶ justfile:1:1
+  │
+1 │ \0foo := 'bar'
+  │ ^
+",
+    )
+    .status(EXIT_FAILURE)
+    .run();
 }
 
 test! {