Canonicalize unicode code point escapes

compiler/can/src/expr.rs

@@ -22,7 +22,7 @@ use roc_region::all::{Located, Region};
 use roc_types::subs::{VarStore, Variable};
 use roc_types::types::Alias;
 use std::fmt::Debug;
-use std::i64;
+use std::{char, i64, u32};
 
 #[derive(Clone, Default, Debug, PartialEq)]
 pub struct Output {
@@ -1359,7 +1359,7 @@ fn flatten_str_lines<'a>(
     use ast::StrSegment::*;
 
     let mut buf = String::new();
-    let mut interpolations = Vec::new();
+    let mut segments = Vec::new();
     let mut output = Output::default();
 
     for line in lines {
@@ -1368,11 +1368,41 @@ fn flatten_str_lines<'a>(
                 Plaintext(string) => {
                     buf.push_str(string);
                 }
-                Unicode(loc_digits) => {
+                Unicode(loc_hex_digits) => match u32::from_str_radix(loc_hex_digits.value, 16) {
-                    todo!("parse unicode digits {:?}", loc_digits);
+                    Ok(code_pt) => match char::from_u32(code_pt) {
+                        Some(ch) => {
+                            buf.push(ch);
                         }
+                        None => {
+                            env.problem(Problem::InvalidUnicodeCodePoint(loc_hex_digits.region));
+
+                            return (
+                                Expr::RuntimeError(RuntimeError::InvalidUnicodeCodePoint(
+                                    loc_hex_digits.region,
+                                )),
+                                output,
+                            );
+                        }
+                    },
+                    Err(_) => {
+                        env.problem(Problem::InvalidHexadecimal(loc_hex_digits.region));
+
+                        return (
+                            Expr::RuntimeError(RuntimeError::InvalidHexadecimal(
+                                loc_hex_digits.region,
+                            )),
+                            output,
+                        );
+                    }
+                },
                 Interpolated(loc_expr) => {
                     if is_valid_interpolation(loc_expr.value) {
+                        if !buf.is_empty() {
+                            segments.push(StrSegment::Plaintext(buf.into()));
+
+                            buf = String::new();
+                        }
+
                         let (loc_expr, new_output) = canonicalize_expr(
                             env,
                             var_store,
@@ -1383,7 +1413,7 @@ fn flatten_str_lines<'a>(
 
                         output.union(new_output);
 
-                        interpolations.push(StrSegment::Interpolation(loc_expr));
+                        segments.push(StrSegment::Interpolation(loc_expr));
                     } else {
                         env.problem(Problem::InvalidInterpolation(loc_expr.region));
 
@@ -1398,7 +1428,11 @@ fn flatten_str_lines<'a>(
         }
     }
 
-    (Expr::Str(interpolations), output)
+    if !buf.is_empty() {
+        segments.push(StrSegment::Plaintext(buf.into()));
+    }
+
+    (Expr::Str(segments), output)
 }
 
 /// Returns the char that would have been originally parsed to

compiler/can/tests/test_can.rs

@@ -15,7 +15,7 @@ mod test_can {
     use crate::helpers::{can_expr_with, test_home, CanExprOut};
     use bumpalo::Bump;
     use roc_can::expr::Expr::{self, *};
-    use roc_can::expr::Recursive;
+    use roc_can::expr::{Recursive, StrSegment};
     use roc_problem::can::{FloatErrorKind, IntErrorKind, Problem, RuntimeError};
     use roc_region::all::Region;
     use std::{f64, i64};
@@ -69,6 +69,10 @@ mod test_can {
         }
     }
 
+    fn expr_str(contents: &str) -> Expr {
+        Expr::Str(vec![StrSegment::Plaintext(contents.into())])
+    }
+
     // NUMBER LITERALS
 
     #[test]
@@ -1179,14 +1183,13 @@ mod test_can {
     //}
     //
     //
-    //// STRING LITERALS
+    // STRING LITERALS
 
-    //
+    #[test]
-    // #[test]
+    fn string_with_valid_unicode_escapes() {
-    // fn string_with_valid_unicode_escapes() {
+        assert_can(r#""x\u(00A0)x""#, expr_str("x\u{00A0}x"));
-    //     expect_parsed_str("x\u{00A0}x", r#""x\u{00A0}x""#);
+        assert_can(r#""x\u(101010)x""#, expr_str("x\u{101010}x"));
-    //     expect_parsed_str("x\u{101010}x", r#""x\u{101010}x""#);
+    }
-    // }
 
     //     #[test]
     //     fn string_with_too_large_unicode_escape() {
@@ -1248,7 +1251,6 @@ mod test_can {
     //             Str(r#"abcd\(efg)hij"#.into()),
     //         );
     //     }
-    //
 
     //     #[test]
     //     fn string_without_escape() {

compiler/problem/src/can.rs

@@ -56,6 +56,8 @@ pub enum Problem {
         region: Region,
     },
     InvalidInterpolation(Region),
+    InvalidHexadecimal(Region),
+    InvalidUnicodeCodePoint(Region),
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -127,6 +129,8 @@ pub enum RuntimeError {
     NonExhaustivePattern,
 
     InvalidInterpolation(Region),
+    InvalidHexadecimal(Region),
+    InvalidUnicodeCodePoint(Region),
 
     /// When the author specifies a type annotation but no implementation
     NoImplementation,

compiler/reporting/src/error/canonicalize.rs

@@ -262,6 +262,18 @@ pub fn can_problem<'b>(
                 alloc.reflow(" can occur in this position."),
             ]),
         ]),
+        Problem::InvalidHexadecimal(region) => {
+            todo!(
+                "TODO report an invalid hexadecimal number in a \\u(...) code point at region {:?}",
+                region
+            );
+        }
+        Problem::InvalidUnicodeCodePoint(region) => {
+            todo!(
+                "TODO report an invalid \\u(...) code point at region {:?}",
+                region
+            );
+        }
         Problem::InvalidInterpolation(region) => {
             todo!(
                 "TODO report an invalid string interpolation at region {:?}",
@@ -530,6 +542,18 @@ fn pretty_runtime_error<'b>(
             alloc.region(region),
             alloc.reflow("Only variables can be updated with record update syntax."),
         ]),
+        RuntimeError::InvalidHexadecimal(region) => {
+            todo!(
+                "TODO runtime error for an invalid hexadecimal number in a \\u(...) code point at region {:?}",
+                region
+            );
+        }
+        RuntimeError::InvalidUnicodeCodePoint(region) => {
+            todo!(
+                "TODO runtime error for an invalid \\u(...) code point at region {:?}",
+                region
+            );
+        }
         RuntimeError::InvalidInterpolation(region) => {
             todo!(
                 "TODO runtime error for an invalid string interpolation at region {:?}",