Merge pull request #3371 from rtfeldman/starts-with-scalar

Str.startsWithCodePt -> Str.startsWithScalar

crates/compiler/builtins/bitcode/src/main.zig

@@ -157,7 +157,7 @@ comptime {
     exportStrFn(str.countSegments, "count_segments");
     exportStrFn(str.countGraphemeClusters, "count_grapheme_clusters");
     exportStrFn(str.startsWith, "starts_with");
-    exportStrFn(str.startsWithCodePt, "starts_with_code_point");
+    exportStrFn(str.startsWithScalar, "starts_with_scalar");
     exportStrFn(str.endsWith, "ends_with");
     exportStrFn(str.strConcatC, "concat");
     exportStrFn(str.strJoinWithC, "joinWith");

crates/compiler/builtins/bitcode/src/str.zig

@@ -490,74 +490,14 @@ fn strToScalars(string: RocStr) callconv(.C) RocList {
     // should not require a second allocation.
     var answer = RocList.allocate(@alignOf(u32), capacity, @sizeOf(u32));
 
-    // We already did an early return to verify the string was nonempty.
+    // `orelse unreachable` is fine here, because we already did an early
+    // return to verify the string was nonempty.
     var answer_elems = answer.elements(u32) orelse unreachable;
     var src_index: usize = 0;
     var answer_index: usize = 0;
 
     while (src_index < str_len) {
-        const utf8_byte = string.getUnchecked(src_index);
-
-        // How UTF-8 bytes work:
-        // https://docs.teradata.com/r/Teradata-Database-International-Character-Set-Support/June-2017/Client-Character-Set-Options/UTF8-Client-Character-Set-Support/UTF8-Multibyte-Sequences
-        if (utf8_byte <= 127) {
-            // It's an ASCII character. Copy it over directly.
-            answer_elems[answer_index] = @intCast(u32, utf8_byte);
-            src_index += 1;
-        } else if (utf8_byte >> 5 == 0b0000_0110) {
-            // Its three high order bits are 110, so this is a two-byte sequence.
-
-            // Example:
-            //     utf-8:   1100 1111   1011 0001
-            //     code pt: 0000 0011   1111 0001 (decimal: 1009)
-
-            // Discard the first byte's high order bits of 110.
-            var code_pt = @intCast(u32, utf8_byte & 0b0001_1111);
-
-            // Discard the second byte's high order bits of 10.
-            code_pt <<= 6;
-            code_pt |= string.getUnchecked(src_index + 1) & 0b0011_1111;
-
-            answer_elems[answer_index] = code_pt;
-            src_index += 2;
-        } else if (utf8_byte >> 4 == 0b0000_1110) {
-            // Its four high order bits are 1110, so this is a three-byte sequence.
-
-            // Discard the first byte's high order bits of 1110.
-            var code_pt = @intCast(u32, utf8_byte & 0b0000_1111);
-
-            // Discard the second byte's high order bits of 10.
-            code_pt <<= 6;
-            code_pt |= string.getUnchecked(src_index + 1) & 0b0011_1111;
-
-            // Discard the third byte's high order bits of 10 (same as second byte).
-            code_pt <<= 6;
-            code_pt |= string.getUnchecked(src_index + 2) & 0b0011_1111;
-
-            answer_elems[answer_index] = code_pt;
-            src_index += 3;
-        } else {
-            // This must be a four-byte sequence, so the five high order bits should be 11110.
-
-            // Discard the first byte's high order bits of 11110.
-            var code_pt = @intCast(u32, utf8_byte & 0b0000_0111);
-
-            // Discard the second byte's high order bits of 10.
-            code_pt <<= 6;
-            code_pt |= string.getUnchecked(src_index + 1) & 0b0011_1111;
-
-            // Discard the third byte's high order bits of 10 (same as second byte).
-            code_pt <<= 6;
-            code_pt |= string.getUnchecked(src_index + 2) & 0b0011_1111;
-
-            // Discard the fourth byte's high order bits of 10 (same as second and third).
-            code_pt <<= 6;
-            code_pt |= string.getUnchecked(src_index + 3) & 0b0011_1111;
-
-            answer_elems[answer_index] = code_pt;
-            src_index += 4;
-        }
-
+        src_index += writeNextScalar(string, src_index, answer_elems, answer_index);
         answer_index += 1;
     }
 
@@ -566,6 +506,78 @@ fn strToScalars(string: RocStr) callconv(.C) RocList {
     return answer;
 }
 
+// Given a non-empty RocStr, and a src_index byte index into that string,
+// and a destination [*]u32, and an index into that destination,
+// Parses the next scalar value out of the string (at the given byte index),
+// writes it into the destination, and returns the number of bytes parsed.
+inline fn writeNextScalar(non_empty_string: RocStr, src_index: usize, dest: [*]u32, dest_index: usize) usize {
+    const utf8_byte = non_empty_string.getUnchecked(src_index);
+
+    // How UTF-8 bytes work:
+    // https://docs.teradata.com/r/Teradata-Database-International-Character-Set-Support/June-2017/Client-Character-Set-Options/UTF8-Client-Character-Set-Support/UTF8-Multibyte-Sequences
+    if (utf8_byte <= 127) {
+        // It's an ASCII character. Copy it over directly.
+        dest[dest_index] = @intCast(u32, utf8_byte);
+
+        return 1;
+    } else if (utf8_byte >> 5 == 0b0000_0110) {
+        // Its three high order bits are 110, so this is a two-byte sequence.
+
+        // Example:
+        //     utf-8:   1100 1111   1011 0001
+        //     code pt: 0000 0011   1111 0001 (decimal: 1009)
+
+        // Discard the first byte's high order bits of 110.
+        var code_pt = @intCast(u32, utf8_byte & 0b0001_1111);
+
+        // Discard the second byte's high order bits of 10.
+        code_pt <<= 6;
+        code_pt |= non_empty_string.getUnchecked(src_index + 1) & 0b0011_1111;
+
+        dest[dest_index] = code_pt;
+
+        return 2;
+    } else if (utf8_byte >> 4 == 0b0000_1110) {
+        // Its four high order bits are 1110, so this is a three-byte sequence.
+
+        // Discard the first byte's high order bits of 1110.
+        var code_pt = @intCast(u32, utf8_byte & 0b0000_1111);
+
+        // Discard the second byte's high order bits of 10.
+        code_pt <<= 6;
+        code_pt |= non_empty_string.getUnchecked(src_index + 1) & 0b0011_1111;
+
+        // Discard the third byte's high order bits of 10 (same as second byte).
+        code_pt <<= 6;
+        code_pt |= non_empty_string.getUnchecked(src_index + 2) & 0b0011_1111;
+
+        dest[dest_index] = code_pt;
+
+        return 3;
+    } else {
+        // This must be a four-byte sequence, so the five high order bits should be 11110.
+
+        // Discard the first byte's high order bits of 11110.
+        var code_pt = @intCast(u32, utf8_byte & 0b0000_0111);
+
+        // Discard the second byte's high order bits of 10.
+        code_pt <<= 6;
+        code_pt |= non_empty_string.getUnchecked(src_index + 1) & 0b0011_1111;
+
+        // Discard the third byte's high order bits of 10 (same as second byte).
+        code_pt <<= 6;
+        code_pt |= non_empty_string.getUnchecked(src_index + 2) & 0b0011_1111;
+
+        // Discard the fourth byte's high order bits of 10 (same as second and third).
+        code_pt <<= 6;
+        code_pt |= non_empty_string.getUnchecked(src_index + 3) & 0b0011_1111;
+
+        dest[dest_index] = code_pt;
+
+        return 4;
+    }
+}
+
 test "strToScalars: empty string" {
     const str = RocStr.fromSlice("");
     defer RocStr.deinit(str);
@@ -1211,56 +1223,60 @@ pub fn repeat(string: RocStr, count: usize) callconv(.C) RocStr {
     return ret_string;
 }
 
-// Str.startsWithCodePt
-pub fn startsWithCodePt(string: RocStr, prefix: u32) callconv(.C) bool {
-    const bytes_ptr = string.asU8ptr();
+// Str.startsWithScalar
+pub fn startsWithScalar(string: RocStr, prefix: u32) callconv(.C) bool {
+    const str_len = string.len();
 
-    var buffer: [4]u8 = undefined;
-
-    var width = std.unicode.utf8Encode(@truncate(u21, prefix), &buffer) catch unreachable;
-
-    var i: usize = 0;
-    while (i < width) : (i += 1) {
-        const a = buffer[i];
-        const b = bytes_ptr[i];
-        if (a != b) {
-            return false;
-        }
+    if (str_len == 0) {
+        return false;
     }
 
-    return true;
+    // Write this (non-empty) string's first scalar into `first_scalar`
+    var first_scalar: [1]u32 = undefined;
+
+    _ = writeNextScalar(string, 0, &first_scalar, 0);
+
+    // Return whether `first_scalar` equals `prefix`
+    return @ptrCast(*u32, &first_scalar).* == prefix;
 }
 
-test "startsWithCodePt: ascii char" {
-    const whole = RocStr.init("foobar", 6);
-    const prefix = 'f';
-    try expect(startsWithCodePt(whole, prefix));
+test "startsWithScalar: empty string" {
+    const whole = RocStr.empty();
+    const prefix: u32 = 'x';
+    try expect(!startsWithScalar(whole, prefix));
 }
 
-test "startsWithCodePt: emoji" {
-    const yes = RocStr.init("💖foobar", 10);
-    const no = RocStr.init("foobar", 6);
-    const prefix = '💖';
-    try expect(startsWithCodePt(yes, prefix));
-    try expect(!startsWithCodePt(no, prefix));
+test "startsWithScalar: ascii char" {
+    const whole = RocStr.fromSlice("foobar");
+    const prefix: u32 = 'f';
+    try expect(startsWithScalar(whole, prefix));
+}
+
+test "startsWithScalar: emoji" {
+    const yes = RocStr.fromSlice("💖foobar");
+    const no = RocStr.fromSlice("foobar");
+    const prefix: u32 = '💖';
+
+    try expect(startsWithScalar(yes, prefix));
+    try expect(!startsWithScalar(no, prefix));
 }
 
 test "startsWith: foo starts with fo" {
-    const foo = RocStr.init("foo", 3);
-    const fo = RocStr.init("fo", 2);
+    const foo = RocStr.fromSlice("foo");
+    const fo = RocStr.fromSlice("fo");
     try expect(startsWith(foo, fo));
 }
 
 test "startsWith: 123456789123456789 starts with 123456789123456789" {
-    const str = RocStr.init("123456789123456789", 18);
+    const str = RocStr.fromSlice("123456789123456789");
     defer str.deinit();
     try expect(startsWith(str, str));
 }
 
 test "startsWith: 12345678912345678910 starts with 123456789123456789" {
-    const str = RocStr.init("12345678912345678910", 20);
+    const str = RocStr.fromSlice("12345678912345678910");
     defer str.deinit();
-    const prefix = RocStr.init("123456789123456789", 18);
+    const prefix = RocStr.fromSlice("123456789123456789");
     defer prefix.deinit();
 
     try expect(startsWith(str, prefix));

crates/compiler/builtins/roc/Str.roc

@@ -9,7 +9,7 @@ interface Str
             split,
             repeat,
             countGraphemes,
-            startsWithCodePt,
+            startsWithScalar,
             toUtf8,
             fromUtf8,
             fromUtf8Range,
@@ -165,13 +165,13 @@ countGraphemes : Str -> Nat
 ##
 ## **Performance Note:** This runs slightly faster than [Str.startsWith], so
 ## if you want to check whether a string begins with something that's representable
-## in a single code point, you can use (for example) `Str.startsWithCodePt '鹏'`
-## instead of `Str.startsWithCodePt "鹏"`. ('鹏' evaluates to the [U32]
+## in a single code point, you can use (for example) `Str.startsWithScalar '鹏'`
+## instead of `Str.startsWithScalar "鹏"`. ('鹏' evaluates to the [U32]
 ## value `40527`.) This will not work for graphemes which take up multiple code
-## points, however; `Str.startsWithCodePt '👩‍👩‍👦‍👦'` would be a compiler error
+## points, however; `Str.startsWithScalar '👩‍👩‍👦‍👦'` would be a compiler error
 ## because 👩‍👩‍👦‍👦 takes up multiple code points and cannot be represented as a
-## single [U32]. You'd need to use `Str.startsWithCodePt "🕊"` instead.
-startsWithCodePt : Str, U32 -> Bool
+## single [U32]. You'd need to use `Str.startsWithScalar "🕊"` instead.
+startsWithScalar : Str, U32 -> Bool
 
 ## Return a [List] of the [unicode scalar values](https://unicode.org/glossary/#unicode_scalar_value)
 ## in the given string.

crates/compiler/builtins/src/bitcode.rs

@@ -314,7 +314,7 @@ pub const STR_STR_SPLIT_IN_PLACE: &str = "roc_builtins.str.str_split_in_place";
 pub const STR_TO_SCALARS: &str = "roc_builtins.str.to_scalars";
 pub const STR_COUNT_GRAPEHEME_CLUSTERS: &str = "roc_builtins.str.count_grapheme_clusters";
 pub const STR_STARTS_WITH: &str = "roc_builtins.str.starts_with";
-pub const STR_STARTS_WITH_CODE_PT: &str = "roc_builtins.str.starts_with_code_point";
+pub const STR_STARTS_WITH_SCALAR: &str = "roc_builtins.str.starts_with_scalar";
 pub const STR_ENDS_WITH: &str = "roc_builtins.str.ends_with";
 pub const STR_NUMBER_OF_BYTES: &str = "roc_builtins.str.number_of_bytes";
 pub const STR_FROM_INT: IntrinsicName = int_intrinsic!("roc_builtins.str.from_int");

crates/compiler/builtins/src/std.rs

@@ -894,9 +894,9 @@ pub fn types() -> MutMap<Symbol, (SolvedType, Region)> {
         Box::new(bool_type())
     );
 
-    // startsWithCodePt : Str, U32 -> Bool
+    // startsWithScalar : Str, U32 -> Bool
     add_top_level_function_type!(
-        Symbol::STR_STARTS_WITH_CODE_PT,
+        Symbol::STR_STARTS_WITH_SCALAR,
         vec![str_type(), u32_type()],
         Box::new(bool_type())
     );

crates/compiler/can/src/builtins.rs

@@ -77,7 +77,7 @@ pub fn builtin_defs_map(symbol: Symbol, var_store: &mut VarStore) -> Option<Def>
         STR_SPLIT => str_split,
         STR_IS_EMPTY => str_is_empty,
         STR_STARTS_WITH => str_starts_with,
-        STR_STARTS_WITH_CODE_PT => str_starts_with_code_point,
+        STR_STARTS_WITH_SCALAR => str_starts_with_scalar,
         STR_ENDS_WITH => str_ends_with,
         STR_COUNT_GRAPHEMES => str_count_graphemes,
         STR_FROM_UTF8 => str_from_utf8,
@@ -1741,9 +1741,9 @@ fn str_starts_with(symbol: Symbol, var_store: &mut VarStore) -> Def {
     lowlevel_2(symbol, LowLevel::StrStartsWith, var_store)
 }
 
-/// Str.startsWithCodePt : Str, U32 -> Bool
-fn str_starts_with_code_point(symbol: Symbol, var_store: &mut VarStore) -> Def {
-    lowlevel_2(symbol, LowLevel::StrStartsWithCodePt, var_store)
+/// Str.startsWithScalar : Str, U32 -> Bool
+fn str_starts_with_scalar(symbol: Symbol, var_store: &mut VarStore) -> Def {
+    lowlevel_2(symbol, LowLevel::StrStartsWithScalar, var_store)
 }
 
 /// Str.endsWith : Str, Str -> Bool

crates/compiler/gen_llvm/src/llvm/build.rs

@@ -5327,14 +5327,14 @@ fn run_low_level<'a, 'ctx, 'env>(
 
             call_bitcode_fn(env, &[string, prefix], bitcode::STR_STARTS_WITH)
         }
-        StrStartsWithCodePt => {
-            // Str.startsWithCodePt : Str, U32 -> Bool
+        StrStartsWithScalar => {
+            // Str.startsWithScalar : Str, U32 -> Bool
             debug_assert_eq!(args.len(), 2);
 
             let string = load_symbol(scope, &args[0]);
             let prefix = load_symbol(scope, &args[1]);
 
-            call_bitcode_fn(env, &[string, prefix], bitcode::STR_STARTS_WITH_CODE_PT)
+            call_bitcode_fn(env, &[string, prefix], bitcode::STR_STARTS_WITH_SCALAR)
         }
         StrEndsWith => {
             // Str.startsWith : Str, Str -> Bool

crates/compiler/gen_wasm/src/low_level.rs

@@ -231,8 +231,8 @@ impl<'a> LowLevelCall<'a> {
                 _ => internal_error!("invalid storage for Str"),
             },
             StrStartsWith => self.load_args_and_call_zig(backend, bitcode::STR_STARTS_WITH),
-            StrStartsWithCodePt => {
-                self.load_args_and_call_zig(backend, bitcode::STR_STARTS_WITH_CODE_PT)
+            StrStartsWithScalar => {
+                self.load_args_and_call_zig(backend, bitcode::STR_STARTS_WITH_SCALAR)
             }
             StrEndsWith => self.load_args_and_call_zig(backend, bitcode::STR_ENDS_WITH),
             StrSplit => {

crates/compiler/module/src/low_level.rs

@@ -9,7 +9,7 @@ pub enum LowLevel {
     StrJoinWith,
     StrIsEmpty,
     StrStartsWith,
-    StrStartsWithCodePt,
+    StrStartsWithScalar,
     StrEndsWith,
     StrSplit,
     StrCountGraphemes,
@@ -189,7 +189,7 @@ impl LowLevelWrapperType {
             Symbol::STR_JOIN_WITH => CanBeReplacedBy(StrJoinWith),
             Symbol::STR_IS_EMPTY => CanBeReplacedBy(StrIsEmpty),
             Symbol::STR_STARTS_WITH => CanBeReplacedBy(StrStartsWith),
-            Symbol::STR_STARTS_WITH_CODE_PT => CanBeReplacedBy(StrStartsWithCodePt),
+            Symbol::STR_STARTS_WITH_SCALAR => CanBeReplacedBy(StrStartsWithScalar),
             Symbol::STR_ENDS_WITH => CanBeReplacedBy(StrEndsWith),
             Symbol::STR_SPLIT => CanBeReplacedBy(StrSplit),
             Symbol::STR_COUNT_GRAPHEMES => CanBeReplacedBy(StrCountGraphemes),

crates/compiler/module/src/symbol.rs

@@ -1168,7 +1168,7 @@ define_builtins! {
         10 STR_UT8_PROBLEM: "Utf8Problem" // the Utf8Problem type alias
         11 STR_UT8_BYTE_PROBLEM: "Utf8ByteProblem" // the Utf8ByteProblem type alias
         12 STR_TO_UTF8: "toUtf8"
-        13 STR_STARTS_WITH_CODE_PT: "startsWithCodePt"
+        13 STR_STARTS_WITH_SCALAR: "startsWithScalar"
         14 STR_ALIAS_ANALYSIS_STATIC: "#aliasAnalysisStatic" // string with the static lifetime
         15 STR_FROM_UTF8_RANGE: "fromUtf8Range"
         16 STR_REPEAT: "repeat"

crates/compiler/mono/src/borrow.rs

@@ -939,7 +939,7 @@ pub fn lowlevel_borrow_signature(arena: &Bump, op: LowLevel) -> &[bool] {
         NumBytesToU16 => arena.alloc_slice_copy(&[borrowed, irrelevant]),
         NumBytesToU32 => arena.alloc_slice_copy(&[borrowed, irrelevant]),
         StrStartsWith | StrEndsWith => arena.alloc_slice_copy(&[owned, borrowed]),
-        StrStartsWithCodePt => arena.alloc_slice_copy(&[borrowed, irrelevant]),
+        StrStartsWithScalar => arena.alloc_slice_copy(&[borrowed, irrelevant]),
         StrFromUtf8 => arena.alloc_slice_copy(&[owned]),
         StrFromUtf8Range => arena.alloc_slice_copy(&[borrowed, irrelevant]),
         StrToUtf8 => arena.alloc_slice_copy(&[owned]),

crates/compiler/mono/src/low_level.rs

@@ -105,7 +105,7 @@ enum FirstOrder {
     StrJoinWith,
     StrIsEmpty,
     StrStartsWith,
-    StrStartsWithCodePt,
+    StrStartsWithScalar,
     StrEndsWith,
     StrSplit,
     StrCountGraphemes,

crates/compiler/test_gen/src/gen_str.rs

@@ -496,14 +496,14 @@ fn str_starts_with() {
 
 #[test]
 #[cfg(any(feature = "gen-llvm"))]
-fn str_starts_with_code_point() {
+fn str_starts_with_scalar() {
     assert_evals_to!(
-        &format!(r#"Str.startsWithCodePt "foobar" {}"#, 'f' as u32),
+        &format!(r#"Str.startsWithScalar "foobar" {}"#, 'f' as u32),
         true,
         bool
     );
     assert_evals_to!(
-        &format!(r#"Str.startsWithCodePt "zoobar" {}"#, 'f' as u32),
+        &format!(r#"Str.startsWithScalar "zoobar" {}"#, 'f' as u32),
         false,
         bool
     );

crates/compiler/test_gen/src/wasm_str.rs

@@ -417,14 +417,14 @@ fn str_starts_with() {
 }
 
 #[test]
-fn str_starts_with_code_point() {
+fn str_starts_with_scalar() {
     assert_evals_to!(
-        &format!(r#"Str.startsWithCodePt "foobar" {}"#, 'f' as u32),
+        &format!(r#"Str.startsWithScalar "foobar" {}"#, 'f' as u32),
         true,
         bool
     );
     assert_evals_to!(
-        &format!(r#"Str.startsWithCodePt "zoobar" {}"#, 'f' as u32),
+        &format!(r#"Str.startsWithScalar "zoobar" {}"#, 'f' as u32),
         false,
         bool
     );