builtin: speed up, fix and test impl_utf8_to_utf32 (#26109)

enghitalo · web-flow · commit 9dd89946de60 · 2025-12-25T23:17:55.000+02:00
diff --git a/vlib/builtin/utf8.v b/vlib/builtin/utf8.v
@@ -95,24 +95,37 @@ pub fn (_bytes []u8) utf8_to_utf32() !rune {
 
 @[direct_array_access]
 fn impl_utf8_to_utf32(_bytes &u8, _bytes_len int) rune {
-	if _bytes_len == 0 {
+	if _bytes_len == 0 || _bytes_len > 4 {
 		return 0
 	}
 	// return ASCII unchanged
 	if _bytes_len == 1 {
-		return unsafe { rune(_bytes[0]) }
+		return rune(unsafe { _bytes[0] })
 	}
-	mut b := u8(int(unsafe { _bytes[0] }))
-	b = b << _bytes_len
-	mut res := rune(b)
-	mut shift := 6 - _bytes_len
-	for i := 1; i < _bytes_len; i++ {
-		c := rune(unsafe { _bytes[i] })
-		res = rune(res) << shift
-		res |= c & 63 // 0x3f
-		shift = 6
+
+	match _bytes_len {
+		2 {
+			b0 := rune(unsafe { _bytes[0] })
+			b1 := rune(unsafe { _bytes[1] })
+			return ((b0 & 0x1F) << 6) | (b1 & 0x3F)
+		}
+		3 {
+			b0 := rune(unsafe { _bytes[0] })
+			b1 := rune(unsafe { _bytes[1] })
+			b2 := rune(unsafe { _bytes[2] })
+			return ((b0 & 0x0F) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F)
+		}
+		4 {
+			b0 := rune(unsafe { _bytes[0] })
+			b1 := rune(unsafe { _bytes[1] })
+			b2 := rune(unsafe { _bytes[2] })
+			b3 := rune(unsafe { _bytes[3] })
+			return ((b0 & 0x07) << 18) | ((b1 & 0x3F) << 12) | ((b2 & 0x3F) << 6) | (b3 & 0x3F)
+		}
+		else {
+			return 0
+		}
 	}
-	return res
 }
 
 // Calculate string length for formatting, i.e. number of "characters"
diff --git a/vlib/builtin/utf8_test.v b/vlib/builtin/utf8_test.v
@@ -101,3 +101,63 @@ fn test_string_to_ansi_not_null_terminated() {
 fn test_utf8_str_visible_length() {
 	assert utf8_str_visible_length('𝐀𝐁𝐂') == 3
 }
+
+fn test_utf8_to_utf32_cases() {
+	test_case1 := 'A'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case1.data), test_case1.len) == rune(`A`)
+
+	test_case2 := 'é'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case2.data), test_case2.len) == rune(`é`)
+
+	test_case3 := '€'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case3.data), test_case3.len) == rune(`€`)
+
+	test_case4 := '𐍈'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case4.data), test_case4.len) == rune(0x10348)
+	assert impl_utf8_to_utf32(&u8(test_case4.data), test_case4.len) == rune(`𐍈`)
+
+	test_case5 := '中'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case5.data), test_case5.len) == rune(0x4E2D)
+	assert impl_utf8_to_utf32(&u8(test_case5.data), test_case5.len) == rune(`中`)
+
+	// emoji, 4-byte UTF-8
+	test_case6 := '😀'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case6.data), test_case6.len) == rune(0x1F600)
+	assert impl_utf8_to_utf32(&u8(test_case6.data), test_case6.len) == `😀`
+
+	test_case7 := 'Ж'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case7.data), test_case7.len) == rune(`Ж`)
+
+	test_case8 := 'م'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case8.data), test_case8.len) == rune(`م`)
+
+	test_case9 := '߿'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case9.data), test_case9.len) == rune(0x07FF)
+	assert impl_utf8_to_utf32(&u8(test_case9.data), test_case9.len) == rune(`߿`)
+
+	test_case10 := 'ࠀ'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case10.data), test_case10.len) == rune(0x0800)
+	assert impl_utf8_to_utf32(&u8(test_case10.data), test_case10.len) == rune(`ࠀ`)
+
+	test_case11 := '￿'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case11.data), test_case11.len) == rune(0xFFFF)
+	assert impl_utf8_to_utf32(&u8(test_case11.data), test_case11.len) == rune(`￿`)
+
+	test_case12 := '𐀀'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case12.data), test_case12.len) == rune(0x10000)
+	assert impl_utf8_to_utf32(&u8(test_case12.data), test_case12.len) == rune(`𐀀`)
+
+	test_case13 := '􏿿'.bytes()
+	assert impl_utf8_to_utf32(&u8(test_case13.data), test_case13.len) == rune(0x10FFFF)
+	assert impl_utf8_to_utf32(&u8(test_case13.data), test_case13.len) == rune(`􏿿`)
+}
+
+fn test_utf8_to_utf32_invalid_length() {
+	// More than 4 bytes is invalid
+	invalid := [u8(0xF0), 0x9F, 0x98, 0x80, 0x00]
+	assert impl_utf8_to_utf32(&u8(invalid.data), invalid.len) == 0
+}
+
+fn test_utf8_to_utf32_empty() {
+	assert impl_utf8_to_utf32(&u8([]u8{}.data), 0) == 0
+}
