Implement target-specific functions for neon-i8x32 and neon-i16x16

aneshlya · aneshlya · commit 52b053b01bb1 · 2025-01-24T17:46:15.000Z
diff --git a/builtins/target-neon-i16x16.ll b/builtins/target-neon-i16x16.ll
@@ -4,4 +4,121 @@
 
 define(`WIDTH',`16')
 define(`MASK',`i16')
-define(`ISA',`NEON')
+define(`ISA',`NEON')
+
+include(`util.m4')
+
+define(`NEON_PREFIX',
+`ifelse(RUNTIME, `64', `llvm.aarch64.neon',
+        RUNTIME, `32', `llvm.arm.neon')')
+
+define(`NEON_PREFIX_UDOT',
+`ifelse(RUNTIME, `64', `llvm.aarch64.neon.udot',
+        RUNTIME, `32', `llvm.arm.neon.udot')')
+
+define(`NEON_PREFIX_SDOT',
+`ifelse(RUNTIME, `64', `llvm.aarch64.neon.sdot',
+        RUNTIME, `32', `llvm.arm.neon.sdot')')
+
+define(`NEON_PREFIX_USDOT',
+`ifelse(RUNTIME, `64', `llvm.aarch64.neon.usdot',
+        RUNTIME, `32', `llvm.arm.neon.usdot')')
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; half conversion routines
+
+declare <4 x i16> @NEON_PREFIX.vcvtfp2hf(<4 x float>) nounwind readnone
+declare <4 x float> @NEON_PREFIX.vcvthf2fp(<4 x i16>) nounwind readnone
+
+define float @__half_to_float_uniform(i16 %v) nounwind readnone alwaysinline {
+  %v1 = bitcast i16 %v to <1 x i16>
+  %vec = shufflevector <1 x i16> %v1, <1 x i16> undef,
+           <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %h = call <4 x float> @NEON_PREFIX.vcvthf2fp(<4 x i16> %vec)
+  %r = extractelement <4 x float> %h, i32 0
+  ret float %r
+}
+
+define i16 @__float_to_half_uniform(float %v) nounwind readnone alwaysinline {
+  %v1 = bitcast float %v to <1 x float>
+  %vec = shufflevector <1 x float> %v1, <1 x float> undef,
+           <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %h = call <4 x i16> @NEON_PREFIX.vcvtfp2hf(<4 x float> %vec)
+  %r = extractelement <4 x i16> %h, i32 0
+  ret i16 %r
+}
+
+define <16 x float> @__half_to_float_varying(<16 x i16> %v) nounwind readnone alwaysinline {
+  unary4to16conv(r, i16, float, @NEON_PREFIX.vcvthf2fp, %v)
+  ret <16 x float> %r
+}
+
+define <16 x i16> @__float_to_half_varying(<16 x float> %v) nounwind readnone alwaysinline {
+  unary4to16conv(r, float, i16, @NEON_PREFIX.vcvtfp2hf, %v)
+  ret <16 x i16> %r
+}
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; dot product
+
+declare <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) nounwind readnone
+define <16 x i32> @__dot4add_u8u8packed(<16 x i32> %a, <16 x i32> %b, <16 x i32> %acc) nounwind readnone alwaysinline {
+  v16tov4(i32, %a, %a0, %a1, %a2, %a3)
+  v16tov4(i32, %b, %b0, %b1, %b2, %b3)
+  v16tov4(i32, %acc, %acc0, %acc1, %acc2, %acc3)
+  %a0_cast = bitcast <4 x i32> %a0 to <16 x i8>
+  %b0_cast = bitcast <4 x i32> %b0 to <16 x i8>
+  %a1_cast = bitcast <4 x i32> %a1 to <16 x i8>
+  %b1_cast = bitcast <4 x i32> %b1 to <16 x i8>
+  %a2_cast = bitcast <4 x i32> %a2 to <16 x i8>
+  %b2_cast = bitcast <4 x i32> %b2 to <16 x i8>
+  %a3_cast = bitcast <4 x i32> %a3 to <16 x i8>
+  %b3_cast = bitcast <4 x i32> %b3 to <16 x i8>
+  %ret0 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc0, <16 x i8> %a0_cast, <16 x i8> %b0_cast)
+  %ret1 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc1, <16 x i8> %a1_cast, <16 x i8> %b1_cast)
+  %ret2 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc2, <16 x i8> %a2_cast, <16 x i8> %b2_cast)
+  %ret3 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc3, <16 x i8> %a3_cast, <16 x i8> %b3_cast)
+  v4tov16(i32, %ret0, %ret1, %ret2, %ret3, %ret)
+  ret <16 x i32> %ret
+}
+
+declare <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) nounwind readnone
+define <16 x i32> @__dot4add_i8i8packed(<16 x i32> %a, <16 x i32> %b, <16 x i32> %acc) nounwind readnone alwaysinline {
+  v16tov4(i32, %a, %a0, %a1, %a2, %a3)
+  v16tov4(i32, %b, %b0, %b1, %b2, %b3)
+  v16tov4(i32, %acc, %acc0, %acc1, %acc2, %acc3)
+  %a0_cast = bitcast <4 x i32> %a0 to <16 x i8>
+  %b0_cast = bitcast <4 x i32> %b0 to <16 x i8>
+  %a1_cast = bitcast <4 x i32> %a1 to <16 x i8>
+  %b1_cast = bitcast <4 x i32> %b1 to <16 x i8>
+  %a2_cast = bitcast <4 x i32> %a2 to <16 x i8>
+  %b2_cast = bitcast <4 x i32> %b2 to <16 x i8>
+  %a3_cast = bitcast <4 x i32> %a3 to <16 x i8>
+  %b3_cast = bitcast <4 x i32> %b3 to <16 x i8>
+  %ret0 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc0, <16 x i8> %a0_cast, <16 x i8> %b0_cast)
+  %ret1 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc1, <16 x i8> %a1_cast, <16 x i8> %b1_cast)
+  %ret2 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc2, <16 x i8> %a2_cast, <16 x i8> %b2_cast)
+  %ret3 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc3, <16 x i8> %a3_cast, <16 x i8> %b3_cast)
+  v4tov16(i32, %ret0, %ret1, %ret2, %ret3, %ret)
+  ret <16 x i32> %ret
+}
+
+declare <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) nounwind readnone
+define <16 x i32> @__dot4add_u8i8packed(<16 x i32> %a, <16 x i32> %b, <16 x i32> %acc) nounwind readnone alwaysinline {
+  v16tov4(i32, %a, %a0, %a1, %a2, %a3)
+  v16tov4(i32, %b, %b0, %b1, %b2, %b3)
+  v16tov4(i32, %acc, %acc0, %acc1, %acc2, %acc3)
+  %a0_cast = bitcast <4 x i32> %a0 to <16 x i8>
+  %b0_cast = bitcast <4 x i32> %b0 to <16 x i8>
+  %a1_cast = bitcast <4 x i32> %a1 to <16 x i8>
+  %b1_cast = bitcast <4 x i32> %b1 to <16 x i8>
+  %a2_cast = bitcast <4 x i32> %a2 to <16 x i8>
+  %b2_cast = bitcast <4 x i32> %b2 to <16 x i8>
+  %a3_cast = bitcast <4 x i32> %a3 to <16 x i8>
+  %b3_cast = bitcast <4 x i32> %b3 to <16 x i8>
+  %ret0 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc0, <16 x i8> %a0_cast, <16 x i8> %b0_cast)
+  %ret1 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc1, <16 x i8> %a1_cast, <16 x i8> %b1_cast)
+  %ret2 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc2, <16 x i8> %a2_cast, <16 x i8> %b2_cast)
+  %ret3 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc3, <16 x i8> %a3_cast, <16 x i8> %b3_cast)
+  v4tov16(i32, %ret0, %ret1, %ret2, %ret3, %ret)
+  ret <16 x i32> %ret
+}
diff --git a/builtins/target-neon-i8x32.ll b/builtins/target-neon-i8x32.ll
@@ -4,4 +4,158 @@
 
 define(`WIDTH',`32')
 define(`MASK',`i8')
-define(`ISA',`NEON')
+define(`ISA',`NEON')
+
+include(`util.m4')
+
+define(`NEON_PREFIX',
+`ifelse(RUNTIME, `64', `llvm.aarch64.neon',
+        RUNTIME, `32', `llvm.arm.neon')')
+
+define(`NEON_PREFIX_UDOT',
+`ifelse(RUNTIME, `64', `llvm.aarch64.neon.udot',
+        RUNTIME, `32', `llvm.arm.neon.udot')')
+
+define(`NEON_PREFIX_SDOT',
+`ifelse(RUNTIME, `64', `llvm.aarch64.neon.sdot',
+        RUNTIME, `32', `llvm.arm.neon.sdot')')
+
+define(`NEON_PREFIX_USDOT',
+`ifelse(RUNTIME, `64', `llvm.aarch64.neon.usdot',
+        RUNTIME, `32', `llvm.arm.neon.usdot')')
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; half conversion routines
+
+declare <4 x i16> @NEON_PREFIX.vcvtfp2hf(<4 x float>) nounwind readnone
+declare <4 x float> @NEON_PREFIX.vcvthf2fp(<4 x i16>) nounwind readnone
+
+define float @__half_to_float_uniform(i16 %v) nounwind readnone alwaysinline {
+  %v1 = bitcast i16 %v to <1 x i16>
+  %vec = shufflevector <1 x i16> %v1, <1 x i16> undef,
+           <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %h = call <4 x float> @NEON_PREFIX.vcvthf2fp(<4 x i16> %vec)
+  %r = extractelement <4 x float> %h, i32 0
+  ret float %r
+}
+
+define i16 @__float_to_half_uniform(float %v) nounwind readnone alwaysinline {
+  %v1 = bitcast float %v to <1 x float>
+  %vec = shufflevector <1 x float> %v1, <1 x float> undef,
+           <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  %h = call <4 x i16> @NEON_PREFIX.vcvtfp2hf(<4 x float> %vec)
+  %r = extractelement <4 x i16> %h, i32 0
+  ret i16 %r
+}
+
+define <32 x float> @__half_to_float_varying(<32 x i16> %v) nounwind readnone alwaysinline {
+  unary4to32conv(r, i16, float, @NEON_PREFIX.vcvthf2fp, %v)
+  ret <32 x float> %r;
+}
+
+define <32 x i16> @__float_to_half_varying(<32 x float> %v) nounwind readnone alwaysinline {
+  unary4to32conv(r, float, i16, @NEON_PREFIX.vcvtfp2hf, %v)
+  ret <32 x i16> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; dot product
+
+declare <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) nounwind readnone
+define <32 x i32> @__dot4add_u8u8packed(<32 x i32> %a, <32 x i32> %b, <32 x i32> %acc) nounwind readnone alwaysinline {
+  v32tov4(i32, %a, %a0, %a1, %a2, %a3, %a4, %a5, %a6, %a7)
+  v32tov4(i32, %b, %b0, %b1, %b2, %b3, %b4, %b5, %b6, %b7)
+  v32tov4(i32, %acc, %acc0, %acc1, %acc2, %acc3, %acc4, %acc5, %acc6, %acc7)
+  %a0_cast = bitcast <4 x i32> %a0 to <16 x i8>
+  %b0_cast = bitcast <4 x i32> %b0 to <16 x i8>
+  %a1_cast = bitcast <4 x i32> %a1 to <16 x i8>
+  %b1_cast = bitcast <4 x i32> %b1 to <16 x i8>
+  %a2_cast = bitcast <4 x i32> %a2 to <16 x i8>
+  %b2_cast = bitcast <4 x i32> %b2 to <16 x i8>
+  %a3_cast = bitcast <4 x i32> %a3 to <16 x i8>
+  %b3_cast = bitcast <4 x i32> %b3 to <16 x i8>
+  %a4_cast = bitcast <4 x i32> %a4 to <16 x i8>
+  %b4_cast = bitcast <4 x i32> %b4 to <16 x i8>
+  %a5_cast = bitcast <4 x i32> %a5 to <16 x i8>
+  %b5_cast = bitcast <4 x i32> %b5 to <16 x i8>
+  %a6_cast = bitcast <4 x i32> %a6 to <16 x i8>
+  %b6_cast = bitcast <4 x i32> %b6 to <16 x i8>
+  %a7_cast = bitcast <4 x i32> %a7 to <16 x i8>
+  %b7_cast = bitcast <4 x i32> %b7 to <16 x i8>
+  %ret0 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc0, <16 x i8> %a0_cast, <16 x i8> %b0_cast)
+  %ret1 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc1, <16 x i8> %a1_cast, <16 x i8> %b1_cast)
+  %ret2 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc2, <16 x i8> %a2_cast, <16 x i8> %b2_cast)
+  %ret3 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc3, <16 x i8> %a3_cast, <16 x i8> %b3_cast)
+  %ret4 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc4, <16 x i8> %a4_cast, <16 x i8> %b4_cast)
+  %ret5 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc5, <16 x i8> %a5_cast, <16 x i8> %b5_cast)
+  %ret6 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc6, <16 x i8> %a6_cast, <16 x i8> %b6_cast)
+  %ret7 = call <4 x i32> @NEON_PREFIX_UDOT.v4i32.v16i8(<4 x i32> %acc7, <16 x i8> %a7_cast, <16 x i8> %b7_cast)
+  v4tov32(i32, %ret0, %ret1, %ret2, %ret3, %ret4, %ret5, %ret6, %ret7, %ret)
+  ret <32 x i32> %ret
+}
+
+declare <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) nounwind readnone
+define <32 x i32> @__dot4add_i8i8packed(<32 x i32> %a, <32 x i32> %b, <32 x i32> %acc) nounwind readnone alwaysinline {
+  v32tov4(i32, %a, %a0, %a1, %a2, %a3, %a4, %a5, %a6, %a7)
+  v32tov4(i32, %b, %b0, %b1, %b2, %b3, %b4, %b5, %b6, %b7)
+  v32tov4(i32, %acc, %acc0, %acc1, %acc2, %acc3, %acc4, %acc5, %acc6, %acc7)
+  %a0_cast = bitcast <4 x i32> %a0 to <16 x i8>
+  %b0_cast = bitcast <4 x i32> %b0 to <16 x i8>
+  %a1_cast = bitcast <4 x i32> %a1 to <16 x i8>
+  %b1_cast = bitcast <4 x i32> %b1 to <16 x i8>
+  %a2_cast = bitcast <4 x i32> %a2 to <16 x i8>
+  %b2_cast = bitcast <4 x i32> %b2 to <16 x i8>
+  %a3_cast = bitcast <4 x i32> %a3 to <16 x i8>
+  %b3_cast = bitcast <4 x i32> %b3 to <16 x i8>
+  %a4_cast = bitcast <4 x i32> %a4 to <16 x i8>
+  %b4_cast = bitcast <4 x i32> %b4 to <16 x i8>
+  %a5_cast = bitcast <4 x i32> %a5 to <16 x i8>
+  %b5_cast = bitcast <4 x i32> %b5 to <16 x i8>
+  %a6_cast = bitcast <4 x i32> %a6 to <16 x i8>
+  %b6_cast = bitcast <4 x i32> %b6 to <16 x i8>
+  %a7_cast = bitcast <4 x i32> %a7 to <16 x i8>
+  %b7_cast = bitcast <4 x i32> %b7 to <16 x i8>
+  %ret0 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc0, <16 x i8> %a0_cast, <16 x i8> %b0_cast)
+  %ret1 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc1, <16 x i8> %a1_cast, <16 x i8> %b1_cast)
+  %ret2 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc2, <16 x i8> %a2_cast, <16 x i8> %b2_cast)
+  %ret3 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc3, <16 x i8> %a3_cast, <16 x i8> %b3_cast)
+  %ret4 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc4, <16 x i8> %a4_cast, <16 x i8> %b4_cast)
+  %ret5 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc5, <16 x i8> %a5_cast, <16 x i8> %b5_cast)
+  %ret6 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc6, <16 x i8> %a6_cast, <16 x i8> %b6_cast)
+  %ret7 = call <4 x i32> @NEON_PREFIX_SDOT.v4i32.v16i8(<4 x i32> %acc7, <16 x i8> %a7_cast, <16 x i8> %b7_cast)
+  v4tov32(i32, %ret0, %ret1, %ret2, %ret3, %ret4, %ret5, %ret6, %ret7, %ret)
+  ret <32 x i32> %ret
+}
+
+declare <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) nounwind readnone
+define <32 x i32> @__dot4add_u8i8packed(<32 x i32> %a, <32 x i32> %b, <32 x i32> %acc) nounwind readnone alwaysinline {
+  v32tov4(i32, %a, %a0, %a1, %a2, %a3, %a4, %a5, %a6, %a7)
+  v32tov4(i32, %b, %b0, %b1, %b2, %b3, %b4, %b5, %b6, %b7)
+  v32tov4(i32, %acc, %acc0, %acc1, %acc2, %acc3, %acc4, %acc5, %acc6, %acc7)
+  %a0_cast = bitcast <4 x i32> %a0 to <16 x i8>
+  %b0_cast = bitcast <4 x i32> %b0 to <16 x i8>
+  %a1_cast = bitcast <4 x i32> %a1 to <16 x i8>
+  %b1_cast = bitcast <4 x i32> %b1 to <16 x i8>
+  %a2_cast = bitcast <4 x i32> %a2 to <16 x i8>
+  %b2_cast = bitcast <4 x i32> %b2 to <16 x i8>
+  %a3_cast = bitcast <4 x i32> %a3 to <16 x i8>
+  %b3_cast = bitcast <4 x i32> %b3 to <16 x i8>
+  %a4_cast = bitcast <4 x i32> %a4 to <16 x i8>
+  %b4_cast = bitcast <4 x i32> %b4 to <16 x i8>
+  %a5_cast = bitcast <4 x i32> %a5 to <16 x i8>
+  %b5_cast = bitcast <4 x i32> %b5 to <16 x i8>
+  %a6_cast = bitcast <4 x i32> %a6 to <16 x i8>
+  %b6_cast = bitcast <4 x i32> %b6 to <16 x i8>
+  %a7_cast = bitcast <4 x i32> %a7 to <16 x i8>
+  %b7_cast = bitcast <4 x i32> %b7 to <16 x i8>
+  %ret0 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc0, <16 x i8> %a0_cast, <16 x i8> %b0_cast)
+  %ret1 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc1, <16 x i8> %a1_cast, <16 x i8> %b1_cast)
+  %ret2 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc2, <16 x i8> %a2_cast, <16 x i8> %b2_cast)
+  %ret3 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc3, <16 x i8> %a3_cast, <16 x i8> %b3_cast)
+  %ret4 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc4, <16 x i8> %a4_cast, <16 x i8> %b4_cast)
+  %ret5 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc5, <16 x i8> %a5_cast, <16 x i8> %b5_cast)
+  %ret6 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc6, <16 x i8> %a6_cast, <16 x i8> %b6_cast)
+  %ret7 = call <4 x i32> @NEON_PREFIX_USDOT.v4i32.v16i8(<4 x i32> %acc7, <16 x i8> %a7_cast, <16 x i8> %b7_cast)
+  v4tov32(i32, %ret0, %ret1, %ret2, %ret3, %ret4, %ret5, %ret6, %ret7, %ret)
+  ret <32 x i32> %ret
+}
diff --git a/builtins/util.m4 b/builtins/util.m4
@@ -1170,6 +1170,47 @@ define(`unary4to16conv', `
 '
 )
 
+define(`unary4to32conv', `
+  %$1_0 = shufflevector <32 x $2> $5, <32 x $2> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v$1_0 = call <4 x $3> $4(<4 x $2> %$1_0)
+  %$1_1 = shufflevector <32 x $2> $5, <32 x $2> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %v$1_1 = call <4 x $3> $4(<4 x $2> %$1_1)
+  %$1_2 = shufflevector <32 x $2> $5, <32 x $2> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  %v$1_2 = call <4 x $3> $4(<4 x $2> %$1_2)
+  %$1_3 = shufflevector <32 x $2> $5, <32 x $2> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %v$1_3 = call <4 x $3> $4(<4 x $2> %$1_3)
+  %$1_4 = shufflevector <32 x $2> $5, <32 x $2> undef, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+  %v$1_4 = call <4 x $3> $4(<4 x $2> %$1_4)
+  %$1_5 = shufflevector <32 x $2> $5, <32 x $2> undef, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+  %v$1_5 = call <4 x $3> $4(<4 x $2> %$1_5)
+  %$1_6 = shufflevector <32 x $2> $5, <32 x $2> undef, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+  %v$1_6 = call <4 x $3> $4(<4 x $2> %$1_6)
+  %$1_7 = shufflevector <32 x $2> $5, <32 x $2> undef, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+  %v$1_7 = call <4 x $3> $4(<4 x $2> %$1_7)
+
+  %$1a = shufflevector <4 x $3> %v$1_0, <4 x $3> %v$1_1,
+           <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %$1b = shufflevector <4 x $3> %v$1_2, <4 x $3> %v$1_3,
+           <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %$1c = shufflevector <4 x $3> %v$1_4, <4 x $3> %v$1_5,
+           <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %$1d = shufflevector <4 x $3> %v$1_6, <4 x $3> %v$1_7,
+           <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+
+  %$1e = shufflevector <8 x $3> %$1a, <8 x $3> %$1b,
+           <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                       i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %$1f = shufflevector <8 x $3> %$1c, <8 x $3> %$1d,
+           <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                       i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %$1 = shufflevector <16 x $3> %$1e, <16 x $3> %$1f,
+           <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                       i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+                       i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
+                       i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+'
+)
+
 ;; $1: name of variable into which the final result should go
 ;; $2: scalar type of the vector elements
 ;; $3: 4-wide unary vector function to apply
diff --git a/src/ispc.cpp b/src/ispc.cpp
@@ -1743,7 +1743,11 @@ Target::Target(Arch arch, const char *cpu, ISPCTarget ispc_target, PICLevel picL
         this->m_dataTypeWidth = 8;
         this->m_vectorWidth = 32;
         this->m_maskingIsFree = (arch == Arch::aarch64);
+        this->m_hasHalfConverts = true;
         this->m_maskBitCount = 8;
+        // TODO: this is a workaround for the bug in GatherCoalescePass for x32 targets.
+        // see issue #3153
+        this->m_hasGather = true;
         break;
     case ISPCTarget::neon_i16x8:
         this->m_isa = Target::NEON;
@@ -1762,6 +1766,7 @@ Target::Target(Arch arch, const char *cpu, ISPCTarget ispc_target, PICLevel picL
         this->m_dataTypeWidth = 16;
         this->m_vectorWidth = 16;
         this->m_maskingIsFree = (arch == Arch::aarch64);
+        this->m_hasHalfConverts = true;
         this->m_maskBitCount = 16;
         break;
     case ISPCTarget::neon_i32x4:
