1;; riscv64 instruction selection and CLIF-to-MachInst lowering. 2 3;; The main lowering constructor term: takes a clif `Inst` and returns the 4;; register(s) within which the lowered instruction's result values live. 5(decl partial lower (Inst) InstOutput) 6 7;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 9(rule (lower (has_type ty (iconst _ (u64_from_imm64 n)))) 10 (imm ty n)) 11 12;; ;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 13 14(rule (lower (has_type (ty_supported_vec ty) (vconst _ n))) 15 (gen_constant ty (const_to_vconst n))) 16 17;;;; Rules for `f16const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 18 19(rule (lower (f16const _ (u16_from_ieee16 n))) 20 (imm $F16 n)) 21 22;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 23 24(rule (lower (f32const _ (u32_from_ieee32 n))) 25 (imm $F32 n)) 26 27;;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 28 29(rule (lower (f64const _ (u64_from_ieee64 n))) 30 (imm $F64 n)) 31 32;;;; Rules for `f128const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 33 34(rule (lower (f128const _ (u128_from_constant n))) 35 (value_regs (imm $I64 (u128_low_bits n)) (imm $I64 (u128_high_bits n)))) 36 37(rule 1 (lower (f128const _ (u128_from_constant (u128_replicated_u64 n)))) 38 (let ((r Reg (imm $I64 n))) 39 (value_regs r r))) 40 41;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 42 43;; Base case, simply adding things in registers. 44(rule -1 (lower (has_type (fits_in_32 (ty_int ty)) (iadd _ x y))) 45 (rv_addw x y)) 46 47(rule 0 (lower (has_type $I64 (iadd _ x y))) 48 (rv_add x y)) 49 50;; Special cases for when one operand is an immediate that fits in 12 bits. 51(rule 1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (imm12_from_value y)))) 52 (alu_rr_imm12 (select_addi ty) x y)) 53 54(rule 2 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (imm12_from_value x) y))) 55 (alu_rr_imm12 (select_addi ty) y x)) 56 57;; Special case when one of the operands is uextended 58;; Needs `Zba` 59(rule 3 (lower (has_type $I64 (iadd _ x (uextend _ y @ (value_type $I32))))) 60 (if-let true (has_zba)) 61 (rv_adduw y x)) 62 63(rule 4 (lower (has_type $I64 (iadd _ (uextend _ x @ (value_type $I32)) y))) 64 (if-let true (has_zba)) 65 (rv_adduw x y)) 66 67;; Add with const shift. We have a few of these instructions with `Zba`. 68(decl pure partial match_shnadd (Imm64) AluOPRRR) 69(rule (match_shnadd (u64_from_imm64 1)) (AluOPRRR.Sh1add)) 70(rule (match_shnadd (u64_from_imm64 2)) (AluOPRRR.Sh2add)) 71(rule (match_shnadd (u64_from_imm64 3)) (AluOPRRR.Sh3add)) 72 73(rule 3 (lower (has_type $I64 (iadd _ x (ishl _ y (maybe_uextend (iconst _ n)))))) 74 (if-let true (has_zba)) 75 (if-let shnadd (match_shnadd n)) 76 (alu_rrr shnadd y x)) 77 78(rule 4 (lower (has_type $I64 (iadd _ (ishl _ x (maybe_uextend (iconst _ n))) y))) 79 (if-let true (has_zba)) 80 (if-let shnadd (match_shnadd n)) 81 (alu_rrr shnadd x y)) 82 83 84;; Add with uextended const shift. We have a few of these instructions with `Zba`. 85;; 86;; !!! Important !!! 87;; These rules only work for (ishl (uextend _) _) and not for (uextend (ishl _ _))! 88;; Getting this wrong means a potential misscalculation of the shift amount. 89;; Additionally we can only ensure that this is correct if the uextend is 32 to 64 bits. 90(decl pure partial match_shnadd_uw (Imm64) AluOPRRR) 91(rule (match_shnadd_uw (u64_from_imm64 1)) (AluOPRRR.Sh1adduw)) 92(rule (match_shnadd_uw (u64_from_imm64 2)) (AluOPRRR.Sh2adduw)) 93(rule (match_shnadd_uw (u64_from_imm64 3)) (AluOPRRR.Sh3adduw)) 94 95(rule 5 (lower (has_type $I64 (iadd _ x (ishl _ (uextend _ y @ (value_type $I32)) (maybe_uextend (iconst _ n)))))) 96 (if-let true (has_zba)) 97 (if-let shnadd_uw (match_shnadd_uw n)) 98 (alu_rrr shnadd_uw y x)) 99 100(rule 6 (lower (has_type $I64 (iadd _ (ishl _ (uextend _ x @ (value_type $I32)) (maybe_uextend (iconst _ n))) y))) 101 (if-let true (has_zba)) 102 (if-let shnadd_uw (match_shnadd_uw n)) 103 (alu_rrr shnadd_uw x y)) 104 105;; I128 cases 106(rule 7 (lower (has_type $I128 (iadd _ x y))) 107 (let ((low XReg (rv_add (value_regs_get x 0) (value_regs_get y 0))) 108 ;; compute carry. 109 (carry XReg (rv_sltu low (value_regs_get y 0))) 110 ;; 111 (high_tmp XReg (rv_add (value_regs_get x 1) (value_regs_get y 1))) 112 ;; add carry. 113 (high XReg (rv_add high_tmp carry))) 114 (value_regs low high))) 115 116;; SIMD Vectors 117(rule 8 (lower (has_type (ty_supported_vec ty) (iadd _ x y))) 118 (rv_vadd_vv x y (unmasked) ty)) 119 120(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ y)))) 121 (rv_vadd_vx x y (unmasked) ty)) 122 123(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ (sextend _ y @ (value_type sext_ty)))))) 124 (if-let half_ty (ty_half_width ty)) 125 (if-let true (ty_equal (lane_type half_ty) sext_ty)) 126 (rv_vwadd_wx x y (unmasked) (vstate_mf2 half_ty))) 127 128(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ (uextend _ y @ (value_type uext_ty)))))) 129 (if-let half_ty (ty_half_width ty)) 130 (if-let true (ty_equal (lane_type half_ty) uext_ty)) 131 (rv_vwaddu_wx x y (unmasked) (vstate_mf2 half_ty))) 132 133(rule 20 (lower (has_type (ty_supported_vec ty) (iadd _ x y))) 134 (if-let y_imm (replicated_imm5 y)) 135 (rv_vadd_vi x y_imm (unmasked) ty)) 136 137 138(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ x) y))) 139 (rv_vadd_vx y x (unmasked) ty)) 140 141(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty))) y))) 142 (if-let half_ty (ty_half_width ty)) 143 (if-let true (ty_equal (lane_type half_ty) sext_ty)) 144 (rv_vwadd_wx y x (unmasked) (vstate_mf2 half_ty))) 145 146(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ (uextend _ x @ (value_type uext_ty))) y))) 147 (if-let half_ty (ty_half_width ty)) 148 (if-let true (ty_equal (lane_type half_ty) uext_ty)) 149 (rv_vwaddu_wx y x (unmasked) (vstate_mf2 half_ty))) 150 151(rule 21 (lower (has_type (ty_supported_vec ty) (iadd _ x y))) 152 (if-let x_imm (replicated_imm5 x)) 153 (rv_vadd_vi y x_imm (unmasked) ty)) 154 155;; Signed Widening Low Additions 156 157(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (swiden_low _ y @ (value_type in_ty))))) 158 (rv_vwadd_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 159 160(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty)) y))) 161 (rv_vwadd_wv y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 162 163(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty)) 164 (swiden_low _ y)))) 165 (rv_vwadd_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 166 167(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty)) 168 (splat _ (sextend _ y @ (value_type sext_ty)))))) 169 (if-let true (ty_equal (lane_type in_ty) sext_ty)) 170 (rv_vwadd_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 171 172(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty))) 173 (swiden_low _ y @ (value_type in_ty))))) 174 (if-let true (ty_equal (lane_type in_ty) sext_ty)) 175 (rv_vwadd_vx y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 176 177;; Signed Widening High Additions 178;; These are the same as the low additions, but we first slide down the inputs. 179 180(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (swiden_high _ y @ (value_type in_ty))))) 181 (rv_vwadd_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 182 183(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty)) y))) 184 (rv_vwadd_wv y (gen_slidedown_half in_ty x) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 185 186(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty)) 187 (swiden_high _ y)))) 188 (rv_vwadd_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 189 190(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty)) 191 (splat _ (sextend _ y @ (value_type sext_ty)))))) 192 (if-let true (ty_equal (lane_type in_ty) sext_ty)) 193 (rv_vwadd_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 194 195(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty))) 196 (swiden_high _ y @ (value_type in_ty))))) 197 (if-let true (ty_equal (lane_type in_ty) sext_ty)) 198 (rv_vwadd_vx (gen_slidedown_half in_ty y) x (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 199 200;; Unsigned Widening Low Additions 201 202(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (uwiden_low _ y @ (value_type in_ty))))) 203 (rv_vwaddu_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 204 205(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty)) y))) 206 (rv_vwaddu_wv y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 207 208(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty)) 209 (uwiden_low _ y)))) 210 (rv_vwaddu_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 211 212(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty)) 213 (splat _ (uextend _ y @ (value_type uext_ty)))))) 214 (if-let true (ty_equal (lane_type in_ty) uext_ty)) 215 (rv_vwaddu_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 216 217(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (uextend _ x @ (value_type uext_ty))) 218 (uwiden_low _ y @ (value_type in_ty))))) 219 (if-let true (ty_equal (lane_type in_ty) uext_ty)) 220 (rv_vwaddu_vx y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 221 222;; Unsigned Widening High Additions 223;; These are the same as the low additions, but we first slide down the inputs. 224 225(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (uwiden_high _ y @ (value_type in_ty))))) 226 (rv_vwaddu_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 227 228(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty)) y))) 229 (rv_vwaddu_wv y (gen_slidedown_half in_ty x) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 230 231(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty)) 232 (uwiden_high _ y)))) 233 (rv_vwaddu_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 234 235(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty)) 236 (splat _ (uextend _ y @ (value_type uext_ty)))))) 237 (if-let true (ty_equal (lane_type in_ty) uext_ty)) 238 (rv_vwaddu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 239 240(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (uextend _ y @ (value_type uext_ty))) 241 (uwiden_high _ x @ (value_type in_ty))))) 242 (if-let true (ty_equal (lane_type in_ty) uext_ty)) 243 (rv_vwaddu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 244 245;; Signed Widening Mixed High/Low Additions 246 247(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty)) 248 (swiden_high _ y)))) 249 (rv_vwadd_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 250 251(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty)) 252 (swiden_low _ y)))) 253 (rv_vwadd_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 254 255;; Unsigned Widening Mixed High/Low Additions 256 257(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty)) 258 (uwiden_high _ y)))) 259 (rv_vwaddu_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 260 261(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty)) 262 (uwiden_low _ y)))) 263 (rv_vwaddu_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 264 265;; Fused Multiply Accumulate Rules `vmacc` 266;; 267;; I dont think we can use `vmadd`/`vmnsub` here since it just modifies the multiplication 268;; register instead of the addition one. The actual pattern matched seems to be 269;; exactly the same. 270 271(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ y z)))) 272 (rv_vmacc_vv x y z (unmasked) ty)) 273 274(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ y (splat _ z))))) 275 (rv_vmacc_vx x y z (unmasked) ty)) 276 277(rule 11 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ (splat _ y) z)))) 278 (rv_vmacc_vx x z y (unmasked) ty)) 279 280(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ x y) z))) 281 (rv_vmacc_vv z x y (unmasked) ty)) 282 283(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ x (splat _ y)) z))) 284 (rv_vmacc_vx z x y (unmasked) ty)) 285 286(rule 14 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ (splat _ x) y) z))) 287 (rv_vmacc_vx z y x (unmasked) ty)) 288 289;; Fused Multiply Subtract Rules `vnmsac` 290 291(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ y z))))) 292 (rv_vnmsac_vv x y z (unmasked) ty)) 293 294(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ y (splat _ z)))))) 295 (rv_vnmsac_vx x y z (unmasked) ty)) 296 297(rule 11 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ (splat _ y) z))))) 298 (rv_vnmsac_vx x z y (unmasked) ty)) 299 300(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ x y)) z))) 301 (rv_vnmsac_vv z x y (unmasked) ty)) 302 303(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ x (splat _ y))) z))) 304 (rv_vnmsac_vx z x y (unmasked) ty)) 305 306(rule 14 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ (splat _ x) y)) z))) 307 (rv_vnmsac_vx z y x (unmasked) ty)) 308 309;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;; 310(rule 0 (lower (has_type (fits_in_32 ty) (uadd_overflow_trap _ x y tc))) 311 (let ((tmp_x XReg (zext x)) 312 (tmp_y XReg (zext y)) 313 (sum XReg (rv_add tmp_x tmp_y)) 314 (test XReg (rv_srli sum (imm12_const (ty_bits ty)))) 315 (_ InstOutput (gen_trapnz test tc))) 316 sum)) 317 318(rule 1 (lower (has_type $I64 (uadd_overflow_trap _ x y tc))) 319 (let ((tmp XReg (rv_add x y)) 320 (_ InstOutput (gen_trapif (IntCC.UnsignedLessThan) tmp x tc))) 321 tmp)) 322 323;;;; Rules for uadd_overflow ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 324 325;; For i64, we can use the fact that if a + b < a, then overflow occurred 326(rule 0 (lower (has_type $I64 (uadd_overflow _ x y))) 327 (let ((sum XReg (rv_add x y)) 328 (overflow XReg (rv_sltu sum x))) 329 (output_pair sum overflow))) 330 331;; i32 case (on RV64 use addw to detect 32-bit overflow correctly) 332(rule 1 (lower (has_type $I32 (uadd_overflow _ x y))) 333 (let ((x64 XReg (zext x)) 334 (y64 XReg (zext y)) 335 (sum XReg (rv_add x64 y64)) 336 (overflow XReg (rv_srli sum (imm12_const 32)))) 337 (output_pair sum overflow))) 338 339;; For i128, we need to handle the high and low parts separately 340(rule 2 (lower (has_type $I128 (uadd_overflow _ x y))) 341 (let ((x_regs ValueRegs x) 342 (y_regs ValueRegs y) 343 (x_lo XReg (value_regs_get x_regs 0)) 344 (x_hi XReg (value_regs_get x_regs 1)) 345 (y_lo XReg (value_regs_get y_regs 0)) 346 (y_hi XReg (value_regs_get y_regs 1)) 347 (sum_lo XReg (rv_add x_lo y_lo)) 348 (carry XReg (rv_sltu sum_lo x_lo)) 349 (sum_hi XReg (rv_add x_hi y_hi)) 350 (sum_hi_with_carry XReg (rv_add sum_hi carry)) 351 (overflow XReg (rv_or (rv_sltu sum_hi_with_carry x_hi) 352 (rv_and carry (rv_seqz (rv_xor sum_hi_with_carry x_hi)))))) 353 (output_pair (value_regs sum_lo sum_hi_with_carry) overflow))) 354 355;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 356;; Base case, simply subtracting things in registers. 357 358(rule 0 (lower (has_type (fits_in_32 (ty_int ty)) (isub _ x y))) 359 (rv_subw x y)) 360 361(rule 1 (lower (has_type $I64 (isub _ x y))) 362 (rv_sub x y)) 363 364(rule 2 (lower (has_type $I128 (isub _ x y))) 365 (sub_i128 x y)) 366 367;; Switch to an `addi` by a negative if we can fit the value in an `imm12`. 368(rule 3 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x y))) 369 (if-let imm12_neg (imm12_from_negated_value y)) 370 (alu_rr_imm12 (select_addi ty) x imm12_neg)) 371 372;; SIMD Vectors 373(rule 4 (lower (has_type (ty_supported_vec ty) (isub _ x y))) 374 (rv_vsub_vv x y (unmasked) ty)) 375 376(rule 5 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ y)))) 377 (rv_vsub_vx x y (unmasked) ty)) 378 379(rule 6 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ (sextend _ y @ (value_type sext_ty)))))) 380 (if-let half_ty (ty_half_width ty)) 381 (if-let true (ty_equal (lane_type half_ty) sext_ty)) 382 (rv_vwsub_wx x y (unmasked) (vstate_mf2 half_ty))) 383 384(rule 6 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ (uextend _ y @ (value_type uext_ty)))))) 385 (if-let half_ty (ty_half_width ty)) 386 (if-let true (ty_equal (lane_type half_ty) uext_ty)) 387 (rv_vwsubu_wx x y (unmasked) (vstate_mf2 half_ty))) 388 389(rule 7 (lower (has_type (ty_supported_vec ty) (isub _ (splat _ x) y))) 390 (rv_vrsub_vx y x (unmasked) ty)) 391 392(rule 8 (lower (has_type (ty_supported_vec ty) (isub _ x y))) 393 (if-let imm5_neg (negated_replicated_imm5 y)) 394 (rv_vadd_vi x imm5_neg (unmasked) ty)) 395 396(rule 9 (lower (has_type (ty_supported_vec ty) (isub _ x y))) 397 (if-let x_imm (replicated_imm5 x)) 398 (rv_vrsub_vi y x_imm (unmasked) ty)) 399 400 401;; Signed Widening Low Subtractions 402 403(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (swiden_low _ y @ (value_type in_ty))))) 404 (rv_vwsub_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 405 406(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty)) 407 (swiden_low _ y)))) 408 (rv_vwsub_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 409 410(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty)) 411 (splat _ (sextend _ y @ (value_type sext_ty)))))) 412 (if-let true (ty_equal (lane_type in_ty) sext_ty)) 413 (rv_vwsub_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 414 415;; Signed Widening High Subtractions 416;; These are the same as the low widenings, but we first slide down the inputs. 417 418(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (swiden_high _ y @ (value_type in_ty))))) 419 (rv_vwsub_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 420 421(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty)) 422 (swiden_high _ y)))) 423 (rv_vwsub_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 424 425(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty)) 426 (splat _ (sextend _ y @ (value_type sext_ty)))))) 427 (if-let true (ty_equal (lane_type in_ty) sext_ty)) 428 (rv_vwsub_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 429 430;; Unsigned Widening Low Subtractions 431 432(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (uwiden_low _ y @ (value_type in_ty))))) 433 (rv_vwsubu_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 434 435(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty)) 436 (uwiden_low _ y)))) 437 (rv_vwsubu_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 438 439(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty)) 440 (splat _ (uextend _ y @ (value_type uext_ty)))))) 441 (if-let true (ty_equal (lane_type in_ty) uext_ty)) 442 (rv_vwsubu_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 443 444;; Unsigned Widening High Subtractions 445;; These are the same as the low widenings, but we first slide down the inputs. 446 447(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (uwiden_high _ y @ (value_type in_ty))))) 448 (rv_vwsubu_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 449 450(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty)) 451 (uwiden_high _ y)))) 452 (rv_vwsubu_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 453 454(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty)) 455 (splat _ (uextend _ y @ (value_type uext_ty)))))) 456 (if-let true (ty_equal (lane_type in_ty) uext_ty)) 457 (rv_vwsubu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 458 459;; Signed Widening Mixed High/Low Subtractions 460 461(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty)) 462 (swiden_high _ y)))) 463 (rv_vwsub_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 464 465(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty)) 466 (swiden_low _ y)))) 467 (rv_vwsub_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 468 469;; Unsigned Widening Mixed High/Low Subtractions 470 471(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty)) 472 (uwiden_high _ y)))) 473 (rv_vwsubu_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 474 475(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty)) 476 (uwiden_low _ y)))) 477 (rv_vwsubu_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty)))) 478 479 480;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 481 482(rule (lower (has_type (ty_int ty) (ineg _ val))) 483 (neg ty val)) 484 485(rule 1 (lower (has_type (ty_supported_vec ty) (ineg _ x))) 486 (rv_vneg_v x (unmasked) ty)) 487 488 489;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 490 491(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (imul _ x y))) 492 (rv_mul x y)) 493 494(rule 1 (lower (has_type (fits_in_32 (ty_int ty)) (imul _ x y))) 495 (rv_mulw x y)) 496 497;; for I128 498(rule 2 (lower (has_type $I128 (imul _ x y))) 499 (let 500 ((x_regs ValueRegs x) 501 (x_lo XReg (value_regs_get x_regs 0)) 502 (x_hi XReg (value_regs_get x_regs 1)) 503 504 ;; Get the high/low registers for `y`. 505 (y_regs ValueRegs y) 506 (y_lo XReg (value_regs_get y_regs 0)) 507 (y_hi XReg (value_regs_get y_regs 1)) 508 509 ;; 128bit mul formula: 510 ;; dst_lo = x_lo * y_lo 511 ;; dst_hi = mulhu(x_lo, y_lo) + (x_lo * y_hi) + (x_hi * y_lo) 512 ;; 513 ;; We can convert the above formula into the following 514 ;; mulhu dst_hi, x_lo, y_lo 515 ;; madd dst_hi, x_lo, y_hi, dst_hi 516 ;; madd dst_hi, x_hi, y_lo, dst_hi 517 ;; madd dst_lo, x_lo, y_lo, zero 518 (dst_hi1 XReg (rv_mulhu x_lo y_lo)) 519 (dst_hi2 XReg (madd x_lo y_hi dst_hi1)) 520 (dst_hi XReg (madd x_hi y_lo dst_hi2)) 521 (dst_lo XReg (madd x_lo y_lo (zero_reg)))) 522 (value_regs dst_lo dst_hi))) 523 524;; Special case 128-bit multiplication where the operands are extended since 525;; that maps directly to the `mulhu` and `mulh` instructions. 526(rule 6 (lower (has_type $I128 (imul _ (uextend _ x) (uextend _ y)))) 527 (let ((x XReg (zext x)) 528 (y XReg (zext y))) 529 (value_regs (rv_mul x y) (rv_mulhu x y)))) 530 531(rule 6 (lower (has_type $I128 (imul _ (sextend _ x) (sextend _ y)))) 532 (let ((x XReg (sext x)) 533 (y XReg (sext y))) 534 (value_regs (rv_mul x y) (rv_mulh x y)))) 535 536;; Vector multiplication 537 538(rule 3 (lower (has_type (ty_supported_vec ty) (imul _ x y))) 539 (rv_vmul_vv x y (unmasked) ty)) 540 541(rule 4 (lower (has_type (ty_supported_vec ty) (imul _ (splat _ x) y))) 542 (rv_vmul_vx y x (unmasked) ty)) 543 544(rule 5 (lower (has_type (ty_supported_vec ty) (imul _ x (splat _ y)))) 545 (rv_vmul_vx x y (unmasked) ty)) 546 547;;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 548(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (smulhi _ x y))) 549 (lower_smlhi ty (sext x) (sext y))) 550 551(rule 1 (lower (has_type (ty_supported_vec ty) (smulhi _ x y))) 552 (rv_vmulh_vv x y (unmasked) ty)) 553 554(rule 2 (lower (has_type (ty_supported_vec ty) (smulhi _ (splat _ x) y))) 555 (rv_vmulh_vx y x (unmasked) ty)) 556 557(rule 3 (lower (has_type (ty_supported_vec ty) (smulhi _ x (splat _ y)))) 558 (rv_vmulh_vx x y (unmasked) ty)) 559 560;;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 561(rule 0 (lower (has_type (fits_in_32 ty) (umulhi _ x y))) 562 (let ((tmp XReg (rv_mul (zext x) (zext y)))) 563 (rv_srli tmp (imm12_const (ty_bits ty))))) 564 565(rule 1 (lower (has_type $I64 (umulhi _ x y))) 566 (rv_mulhu x y)) 567 568(rule 2 (lower (has_type (ty_supported_vec ty) (umulhi _ x y))) 569 (rv_vmulhu_vv x y (unmasked) ty)) 570 571(rule 3 (lower (has_type (ty_supported_vec ty) (umulhi _ (splat _ x) y))) 572 (rv_vmulhu_vx y x (unmasked) ty)) 573 574(rule 4 (lower (has_type (ty_supported_vec ty) (umulhi _ x (splat _ y)))) 575 (rv_vmulhu_vx x y (unmasked) ty)) 576 577;;;; Rules for `udiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 578 579(rule 0 (lower (has_type (fits_in_16 ty) (udiv _ x y))) 580 (if-let true (has_m)) 581 (rv_divuw (zext x) (nonzero_divisor (zext y)))) 582 583(rule 1 (lower (has_type (fits_in_16 ty) (udiv _ x y @ (iconst _ imm)))) 584 (if-let true (has_m)) 585 (if (safe_divisor_from_imm64 ty imm)) 586 (rv_divuw (zext x) (zext y))) 587 588(rule 2 (lower (has_type $I32 (udiv _ x y))) 589 (if-let true (has_m)) 590 (rv_divuw x (nonzero_divisor (zext y)))) 591 592(rule 3 (lower (has_type $I32 (udiv _ x y @ (iconst _ imm)))) 593 (if-let true (has_m)) 594 (if (safe_divisor_from_imm64 $I32 imm)) 595 (rv_divuw x y)) 596 597(rule 2 (lower (has_type $I64 (udiv _ x y))) 598 (if-let true (has_m)) 599 (rv_divu x (nonzero_divisor y))) 600 601(rule 3 (lower (has_type $I64 (udiv _ x y @ (iconst _ imm)))) 602 (if-let true (has_m)) 603 (if (safe_divisor_from_imm64 $I64 imm)) 604 (rv_divu x y)) 605 606;; Traps if the input register is zero, otherwise returns the same register. 607(decl nonzero_divisor (XReg) XReg) 608(rule (nonzero_divisor val) 609 (let ((_ InstOutput (gen_trapif (IntCC.Equal) val (zero_reg) (TrapCode.INTEGER_DIVISION_BY_ZERO)))) 610 val)) 611 612;;;; Rules for `sdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 613 614(rule 0 (lower (has_type (fits_in_16 ty) (sdiv _ x y))) 615 (if-let true (has_m)) 616 (let ((x XReg (sext x))) 617 (rv_divw x (safe_sdiv_divisor ty x (sext y))))) 618 619(rule 1 (lower (has_type (fits_in_16 ty) (sdiv _ x y @ (iconst _ imm)))) 620 (if-let true (has_m)) 621 (if (safe_divisor_from_imm64 ty imm)) 622 (rv_divw (sext x) (sext y))) 623 624(rule 2 (lower (has_type $I32 (sdiv _ x y))) 625 (if-let true (has_m)) 626 (let ((x XReg (sext x))) 627 (rv_divw x (safe_sdiv_divisor $I32 x (sext y))))) 628 629(rule 3 (lower (has_type $I32 (sdiv _ x y @ (iconst _ imm)))) 630 (if-let true (has_m)) 631 (if (safe_divisor_from_imm64 $I32 imm)) 632 (rv_divw x y)) 633 634(rule 2 (lower (has_type $I64 (sdiv _ x y))) 635 (if-let true (has_m)) 636 (rv_div x (safe_sdiv_divisor $I64 x y))) 637 638(rule 3 (lower (has_type $I64 (sdiv _ x y @ (iconst _ imm)))) 639 (if-let true (has_m)) 640 (if (safe_divisor_from_imm64 $I64 imm)) 641 (rv_div x y)) 642 643;; Check for two trapping conditions: 644;; 645;; * the divisor is 0, or... 646;; * the divisor is -1 and the dividend is $ty::MIN 647(decl safe_sdiv_divisor (Type XReg XReg) XReg) 648(rule (safe_sdiv_divisor ty x y) 649 (let ( 650 (y XReg (nonzero_divisor y)) 651 (min XReg (imm $I64 (u64_wrapping_shl 0xffffffff_ffffffff 652 (u32_wrapping_sub (ty_bits ty) 1)))) 653 (x_is_not_min XReg (rv_xor x min)) 654 (y_is_not_neg_one XReg (rv_not y)) 655 (no_int_overflow XReg (rv_or x_is_not_min y_is_not_neg_one)) 656 (_ InstOutput (gen_trapif 657 (IntCC.Equal) 658 no_int_overflow (zero_reg) 659 (TrapCode.INTEGER_OVERFLOW)))) 660 y)) 661 662;;;; Rules for `urem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 663 664(rule 0 (lower (has_type (fits_in_16 ty) (urem _ x y))) 665 (if-let true (has_m)) 666 (rv_remuw (zext x) (nonzero_divisor (zext y)))) 667 668(rule 1 (lower (has_type (fits_in_16 ty) (urem _ x y @ (iconst _ imm)))) 669 (if-let true (has_m)) 670 (if (safe_divisor_from_imm64 ty imm)) 671 (rv_remuw (zext x) (zext y))) 672 673(rule 2 (lower (has_type $I32 (urem _ x y))) 674 (if-let true (has_m)) 675 (rv_remuw x (nonzero_divisor (zext y)))) 676 677(rule 3 (lower (has_type $I32 (urem _ x y @ (iconst _ imm)))) 678 (if-let true (has_m)) 679 (if (safe_divisor_from_imm64 $I32 imm)) 680 (rv_remuw x y)) 681 682(rule 2 (lower (has_type $I64 (urem _ x y))) 683 (if-let true (has_m)) 684 (rv_remu x (nonzero_divisor y))) 685 686(rule 3 (lower (has_type $I64 (urem _ x y @ (iconst _ imm)))) 687 (if-let true (has_m)) 688 (if (safe_divisor_from_imm64 $I64 imm)) 689 (rv_remu x y)) 690 691;;;; Rules for `srem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 692 693(rule 0 (lower (has_type (fits_in_16 ty) (srem _ x y))) 694 (if-let true (has_m)) 695 (rv_remw (sext x) (nonzero_divisor (sext y)))) 696 697(rule 1 (lower (has_type (fits_in_16 ty) (srem _ x y @ (iconst _ imm)))) 698 (if-let true (has_m)) 699 (if (safe_divisor_from_imm64 ty imm)) 700 (rv_remw (sext x) (sext y))) 701 702(rule 2 (lower (has_type $I32 (srem _ x y))) 703 (if-let true (has_m)) 704 (rv_remw x (nonzero_divisor (sext y)))) 705 706(rule 3 (lower (has_type $I32 (srem _ x y @ (iconst _ imm)))) 707 (if-let true (has_m)) 708 (if (safe_divisor_from_imm64 $I32 imm)) 709 (rv_remw x y)) 710 711(rule 2 (lower (has_type $I64 (srem _ x y))) 712 (if-let true (has_m)) 713 (rv_rem x (nonzero_divisor y))) 714 715(rule 3 (lower (has_type $I64 (srem _ x y @ (iconst _ imm)))) 716 (if-let true (has_m)) 717 (if (safe_divisor_from_imm64 $I64 imm)) 718 (rv_rem x y)) 719 720;;;; Rules for `and` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 721(rule -1 (lower (has_type (fits_in_64 ty) (band _ x y))) 722 (rv_and x y)) 723 724(rule 0 (lower (has_type (ty_reg_pair _) (band _ x y))) 725 (value_regs 726 (rv_and (value_regs_get x 0) (value_regs_get y 0)) 727 (rv_and (value_regs_get x 1) (value_regs_get y 1)))) 728 729;; Special cases for when one operand is an immediate that fits in 12 bits. 730(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x (imm12_from_value y)))) 731 (rv_andi x y)) 732 733(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (band _ (imm12_from_value x) y))) 734 (rv_andi y x)) 735 736(rule 3 (lower (has_type (ty_supported_float_size ty) (band _ x y))) 737 (lower_float_binary (AluOPRRR.And) x y ty)) 738 739;; No need to NaN-box when moving back to the floating point register as the high 740;; bits will already be set. 741(rule 4 (lower (has_type (ty_supported_float_size $F16) (band _ x y))) 742 (if-let false (has_zfhmin)) 743 (lower_float_binary (AluOPRRR.And) x y $F32)) 744 745;; Specialized lowerings for `(band x (bnot y))` which is additionally produced 746;; by Cranelift's `band_not` instruction that is legalized into the simpler 747;; forms early on. 748 749(rule 5 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x (bnot _ y)))) 750 (if-let true (has_zbb)) 751 (rv_andn x y)) 752 753(rule 6 (lower (has_type (fits_in_64 (ty_int ty)) (band _ (bnot _ y) x))) 754 (if-let true (has_zbb)) 755 (rv_andn x y)) 756 757(rule 7 (lower (has_type (ty_reg_pair _) (band _ x (bnot _ y)))) 758 (if-let true (has_zbb)) 759 (let ((low XReg (rv_andn (value_regs_get x 0) (value_regs_get y 0))) 760 (high XReg (rv_andn (value_regs_get x 1) (value_regs_get y 1)))) 761 (value_regs low high))) 762 763(rule 8 (lower (has_type (ty_reg_pair _) (band _ (bnot _ y) x))) 764 (if-let true (has_zbb)) 765 (let ((low XReg (rv_andn (value_regs_get x 0) (value_regs_get y 0))) 766 (high XReg (rv_andn (value_regs_get x 1) (value_regs_get y 1)))) 767 (value_regs low high))) 768 769(rule 9 (lower (has_type (ty_supported_vec ty) (band _ x y))) 770 (rv_vand_vv x y (unmasked) ty)) 771 772(rule 10 (lower (has_type (ty_supported_vec ty) (band _ x (splat _ y)))) 773 (if (ty_vector_not_float ty)) 774 (rv_vand_vx x y (unmasked) ty)) 775 776(rule 11 (lower (has_type (ty_supported_vec ty) (band _ (splat _ x) y))) 777 (if (ty_vector_not_float ty)) 778 (rv_vand_vx y x (unmasked) ty)) 779 780(rule 12 (lower (has_type (ty_supported_vec ty) (band _ x y))) 781 (if-let y_imm (replicated_imm5 y)) 782 (rv_vand_vi x y_imm (unmasked) ty)) 783 784(rule 13 (lower (has_type (ty_supported_vec ty) (band _ x y))) 785 (if-let x_imm (replicated_imm5 x)) 786 (rv_vand_vi y x_imm (unmasked) ty)) 787 788;; `bclr{,i}` specializations from `zbs` 789 790(rule 14 (lower (has_type (fits_in_32 ty) (band _ x (bnot _ (ishl _ (i64_from_iconst 1) y))))) 791 (if-let true (has_zbs)) 792 (rv_bclr x (rv_andi y (imm12_const (u8_wrapping_sub (ty_bits ty) 1))))) 793(rule 15 (lower (has_type (fits_in_32 ty) (band _ (bnot _ (ishl _ (i64_from_iconst 1) y)) x))) 794 (if-let true (has_zbs)) 795 (rv_bclr x (rv_andi y (imm12_const (u8_wrapping_sub (ty_bits ty) 1))))) 796 797(rule 16 (lower (has_type $I64 (band _ x (bnot _ (ishl _ (i64_from_iconst 1) y))))) 798 (if-let true (has_zbs)) 799 (rv_bclr x y)) 800(rule 17 (lower (has_type $I64 (band _ (bnot _ (ishl _ (i64_from_iconst 1) y)) x))) 801 (if-let true (has_zbs)) 802 (rv_bclr x y)) 803 804(rule 18 (lower (has_type (fits_in_64 ty) (band _ x (u64_from_iconst n)))) 805 (if-let true (has_zbs)) 806 (if-let imm (bclr_imm ty n)) 807 (rv_bclri x imm)) 808(rule 19 (lower (has_type (fits_in_64 ty) (band _ (u64_from_iconst n) x))) 809 (if-let true (has_zbs)) 810 (if-let imm (bclr_imm ty n)) 811 (rv_bclri x imm)) 812 813(decl pure partial bclr_imm (Type u64) Imm12) 814(extern constructor bclr_imm bclr_imm) 815 816;; `bext{,i}` specializations from `zbs` 817 818(rule 20 (lower (has_type $I32 (band _ (ushr _ x y) (u64_from_iconst 1)))) 819 (if-let true (has_zbs)) 820 (rv_bext x (rv_andi y (imm12_const 31)))) 821(rule 20 (lower (has_type $I32 (band _ (sshr _ x y) (u64_from_iconst 1)))) 822 (if-let true (has_zbs)) 823 (rv_bext x (rv_andi y (imm12_const 31)))) 824(rule 20 (lower (has_type $I32 (band _ (u64_from_iconst 1) (ushr _ x y)))) 825 (if-let true (has_zbs)) 826 (rv_bext x (rv_andi y (imm12_const 31)))) 827(rule 20 (lower (has_type $I32 (band _ (u64_from_iconst 1) (sshr _ x y)))) 828 (if-let true (has_zbs)) 829 (rv_bext x (rv_andi y (imm12_const 31)))) 830 831(rule 20 (lower (has_type $I64 (band _ (ushr _ x y) (u64_from_iconst 1)))) 832 (if-let true (has_zbs)) 833 (rv_bext x y)) 834(rule 20 (lower (has_type $I64 (band _ (sshr _ x y) (u64_from_iconst 1)))) 835 (if-let true (has_zbs)) 836 (rv_bext x y)) 837(rule 20 (lower (has_type $I64 (band _ (u64_from_iconst 1) (ushr _ x y)))) 838 (if-let true (has_zbs)) 839 (rv_bext x y)) 840(rule 20 (lower (has_type $I64 (band _ (u64_from_iconst 1) (sshr _ x y)))) 841 (if-let true (has_zbs)) 842 (rv_bext x y)) 843 844(rule 21 (lower (has_type $I32 (band _ (ushr _ x (imm12_from_value y)) (u64_from_iconst 1)))) 845 (if-let true (has_zbs)) 846 (rv_bexti x (imm12_and y 31))) 847(rule 21 (lower (has_type $I32 (band _ (sshr _ x (imm12_from_value y)) (u64_from_iconst 1)))) 848 (if-let true (has_zbs)) 849 (rv_bexti x (imm12_and y 31))) 850(rule 21 (lower (has_type $I64 (band _ (ushr _ x (imm12_from_value y)) (u64_from_iconst 1)))) 851 (if-let true (has_zbs)) 852 (rv_bexti x (imm12_and y 63))) 853(rule 21 (lower (has_type $I64 (band _ (sshr _ x (imm12_from_value y)) (u64_from_iconst 1)))) 854 (if-let true (has_zbs)) 855 (rv_bexti x (imm12_and y 63))) 856 857;;;; Rules for `or` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 858(rule -1 (lower (has_type (ty_int ty) (bor _ x y))) 859 (gen_or ty x y)) 860 861(rule 0 (lower (has_type $F128 (bor _ x y))) 862 (gen_or $I128 x y)) 863 864;; Special cases for when one operand is an immediate that fits in 12 bits. 865(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x (imm12_from_value y)))) 866 (rv_ori x y)) 867 868(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ (imm12_from_value x) y))) 869 (rv_ori y x)) 870 871(rule 3 (lower (has_type (ty_supported_float_size ty) (bor _ x y))) 872 (lower_float_binary (AluOPRRR.Or) x y ty)) 873 874;; No need to NaN-box when moving back to the floating point register as the high 875;; bits will already be set. 876(rule 4 (lower (has_type (ty_supported_float_size $F16) (bor _ x y))) 877 (if-let false (has_zfhmin)) 878 (lower_float_binary (AluOPRRR.Or) x y $F32)) 879 880;; Specialized lowerings for `(bor x (bnot y))` which is additionally produced 881;; by Cranelift's `bor_not` instruction that is legalized into the simpler 882;; forms early on. 883 884(rule 5 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x (bnot _ y)))) 885 (if-let true (has_zbb)) 886 (rv_orn x y)) 887 888(rule 6 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ (bnot _ y) x))) 889 (if-let true (has_zbb)) 890 (rv_orn x y)) 891 892(rule 7 (lower (has_type (ty_reg_pair _) (bor _ x (bnot _ y)))) 893 (if-let true (has_zbb)) 894 (let ((low XReg (rv_orn (value_regs_get x 0) (value_regs_get y 0))) 895 (high XReg (rv_orn (value_regs_get x 1) (value_regs_get y 1)))) 896 (value_regs low high))) 897 898(rule 8 (lower (has_type (ty_reg_pair _) (bor _ (bnot _ y) x))) 899 (if-let true (has_zbb)) 900 (let ((low XReg (rv_orn (value_regs_get x 0) (value_regs_get y 0))) 901 (high XReg (rv_orn (value_regs_get x 1) (value_regs_get y 1)))) 902 (value_regs low high))) 903 904(rule 9 (lower (has_type (ty_supported_vec ty) (bor _ x y))) 905 (rv_vor_vv x y (unmasked) ty)) 906 907(rule 10 (lower (has_type (ty_supported_vec ty) (bor _ x (splat _ y)))) 908 (if (ty_vector_not_float ty)) 909 (rv_vor_vx x y (unmasked) ty)) 910 911(rule 11 (lower (has_type (ty_supported_vec ty) (bor _ (splat _ x) y))) 912 (if (ty_vector_not_float ty)) 913 (rv_vor_vx y x (unmasked) ty)) 914 915(rule 12 (lower (has_type (ty_supported_vec ty) (bor _ x y))) 916 (if-let y_imm (replicated_imm5 y)) 917 (rv_vor_vi x y_imm (unmasked) ty)) 918 919(rule 13 (lower (has_type (ty_supported_vec ty) (bor _ x y))) 920 (if-let x_imm (replicated_imm5 x)) 921 (rv_vor_vi y x_imm (unmasked) ty)) 922 923;; `bset{,i}` specializations from `zbs` 924 925(rule 14 (lower (has_type $I32 (bor _ x (ishl _ (i64_from_iconst 1) y)))) 926 (if-let true (has_zbs)) 927 (rv_bset x (rv_andi y (imm12_const 31)))) 928(rule 15 (lower (has_type $I32 (bor _ (ishl _ (i64_from_iconst 1) y) x))) 929 (if-let true (has_zbs)) 930 (rv_bset x (rv_andi y (imm12_const 31)))) 931 932(rule 14 (lower (has_type $I64 (bor _ x (ishl _ (i64_from_iconst 1) y)))) 933 (if-let true (has_zbs)) 934 (rv_bset x y)) 935(rule 15 (lower (has_type $I64 (bor _ (ishl _ (i64_from_iconst 1) y) x))) 936 (if-let true (has_zbs)) 937 (rv_bset x y)) 938 939(rule 16 (lower (has_type (fits_in_64 _) (bor _ x (u64_from_iconst n)))) 940 (if-let true (has_zbs)) 941 (if-let imm (bseti_imm n)) 942 (rv_bseti x imm)) 943(rule 17 (lower (has_type (fits_in_64 _) (bor _ (u64_from_iconst n) x))) 944 (if-let true (has_zbs)) 945 (if-let imm (bseti_imm n)) 946 (rv_bseti x imm)) 947 948(decl pure partial bseti_imm (u64) Imm12) 949(extern constructor bseti_imm bseti_imm) 950 951;;;; Rules for `xor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 952(rule 0 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x y))) 953 (rv_xor x y)) 954 955;; Special cases for when one operand is an immediate that fits in 12 bits. 956(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x (imm12_from_value y)))) 957 (rv_xori x y)) 958 959(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ (imm12_from_value x) y))) 960 (rv_xori y x)) 961 962(rule 3 (lower (has_type (ty_reg_pair _) (bxor _ x y))) 963 (lower_b128_binary (AluOPRRR.Xor) x y)) 964 965(rule 4 (lower (has_type (ty_supported_float_size ty) (bxor _ x y))) 966 (lower_float_binary (AluOPRRR.Xor) x y ty)) 967 968(rule 5 (lower (has_type (ty_supported_vec ty) (bxor _ x y))) 969 (rv_vxor_vv x y (unmasked) ty)) 970 971(rule 6 (lower (has_type (ty_supported_vec ty) (bxor _ x (splat _ y)))) 972 (if (ty_vector_not_float ty)) 973 (rv_vxor_vx x y (unmasked) ty)) 974 975(rule 7 (lower (has_type (ty_supported_vec ty) (bxor _ (splat _ x) y))) 976 (if (ty_vector_not_float ty)) 977 (rv_vxor_vx y x (unmasked) ty)) 978 979(rule 8 (lower (has_type (ty_supported_vec ty) (bxor _ x y))) 980 (if-let y_imm (replicated_imm5 y)) 981 (rv_vxor_vi x y_imm (unmasked) ty)) 982 983(rule 9 (lower (has_type (ty_supported_vec ty) (bxor _ x y))) 984 (if-let x_imm (replicated_imm5 x)) 985 (rv_vxor_vi y x_imm (unmasked) ty)) 986 987;; `binv{,i}` specializations from `zbs` 988 989(rule 13 (lower (has_type $I32 (bxor _ x (ishl _ (i64_from_iconst 1) y)))) 990 (if-let true (has_zbs)) 991 (rv_binv x (rv_andi y (imm12_const 31)))) 992(rule 14 (lower (has_type $I32 (bxor _ (ishl _ (i64_from_iconst 1) y) x))) 993 (if-let true (has_zbs)) 994 (rv_binv x (rv_andi y (imm12_const 31)))) 995 996(rule 13 (lower (has_type $I64 (bxor _ x (ishl _ (i64_from_iconst 1) y)))) 997 (if-let true (has_zbs)) 998 (rv_binv x y)) 999(rule 14 (lower (has_type $I64 (bxor _ (ishl _ (i64_from_iconst 1) y) x))) 1000 (if-let true (has_zbs)) 1001 (rv_binv x y)) 1002 1003(rule 15 (lower (has_type (fits_in_64 _) (bxor _ x (u64_from_iconst n)))) 1004 (if-let true (has_zbs)) 1005 (if-let imm (binvi_imm n)) 1006 (rv_binvi x imm)) 1007(rule 16 (lower (has_type (fits_in_64 _) (bxor _ (u64_from_iconst n) x))) 1008 (if-let true (has_zbs)) 1009 (if-let imm (binvi_imm n)) 1010 (rv_binvi x imm)) 1011 1012(decl pure partial binvi_imm (u64) Imm12) 1013(extern constructor binvi_imm binvi_imm) 1014 1015;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1016 1017(rule 0 (lower (has_type (ty_int_ref_scalar_64 _) (bnot _ x))) 1018 (rv_not x)) 1019 1020(rule 1 (lower (has_type (ty_supported_float_size ty) (bnot _ x))) 1021 (move_x_to_f (rv_not (move_f_to_x x ty)) ty)) 1022 1023(rule 2 (lower (has_type (ty_reg_pair _) (bnot _ x))) 1024 (value_regs 1025 (rv_not (value_regs_get x 0)) 1026 (rv_not (value_regs_get x 1)))) 1027 1028(rule 3 (lower (has_type (ty_supported_vec ty) (bnot _ x))) 1029 (rv_vnot_v x (unmasked) ty)) 1030 1031(rule 4 (lower (has_type (ty_int_ref_scalar_64 _) (bnot _ (bxor _ x y)))) 1032 (if-let true (has_zbb)) 1033 (rv_xnor x y)) 1034 1035;;;; Rules for `bit_reverse` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1036 1037(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (bitrev _ x))) 1038 (gen_bitrev ty x)) 1039 1040(rule 1 (lower (has_type $I128 (bitrev _ x))) 1041 (value_regs 1042 (gen_bitrev $I64 (value_regs_get x 1)) 1043 (gen_bitrev $I64 (value_regs_get x 0)))) 1044 1045 1046;; Constructs a sequence of instructions that reverse all bits in `x` up to 1047;; the given type width. 1048;; 1049;; Recursion: at most once to implement 16- and 32-bit cases in terms of 64-bit. 1050(decl rec gen_bitrev (Type XReg) XReg) 1051 1052(rule 0 (gen_bitrev (ty_16_or_32 (ty_int ty)) x) 1053 (if-let shift_amt (u64_to_imm12 (u64_wrapping_sub 64 (ty_bits ty)))) 1054 (rv_srli (gen_bitrev $I64 x) shift_amt)) 1055 1056(rule 1 (gen_bitrev $I8 x) 1057 (gen_brev8 x $I8)) 1058 1059(rule 1 (gen_bitrev $I64 x) 1060 (gen_brev8 (gen_bswap $I64 x) $I64)) 1061 1062 1063;;;; Rules for `bswap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1064 1065(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bswap _ x))) 1066 (gen_bswap ty x)) 1067 1068(rule 2 (lower (has_type $I128 (bswap _ x))) 1069 (value_regs 1070 (gen_bswap $I64 (value_regs_get x 1)) 1071 (gen_bswap $I64 (value_regs_get x 0)))) 1072 1073;; Builds a sequence of instructions that swaps the bytes in `x` up to the given 1074;; type width. 1075;; 1076;; Recursion: bounded depth since each step halves the type width. 1077(decl rec gen_bswap (Type XReg) XReg) 1078 1079;; This is only here to make the rule below work. bswap.i8 isn't valid 1080(rule 0 (gen_bswap $I8 x) x) 1081(rule 1 (gen_bswap (ty_int_ref_16_to_64 ty) x) 1082 (if-let half_ty (ty_half_width ty)) 1083 (if-let half_size (u64_to_imm12 (ty_bits half_ty))) 1084 (let ( 1085 ;; This swaps the top bytes and zeroes the bottom bytes, so that 1086 ;; we can or it with the bottom bytes later. 1087 (swap_top XReg (gen_bswap half_ty x)) 1088 (top XReg (rv_slli swap_top half_size)) 1089 1090 ;; Get the top half, swap it, and zero extend it so we can `or` it 1091 ;; with the bottom half. Note that zero extension here already knows 1092 ;; that `zbb` isn't available and that `half_ty` is not `$I64`, so this 1093 ;; falls back to the shift-then-shift sequence. 1094 (shifted XReg (rv_srli x half_size)) 1095 (swap_bot XReg (gen_bswap half_ty shifted)) 1096 (shift Imm12 (imm_from_bits (u64_wrapping_sub 64 (ty_bits half_ty)))) 1097 (bot_shifted_left XReg (rv_slli swap_bot shift)) 1098 (bot XReg (rv_srli bot_shifted_left shift))) 1099 (rv_or top bot))) 1100 1101(rule 2 (gen_bswap (ty_16_or_32 (ty_int ty)) x) 1102 (if-let true (has_zbb)) 1103 (if-let shift_amt (u64_to_imm12 (u64_wrapping_sub 64 (ty_bits ty)))) 1104 (rv_srli (rv_rev8 x) shift_amt)) 1105 1106(rule 3 (gen_bswap $I64 x) 1107 (if-let true (has_zbb)) 1108 (rv_rev8 x)) 1109 1110;;;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1111(rule (lower (has_type (fits_in_64 ty) (ctz _ x))) 1112 (lower_ctz ty x)) 1113 1114(rule 1 (lower (has_type $I128 (ctz _ x))) 1115 (let ((x_lo XReg (value_regs_get x 0)) 1116 (x_hi XReg (value_regs_get x 1)) 1117 ;; Count both halves 1118 (high XReg (lower_ctz $I64 x_hi)) 1119 (low XReg (lower_ctz $I64 x_lo)) 1120 ;; Only add the top half if the bottom is zero 1121 (high XReg (gen_select_xreg (cmp_eqz x_lo) high (zero_reg))) 1122 (result XReg (rv_add low high))) 1123 (value_regs result (imm $I64 0)))) 1124 1125;;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1126(rule 0 (lower (has_type (fits_in_64 ty) (clz _ x))) 1127 (gen_cltz true x ty)) 1128 1129(rule 1 (lower (has_type $I128 (clz _ x))) 1130 (let ((x_lo XReg (value_regs_get x 0)) 1131 (x_hi XReg (value_regs_get x 1)) 1132 ;; Count both halves 1133 (high XReg (gen_clz x_hi)) 1134 (low XReg (gen_clz x_lo)) 1135 ;; Only add the bottom zeros if the top half is zero 1136 (low XReg (gen_select_xreg (cmp_eqz x_hi) low (zero_reg)))) 1137 (value_regs (rv_add high low) (imm $I64 0)))) 1138 1139(rule 2 (lower (has_type (fits_in_16 ty) (clz _ x))) 1140 (if-let true (has_zbb)) 1141 (let ((tmp XReg (zext x)) 1142 (count XReg (rv_clz tmp))) 1143 ;; We always do the operation on the full 64-bit register, so subtract 64 from the result. 1144 (rv_addi count (imm12_const_add (ty_bits ty) -64)))) 1145 1146(rule 3 (lower (has_type $I32 (clz _ x))) 1147 (if-let true (has_zbb)) 1148 (rv_clzw x)) 1149 1150(rule 3 (lower (has_type $I64 (clz _ x))) 1151 (if-let true (has_zbb)) 1152 (rv_clz x)) 1153 1154(decl gen_clz (XReg) XReg) 1155(rule 0 (gen_clz rs) 1156 (gen_cltz true rs $I64)) 1157(rule 1 (gen_clz rs) 1158 (if-let true (has_zbb)) 1159 (rv_clz rs)) 1160 1161;;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1162 1163(rule (lower (has_type (fits_in_64 ty) (cls _ x))) 1164 (let ((tmp XReg (sext x)) 1165 (tmp2 XReg (gen_select_xreg (cmp_ltz tmp) (rv_not tmp) tmp)) 1166 (tmp3 XReg (gen_clz tmp2))) 1167 ;; clz counted the full register width, so subtract (64-$width), and then 1168 ;; additionally subtract one more, meaning here -65+width is added. 1169 (rv_addi tmp3 (imm12_const_add (ty_bits ty) -65)))) 1170 1171;; If the sign bit is set, we count the leading zeros of the inverted value. 1172;; Otherwise we can just count the leading zeros of the original value. 1173;; Subtract 1 since the sign bit does not count. 1174(rule 1 (lower (has_type $I128 (cls _ x))) 1175 (let ((low XReg (value_regs_get x 0)) 1176 (high XReg (value_regs_get x 1)) 1177 (low XReg (gen_select_xreg (cmp_ltz high) (rv_not low) low)) 1178 (high XReg (gen_select_xreg (cmp_ltz high) (rv_not high) high)) 1179 1180 ;; Count both halves 1181 (high_cnt XReg (gen_clz high)) 1182 (low_cnt XReg (gen_clz low)) 1183 ;; Only add the bottom zeros if the top half is zero 1184 (low_cnt XReg (gen_select_xreg (cmp_eqz high) low_cnt (zero_reg))) 1185 (count XReg (rv_add high_cnt low_cnt)) 1186 (result XReg (rv_addi count (imm12_const -1)))) 1187 (value_regs result (imm $I64 0)))) 1188 1189 1190;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1191(rule 0 (lower (has_type (fits_in_64 _) (uextend _ val))) 1192 (zext val)) 1193 1194(rule 1 (lower (has_type $I128 (uextend _ val))) 1195 (value_regs (zext val) (imm $I64 0))) 1196 1197;; When the source of an `uextend` is a load, we can merge both ops 1198(rule 2 (lower (has_type (fits_in_64 _) (uextend _ (sinkable_load inst ty flags addr offset)))) 1199 (gen_sunk_load inst (amode addr offset) (uextend_load_op ty) flags)) 1200 1201(decl pure uextend_load_op (Type) LoadOP) 1202(rule (uextend_load_op $I8) (LoadOP.Lbu)) 1203(rule (uextend_load_op $I16) (LoadOP.Lhu)) 1204(rule (uextend_load_op $I32) (LoadOP.Lwu)) 1205 1206;;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1207(rule 0 (lower (has_type (fits_in_64 _) (sextend _ val @ (value_type in_ty)))) 1208 (sext val)) 1209 1210(rule 1 (lower (has_type $I128 (sextend _ val @ (value_type in_ty)))) 1211 (let ((lo XReg (sext val))) 1212 (value_regs lo (rv_srai lo (imm12_const 63))))) 1213 1214;; When the source of an `sextend` is a load, we can merge both ops 1215(rule 2 (lower (has_type (fits_in_64 _) (sextend _ (sinkable_load inst ty flags addr offset)))) 1216 (gen_sunk_load inst (amode addr offset) (sextend_load_op ty) flags)) 1217 1218(decl pure sextend_load_op (Type) LoadOP) 1219(rule (sextend_load_op $I8) (LoadOP.Lb)) 1220(rule (sextend_load_op $I16) (LoadOP.Lh)) 1221(rule (sextend_load_op $I32) (LoadOP.Lw)) 1222 1223;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1224 1225(rule 0 (lower (has_type (fits_in_64 _) (popcnt _ x))) 1226 (gen_popcnt (zext x))) 1227 1228(rule 1 (lower (has_type $I128 (popcnt _ x))) 1229 (let 1230 ((x ValueRegs x) 1231 (low XReg (gen_popcnt (value_regs_get x 0))) 1232 (high XReg (gen_popcnt (value_regs_get x 1))) 1233 (result XReg (rv_add low high))) 1234 (value_regs result (imm $I64 0)))) 1235 1236(rule 2 (lower (has_type (fits_in_64 _) (popcnt _ x))) 1237 (if-let true (has_zbb)) 1238 (rv_cpop (zext x))) 1239 1240(rule 3 (lower (has_type $I32 (popcnt _ x))) 1241 (if-let true (has_zbb)) 1242 (rv_cpopw x)) 1243 1244(rule 3 (lower (has_type $I128 (popcnt _ x))) 1245 (if-let true (has_zbb)) 1246 (let 1247 ((x ValueRegs x) 1248 (low XReg (rv_cpop (value_regs_get x 0))) 1249 (high XReg (rv_cpop (value_regs_get x 1))) 1250 (result XReg (rv_add low high))) 1251 (value_regs result (imm $I64 0)))) 1252 1253;; Popcount using multiply. 1254;; This is popcount64c() from 1255;; http://en.wikipedia.org/wiki/Hamming_weight 1256;; 1257;; Here's the C version for 32 bits: 1258;; x = x - ((x>> 1) & 0x55555555); 1259;; x = (x & 0x33333333) + ((x >> 2) & 0x33333333); 1260;; x = ((x + (x >> 4)) & 0x0F0F0F0F); 1261;; return (x * 0x01010101) >> 24; // Here 24 is the type width - 8. 1262;; 1263;; TODO: LLVM generates a much better implementation for I8X16. See: https://godbolt.org/z/qr6vf9Gr3 1264;; For the other types it seems to be largely the same. 1265(rule 4 (lower (has_type (ty_supported_vec ty) (popcnt _ x))) 1266 (if-let one (u64_to_uimm5 1)) 1267 (if-let two (u64_to_uimm5 2)) 1268 (if-let four (u64_to_uimm5 4)) 1269 1270 (let ( 1271 ;; x = x - ((x >> 1) & 0x55555555); 1272 (mask_55 XReg (imm (lane_type ty) (u64_and 0x5555555555555555 (ty_mask (lane_type ty))))) 1273 (count2_shr VReg (rv_vsrl_vi x one (unmasked) ty)) 1274 (count2_and VReg (rv_vand_vx count2_shr mask_55 (unmasked) ty)) 1275 (count2 VReg (rv_vsub_vv x count2_and (unmasked) ty)) 1276 1277 ;; x = (x & 0x33333333) + ((x >> 2) & 0x33333333); 1278 (mask_33 XReg (imm (lane_type ty) (u64_and 0x3333333333333333 (ty_mask (lane_type ty))))) 1279 (count4_shr VReg (rv_vsrl_vi count2 two (unmasked) ty)) 1280 (count4_and VReg (rv_vand_vx count4_shr mask_33 (unmasked) ty)) 1281 (count4_lhs VReg (rv_vand_vx count2 mask_33 (unmasked) ty)) 1282 (count4 VReg (rv_vadd_vv count4_lhs count4_and (unmasked) ty)) 1283 1284 ;; x = (x + (x >> 4)) & 0x0F0F0F0F; 1285 (mask_0f XReg (imm (lane_type ty) (u64_and 0x0f0f0f0f0f0f0f0f (ty_mask (lane_type ty))))) 1286 (count8_shr VReg (rv_vsrl_vi count4 four (unmasked) ty)) 1287 (count8_add VReg (rv_vadd_vv count4 count8_shr (unmasked) ty)) 1288 (count8 VReg (rv_vand_vx count8_add mask_0f (unmasked) ty)) 1289 1290 ;; (x * 0x01010101) >> (<ty_width> - 8) 1291 (mask_01 XReg (imm (lane_type ty) (u64_and 0x0101010101010101 (ty_mask (lane_type ty))))) 1292 (mul VReg (rv_vmul_vx count8 mask_01 (unmasked) ty)) 1293 (shift XReg (imm $I64 (u64_wrapping_sub (ty_bits (lane_type ty)) 8))) 1294 (res VReg (rv_vsrl_vx mul shift (unmasked) ty))) 1295 res)) 1296 1297;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1298 1299;; 8/16 bit types need a mask on the shift amount 1300(rule 0 (lower (has_type (ty_int (ty_8_or_16 ty)) (ishl _ x y))) 1301 (if-let mask (u64_to_imm12 (ty_shift_mask ty))) 1302 (rv_sllw x (rv_andi (value_regs_get y 0) mask))) 1303 1304;; Using the 32bit version of `sll` automatically masks the shift amount. 1305(rule 1 (lower (has_type $I32 (ishl _ x y))) 1306 (rv_sllw x (value_regs_get y 0))) 1307 1308;; Similarly, the 64bit version does the right thing. 1309(rule 1 (lower (has_type $I64 (ishl _ x y))) 1310 (rv_sll x (value_regs_get y 0))) 1311 1312;; If the shift amount is known. We can mask it and encode it in the instruction. 1313(rule 2 (lower (has_type (int_fits_in_32 ty) (ishl _ x (maybe_uextend (imm12_from_value y))))) 1314 (rv_slliw x (imm12_and y (ty_shift_mask ty)))) 1315 1316;; We technically don't need to mask the shift amount here. The instruction 1317;; does the right thing. But it's neater when pretty printing it. 1318(rule 3 (lower (has_type ty @ $I64 (ishl _ x (maybe_uextend (imm12_from_value y))))) 1319 (rv_slli x (imm12_and y (ty_shift_mask ty)))) 1320 1321;; With `Zba` we have a shift that zero extends the LHS argument. 1322(rule 4 (lower (has_type $I64 (ishl _ (uextend _ x @ (value_type $I32)) (maybe_uextend (imm12_from_value y))))) 1323 (if-let true (has_zba)) 1324 (rv_slliuw x y)) 1325 1326;; I128 cases 1327(rule 4 (lower (has_type $I128 (ishl _ x y))) 1328 (let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0))) 1329 (shamt XReg (value_regs_get tmp 0)) 1330 (len_sub_shamt XReg (value_regs_get tmp 1)) 1331 ;; 1332 (low XReg (rv_sll (value_regs_get x 0) shamt)) 1333 ;; high part. 1334 (high_part1 XReg (rv_srl (value_regs_get x 0) len_sub_shamt)) 1335 (high_part2 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) high_part1)) 1336 ;; 1337 (high_part3 XReg (rv_sll (value_regs_get x 1) shamt)) 1338 (high XReg (rv_or high_part2 high_part3)) 1339 ;; 1340 (const64 XReg (imm $I64 64)) 1341 (shamt_128 XReg (rv_andi (value_regs_get y 0) (imm12_const 127)))) 1342 (gen_select_regs 1343 (cmp_geu shamt_128 const64) 1344 (value_regs (zero_reg) low) 1345 (value_regs low high)))) 1346 1347;; SIMD Cases 1348;; We don't need to mask anything since it is done by the instruction according to SEW. 1349 1350(rule 5 (lower (has_type (ty_supported_vec ty) (ishl _ x y))) 1351 (rv_vsll_vx x (value_regs_get y 0) (unmasked) ty)) 1352 1353(rule 6 (lower (has_type (ty_supported_vec ty) (ishl _ x (maybe_uextend (uimm5_from_value y))))) 1354 (rv_vsll_vi x y (unmasked) ty)) 1355 1356;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1357 1358;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be 1359;; zero extended. 1360(rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (ushr _ x y))) 1361 (if-let mask (u64_to_imm12 (ty_shift_mask ty))) 1362 (rv_srlw (zext x) (rv_andi (value_regs_get y 0) mask))) 1363 1364;; Using the 32bit version of `srl` automatically masks the shift amount. 1365(rule 1 (lower (has_type $I32 (ushr _ x y))) 1366 (rv_srlw x (value_regs_get y 0))) 1367 1368;; Similarly, the 64bit version does the right thing. 1369(rule 1 (lower (has_type $I64 (ushr _ x y))) 1370 (rv_srl x (value_regs_get y 0))) 1371 1372;; When the RHS is known we can just encode it in the instruction. 1373(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr _ x (maybe_uextend (imm12_from_value y))))) 1374 (rv_srliw (zext x) (imm12_and y (ty_shift_mask ty)))) 1375 1376(rule 3 (lower (has_type $I32 (ushr _ x (maybe_uextend (imm12_from_value y))))) 1377 (rv_srliw x y)) 1378 1379(rule 3 (lower (has_type $I64 (ushr _ x (maybe_uextend (imm12_from_value y))))) 1380 (rv_srli x y)) 1381 1382(rule 3 (lower (has_type $I128 (ushr _ x y))) 1383 (let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0))) 1384 (shamt XReg (value_regs_get tmp 0)) 1385 (len_sub_shamt XReg (value_regs_get tmp 1)) 1386 ;; low part. 1387 (low_part1 XReg (rv_sll (value_regs_get x 1) len_sub_shamt)) 1388 (low_part2 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) low_part1)) 1389 ;; 1390 (low_part3 XReg (rv_srl (value_regs_get x 0) shamt)) 1391 (low XReg (rv_or low_part2 low_part3)) 1392 ;; 1393 (const64 XReg (imm $I64 64)) 1394 ;; 1395 (high XReg (rv_srl (value_regs_get x 1) shamt)) 1396 (shamt_128 XReg (rv_andi (value_regs_get y 0) (imm12_const 127)))) 1397 (gen_select_regs 1398 (cmp_geu shamt_128 const64) 1399 (value_regs high (zero_reg)) 1400 (value_regs low high)))) 1401 1402;; SIMD Cases 1403;; We don't need to mask or extend anything since it is done by the instruction according to SEW. 1404 1405(rule 4 (lower (has_type (ty_supported_vec ty) (ushr _ x y))) 1406 (rv_vsrl_vx x (value_regs_get y 0) (unmasked) ty)) 1407 1408(rule 5 (lower (has_type (ty_supported_vec ty) (ushr _ x (maybe_uextend (uimm5_from_value y))))) 1409 (rv_vsrl_vi x y (unmasked) ty)) 1410 1411;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1412 1413;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be 1414;; zero extended. 1415(rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (sshr _ x y))) 1416 (if-let mask (u64_to_imm12 (ty_shift_mask ty))) 1417 (rv_sraw (sext x) (rv_andi (value_regs_get y 0) mask))) 1418 1419;; Using the 32bit version of `sra` automatically masks the shift amount. 1420(rule 1 (lower (has_type $I32 (sshr _ x y))) 1421 (rv_sraw x (value_regs_get y 0))) 1422 1423;; Similarly, the 64bit version does the right thing. 1424(rule 1 (lower (has_type $I64 (sshr _ x y))) 1425 (rv_sra x (value_regs_get y 0))) 1426 1427;; When the RHS is known we can just encode it in the instruction. 1428(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr _ x (maybe_uextend (imm12_from_value y))))) 1429 (rv_sraiw (sext x) (imm12_and y (ty_shift_mask ty)))) 1430 1431(rule 3 (lower (has_type $I32 (sshr _ x (maybe_uextend (imm12_from_value y))))) 1432 (rv_sraiw x y)) 1433 1434(rule 3 (lower (has_type $I64 (sshr _ x (maybe_uextend (imm12_from_value y))))) 1435 (rv_srai x y)) 1436 1437(rule 3 (lower (has_type $I128 (sshr _ x y))) 1438 (let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0))) 1439 (shamt XReg (value_regs_get tmp 0)) 1440 (len_sub_shamt XReg (value_regs_get tmp 1)) 1441 ;; low part. 1442 (low_part1 XReg (rv_sll (value_regs_get x 1) len_sub_shamt)) 1443 (low_part2 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) low_part1)) 1444 ;; 1445 (low_part3 XReg (rv_srl (value_regs_get x 0) shamt)) 1446 (low XReg (rv_or low_part2 low_part3)) 1447 ;; 1448 (const64 XReg (imm $I64 64)) 1449 ;; 1450 (high XReg (rv_sra (value_regs_get x 1) shamt)) 1451 ;; 1452 (const_neg_1 XReg (imm $I64 (i64_cast_unsigned -1))) 1453 ;; 1454 (high_replacement XReg (gen_select_xreg (cmp_ltz (value_regs_get x 1)) const_neg_1 (zero_reg))) 1455 (const64 XReg (imm $I64 64)) 1456 (shamt_128 XReg (rv_andi (value_regs_get y 0) (imm12_const 127)))) 1457 (gen_select_regs 1458 (cmp_geu shamt_128 const64) 1459 (value_regs high high_replacement) 1460 (value_regs low high)))) 1461 1462;; SIMD Cases 1463;; We don't need to mask or extend anything since it is done by the instruction according to SEW. 1464 1465(rule 4 (lower (has_type (ty_supported_vec ty) (sshr _ x y))) 1466 (rv_vsra_vx x (value_regs_get y 0) (unmasked) ty)) 1467 1468(rule 5 (lower (has_type (ty_supported_vec ty) (sshr _ x (maybe_uextend (uimm5_from_value y))))) 1469 (rv_vsra_vi x y (unmasked) ty)) 1470 1471 1472;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1473 1474(rule 0 (lower (has_type (fits_in_64 ty) (rotl _ rs amount))) 1475 (let 1476 ((rs XReg (zext rs)) 1477 (amount XReg (value_regs_get amount 0)) 1478 (x ValueRegs (gen_shamt ty amount)) 1479 (shamt XReg (value_regs_get x 0)) 1480 (len_sub_shamt Reg (value_regs_get x 1)) 1481 (part1 Reg (rv_sll rs shamt)) 1482 (part2 Reg (rv_srl rs len_sub_shamt)) 1483 (part3 Reg (gen_select_xreg (cmp_eqz shamt) (zero_reg) part2))) 1484 (rv_or part1 part3))) 1485 1486(rule 1 (lower (has_type $I32 (rotl _ rs amount))) 1487 (if-let true (has_zbb)) 1488 (rv_rolw rs (value_regs_get amount 0))) 1489 1490(rule 2 (lower (has_type $I32 (rotl _ rs (u64_from_iconst n)))) 1491 (if-let true (has_zbb)) 1492 (if-let (imm12_from_u64 imm) (u64_wrapping_sub 32 (u64_and n 31))) 1493 (rv_roriw rs imm)) 1494 1495(rule 1 (lower (has_type $I64 (rotl _ rs amount))) 1496 (if-let true (has_zbb)) 1497 (rv_rol rs (value_regs_get amount 0))) 1498 1499(rule 2 (lower (has_type $I64 (rotl _ rs (u64_from_iconst n)))) 1500 (if-let true (has_zbb)) 1501 (if-let (imm12_from_u64 imm) (u64_wrapping_sub 64 (u64_and n 63))) 1502 (rv_rori rs imm)) 1503 1504(rule 1 (lower (has_type $I128 (rotl _ x y))) 1505 (let 1506 ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0))) 1507 (shamt XReg (value_regs_get tmp 0)) 1508 (len_sub_shamt XReg (value_regs_get tmp 1)) 1509 (low_part1 XReg (rv_sll (value_regs_get x 0) shamt)) 1510 (low_part2 XReg (rv_srl (value_regs_get x 1) len_sub_shamt)) 1511 ;;; if shamt == 0 low_part2 will overflow we should zero instead. 1512 (low_part3 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) low_part2)) 1513 (low XReg (rv_or low_part1 low_part3)) 1514 (high_part1 XReg (rv_sll (value_regs_get x 1) shamt)) 1515 (high_part2 XReg (rv_srl (value_regs_get x 0) len_sub_shamt)) 1516 (high_part3 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) high_part2)) 1517 (high XReg (rv_or high_part1 high_part3)) 1518 (const64 XReg (imm $I64 64)) 1519 (shamt_128 XReg (rv_andi (value_regs_get y 0) (imm12_const 127)))) 1520 ;; right now we only rotate less than 64 bits. 1521 ;; if shamt is greater than or equal 64 , we should switch low and high. 1522 (gen_select_regs 1523 (cmp_geu shamt_128 const64) 1524 (value_regs high low) 1525 (value_regs low high) 1526 ))) 1527 1528;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1529 1530(rule (lower (has_type (fits_in_64 ty) (rotr _ rs amount))) 1531 (let 1532 ((rs XReg (zext rs)) 1533 (amount XReg (value_regs_get amount 0)) 1534 (x ValueRegs (gen_shamt ty amount)) 1535 (shamt XReg (value_regs_get x 0)) 1536 (len_sub_shamt XReg (value_regs_get x 1)) 1537 (part1 XReg (rv_srl rs shamt)) 1538 (part2 XReg (rv_sll rs len_sub_shamt)) 1539 (part3 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) part2))) 1540 (rv_or part1 part3))) 1541 1542(rule 1 (lower (has_type $I32 (rotr _ rs amount))) 1543 (if-let true (has_zbb)) 1544 (rv_rorw rs (value_regs_get amount 0))) 1545 1546(rule 2 (lower (has_type $I32 (rotr _ rs (imm12_from_value n)))) 1547 (if-let true (has_zbb)) 1548 (rv_roriw rs n)) 1549 1550(rule 1 (lower (has_type $I64 (rotr _ rs amount))) 1551 (if-let true (has_zbb)) 1552 (rv_ror rs (value_regs_get amount 0))) 1553 1554(rule 2 (lower (has_type $I64 (rotr _ rs (imm12_from_value n)))) 1555 (if-let true (has_zbb)) 1556 (rv_rori rs n)) 1557 1558(rule 1 (lower (has_type $I128 (rotr _ x y))) 1559 (let 1560 ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0))) 1561 (shamt XReg (value_regs_get tmp 0)) 1562 (len_sub_shamt XReg (value_regs_get tmp 1)) 1563 (low_part1 XReg (rv_srl (value_regs_get x 0) shamt)) 1564 (low_part2 XReg (rv_sll (value_regs_get x 1) len_sub_shamt)) 1565 ;;; if shamt == 0 low_part2 will overflow we should zero instead. 1566 (low_part3 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) low_part2)) 1567 (low XReg (rv_or low_part1 low_part3)) 1568 (high_part1 XReg (rv_srl (value_regs_get x 1) shamt)) 1569 (high_part2 XReg (rv_sll (value_regs_get x 0) len_sub_shamt)) 1570 (high_part3 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) high_part2)) 1571 (high XReg (rv_or high_part1 high_part3)) 1572 (const64 XReg (imm $I64 64)) 1573 (shamt_128 XReg (rv_andi (value_regs_get y 0) (imm12_const 127)))) 1574 ;; right now we only rotate less than 64 bits. 1575 ;; if shamt is greater than or equal 64 , we should switch low and high. 1576 (gen_select_regs 1577 (cmp_geu shamt_128 const64) 1578 (value_regs high low) 1579 (value_regs low high) 1580 ))) 1581 1582;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1583(rule 0 (lower (has_type (ty_supported_float_full ty) (fabs _ x))) 1584 (rv_fabs ty x)) 1585 1586(rule 1 (lower (has_type (ty_supported_vec ty) (fabs _ x))) 1587 (rv_vfabs_v x (unmasked) ty)) 1588 1589;;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1590(rule 0 (lower (has_type (ty_supported_float_full ty) (fneg _ x))) 1591 (rv_fneg ty x)) 1592 1593(rule 1 (lower (has_type (ty_supported_vec ty) (fneg _ x))) 1594 (rv_vfneg_v x (unmasked) ty)) 1595 1596;;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1597(rule 0 (lower (has_type (ty_supported_float_full ty) (fcopysign _ x y))) 1598 (rv_fsgnj ty x y)) 1599 1600(rule 1 (lower (has_type (ty_supported_vec ty) (fcopysign _ x y))) 1601 (rv_vfsgnj_vv x y (unmasked) ty)) 1602 1603(rule 2 (lower (has_type (ty_supported_vec ty) (fcopysign _ x (splat _ y)))) 1604 (rv_vfsgnj_vf x y (unmasked) ty)) 1605 1606;;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1607 1608;; RISC-V has 4 FMA instructions that do a slightly different computation. 1609;; 1610;; fmadd: (rs1 * rs2) + rs3 1611;; fmsub: (rs1 * rs2) - rs3 1612;; fnmadd: -(rs1 * rs2) - rs3 1613;; fnmsub: -(rs1 * rs2) + rs3 1614;; 1615;; Additionally there are vector versions of these instructions with slightly different names. 1616;; The vector instructions also have two variants each. `.vv` and `.vf`, where `.vv` variants 1617;; take two vector operands and the `.vf` variants take a vector operand and a scalar operand. 1618;; 1619;; Due to this, variation they receive the arguments in a different order. So we need to swap 1620;; the arguments below. 1621;; 1622;; vfmacc: vd[i] = +(vs1[i] * vs2[i]) + vd[i] 1623;; vfmsac: vd[i] = +(vs1[i] * vs2[i]) - vd[i] 1624;; vfnmacc: vd[i] = -(vs1[i] * vs2[i]) - vd[i] 1625;; vfnmsac: vd[i] = -(vs1[i] * vs2[i]) + vd[i] 1626 1627(type IsFneg (enum (Result (negate u64) (value Value)))) 1628 1629(decl pure is_fneg (Value) IsFneg) 1630(rule 1 (is_fneg (fneg _ x)) (IsFneg.Result 1 x)) 1631(rule 0 (is_fneg x) (IsFneg.Result 0 x)) 1632 1633(decl pure is_fneg_neg (IsFneg) u64) 1634(rule (is_fneg_neg (IsFneg.Result n _)) n) 1635 1636(decl pure get_fneg_value (IsFneg) Value) 1637(rule (get_fneg_value (IsFneg.Result _ v)) v) 1638 1639(rule (lower (has_type ty (fma _ x_src y_src z_src))) 1640 (let 1641 ((x_res IsFneg (is_fneg x_src)) 1642 (y_res IsFneg (is_fneg y_src)) 1643 (z_res IsFneg (is_fneg z_src)) 1644 (x Value (get_fneg_value x_res)) 1645 (y Value (get_fneg_value y_res)) 1646 (z Value (get_fneg_value z_res))) 1647 (rv_fma ty (u64_xor (is_fneg_neg x_res) (is_fneg_neg y_res)) (is_fneg_neg z_res) x y z))) 1648 1649; parity arguments indicate whether to negate the x*y term or the z term, respectively 1650(decl rv_fma (Type u64 u64 Value Value Value) InstOutput) 1651(rule 0 (rv_fma (ty_supported_float_full ty) 0 0 x y z) (rv_fmadd ty (FRM.RNE) x y z)) 1652(rule 0 (rv_fma (ty_supported_float_full ty) 0 1 x y z) (rv_fmsub ty (FRM.RNE) x y z)) 1653(rule 0 (rv_fma (ty_supported_float_full ty) 1 0 x y z) (rv_fnmsub ty (FRM.RNE) x y z)) 1654(rule 0 (rv_fma (ty_supported_float_full ty) 1 1 x y z) (rv_fnmadd ty (FRM.RNE) x y z)) 1655(rule 1 (rv_fma (ty_supported_vec ty) 0 0 x y z) (rv_vfmacc_vv z y x (unmasked) ty)) 1656(rule 1 (rv_fma (ty_supported_vec ty) 0 1 x y z) (rv_vfmsac_vv z y x (unmasked) ty)) 1657(rule 1 (rv_fma (ty_supported_vec ty) 1 0 x y z) (rv_vfnmsac_vv z y x (unmasked) ty)) 1658(rule 1 (rv_fma (ty_supported_vec ty) 1 1 x y z) (rv_vfnmacc_vv z y x (unmasked) ty)) 1659(rule 2 (rv_fma (ty_supported_vec ty) 0 0 (splat _ x) y z) (rv_vfmacc_vf z y x (unmasked) ty)) 1660(rule 2 (rv_fma (ty_supported_vec ty) 0 1 (splat _ x) y z) (rv_vfmsac_vf z y x (unmasked) ty)) 1661(rule 2 (rv_fma (ty_supported_vec ty) 1 0 (splat _ x) y z) (rv_vfnmsac_vf z y x (unmasked) ty)) 1662(rule 2 (rv_fma (ty_supported_vec ty) 1 1 (splat _ x) y z) (rv_vfnmacc_vf z y x (unmasked) ty)) 1663(rule 3 (rv_fma (ty_supported_vec ty) 0 0 x (splat _ y) z) (rv_vfmacc_vf z x y (unmasked) ty)) 1664(rule 3 (rv_fma (ty_supported_vec ty) 0 1 x (splat _ y) z) (rv_vfmsac_vf z x y (unmasked) ty)) 1665(rule 3 (rv_fma (ty_supported_vec ty) 1 0 x (splat _ y) z) (rv_vfnmsac_vf z x y (unmasked) ty)) 1666(rule 3 (rv_fma (ty_supported_vec ty) 1 1 x (splat _ y) z) (rv_vfnmacc_vf z x y (unmasked) ty)) 1667 1668;;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1669(rule 0 (lower (has_type (ty_supported_float_full ty) (sqrt _ x))) 1670 (rv_fsqrt ty (FRM.RNE) x)) 1671 1672(rule 1 (lower (has_type (ty_supported_vec ty) (sqrt _ x))) 1673 (rv_vfsqrt_v x (unmasked) ty)) 1674 1675;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1676(rule -1 1677 ;; 1678 (lower 1679 (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) op addr x))) 1680 (gen_atomic (get_atomic_rmw_op ty op) addr x (atomic_amo))) 1681 1682;;; for I8 and I16 1683(rule 1 1684 (lower 1685 (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) op addr x))) 1686 (gen_atomic_rmw_loop op ty addr x)) 1687 1688;;;special for I8 and I16 max min etc. 1689;;;because I need uextend or sextend the value. 1690(rule 2 1691 (lower 1692 (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) (is_atomic_rmw_max_etc op true) addr x))) 1693 (gen_atomic_rmw_loop op ty addr (sext x))) 1694 1695 1696(rule 2 1697 ;; 1698 (lower 1699 (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) (is_atomic_rmw_max_etc op false) addr x))) 1700 ;; 1701 (gen_atomic_rmw_loop op ty addr (zext x))) 1702 1703;;;;; Rules for `AtomicRmwOp.Sub` 1704(rule 1705 (lower 1706 (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) addr x))) 1707 (let 1708 ((tmp WritableReg (temp_writable_reg ty)) 1709 (x2 Reg (rv_neg x))) 1710 (gen_atomic (get_atomic_rmw_op ty (AtomicRmwOp.Add)) addr x2 (atomic_amo)))) 1711 1712(decl gen_atomic_rmw_loop (AtomicRmwOp Type XReg XReg) XReg) 1713(rule 1714 (gen_atomic_rmw_loop op ty addr x) 1715 (let 1716 ((dst WritableXReg (temp_writable_xreg)) 1717 (t0 WritableXReg (temp_writable_xreg)) 1718 (_ Unit (emit (MInst.AtomicRmwLoop (gen_atomic_offset addr ty) op dst ty (gen_atomic_p addr ty) x t0)))) 1719 (writable_reg_to_reg dst))) 1720 1721;;;;; Rules for `AtomicRmwOp.Nand` 1722(rule 1723 (lower 1724 (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Nand) addr x))) 1725 (gen_atomic_rmw_loop (AtomicRmwOp.Nand) ty addr x)) 1726 1727(decl is_atomic_rmw_max_etc (AtomicRmwOp bool) AtomicRmwOp) 1728(extern extractor is_atomic_rmw_max_etc is_atomic_rmw_max_etc) 1729 1730;;;;; Rules for `atomic load`;;;;;;;;;;;;;;;;; 1731(rule 1732 (lower (has_type (valid_atomic_transaction ty) (atomic_load _ (little_or_native_endian flags) p))) 1733 (gen_atomic_load p ty)) 1734 1735 1736;;;;; Rules for `atomic store`;;;;;;;;;;;;;;;;; 1737(rule 1738 (lower (atomic_store (little_or_native_endian flags) src @ (value_type (valid_atomic_transaction ty)) p)) 1739 (gen_atomic_store p ty src)) 1740 1741(decl gen_atomic_offset (XReg Type) XReg) 1742(rule 1 (gen_atomic_offset p (fits_in_16 ty)) 1743 (rv_slli (rv_andi p (imm12_const 3)) (imm12_const 3))) 1744 1745(rule (gen_atomic_offset p _) 1746 (zero_reg)) 1747 1748(decl gen_atomic_p (XReg Type) XReg) 1749(rule 1 (gen_atomic_p p (fits_in_16 ty)) 1750 (rv_andi p (imm12_const -4))) 1751 1752(rule (gen_atomic_p p _) 1753 p) 1754 1755 1756;;;;; Rules for `atomic cas`;;;;;;;;;;;;;;;;; 1757(rule 1758 (lower (has_type (valid_atomic_transaction ty) (atomic_cas _ (little_or_native_endian flags) p e x))) 1759 (let 1760 ((t0 WritableReg (temp_writable_reg ty)) 1761 (dst WritableReg (temp_writable_reg ty)) 1762 (_ Unit (emit (MInst.AtomicCas (gen_atomic_offset p ty) t0 dst (zext e) (gen_atomic_p p ty) x ty)))) 1763 (writable_reg_to_reg dst))) 1764 1765;;;;; Rules for `ireduce`;;;;;;;;;;;;;;;;; 1766(rule 1767 (lower (has_type ty (ireduce _ x))) 1768 (value_regs_get x 0)) 1769 1770;;;;; Rules for `fpromote`;;;;;;;;;;;;;;;;; 1771(rule (lower (fpromote _ x)) 1772 (rv_fcvtds x)) 1773 1774;;;;; Rules for `fvpromote_low`;;;;;;;;;;;; 1775 1776(rule (lower (has_type (ty_supported_vec ty) (fvpromote_low _ x))) 1777 (if-let half_ty (ty_half_width ty)) 1778 (rv_vfwcvt_f_f_v x (unmasked) (vstate_mf2 half_ty))) 1779 1780;;;;; Rules for `fdemote`;;;;;;;;;;;;;;;;;; 1781(rule (lower (fdemote _ x)) 1782 (rv_fcvtsd (FRM.RNE) x)) 1783 1784;;;;; Rules for `fvdemote`;;;;;;;;;;;;;;;;; 1785 1786;; `vfncvt...` leaves the upper bits of the register undefined so 1787;; we need to zero them out. 1788(rule (lower (has_type (ty_supported_vec ty @ $F32X4) (fvdemote _ x))) 1789 (if-let zero (i8_to_imm5 0)) 1790 (let ((narrow VReg (rv_vfncvt_f_f_w x (unmasked) (vstate_mf2 ty))) 1791 (mask VReg (gen_vec_mask 0xC))) 1792 (rv_vmerge_vim narrow zero mask ty))) 1793 1794 1795;;;;; Rules for for float arithmetic 1796 1797 1798;;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1799 1800(rule 0 (lower (has_type (ty_supported_float_full ty) (fadd _ x y))) 1801 (rv_fadd ty (FRM.RNE) x y)) 1802 1803(rule 1 (lower (has_type (ty_supported_vec ty) (fadd _ x y))) 1804 (rv_vfadd_vv x y (unmasked) ty)) 1805 1806(rule 2 (lower (has_type (ty_supported_vec ty) (fadd _ x (splat _ y)))) 1807 (rv_vfadd_vf x y (unmasked) ty)) 1808 1809(rule 3 (lower (has_type (ty_supported_vec ty) (fadd _ (splat _ x) y))) 1810 (rv_vfadd_vf y x (unmasked) ty)) 1811 1812 1813;;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1814(rule 0 (lower (has_type (ty_supported_float_full ty) (fsub _ x y))) 1815 (rv_fsub ty (FRM.RNE) x y)) 1816 1817(rule 1 (lower (has_type (ty_supported_vec ty) (fsub _ x y))) 1818 (rv_vfsub_vv x y (unmasked) ty)) 1819 1820(rule 2 (lower (has_type (ty_supported_vec ty) (fsub _ x (splat _ y)))) 1821 (rv_vfsub_vf x y (unmasked) ty)) 1822 1823(rule 3 (lower (has_type (ty_supported_vec ty) (fsub _ (splat _ x) y))) 1824 (rv_vfrsub_vf y x (unmasked) ty)) 1825 1826;;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1827(rule 0 (lower (has_type (ty_supported_float_full ty) (fmul _ x y))) 1828 (rv_fmul ty (FRM.RNE) x y)) 1829 1830(rule 1 (lower (has_type (ty_supported_vec ty) (fmul _ x y))) 1831 (rv_vfmul_vv x y (unmasked) ty)) 1832 1833(rule 2 (lower (has_type (ty_supported_vec ty) (fmul _ x (splat _ y)))) 1834 (rv_vfmul_vf x y (unmasked) ty)) 1835 1836(rule 3 (lower (has_type (ty_supported_vec ty) (fmul _ (splat _ x) y))) 1837 (rv_vfmul_vf y x (unmasked) ty)) 1838 1839 1840;;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1841(rule 0 (lower (has_type (ty_supported_float_full ty) (fdiv _ x y))) 1842 (rv_fdiv ty (FRM.RNE) x y)) 1843 1844(rule 1 (lower (has_type (ty_supported_vec ty) (fdiv _ x y))) 1845 (rv_vfdiv_vv x y (unmasked) ty)) 1846 1847(rule 2 (lower (has_type (ty_supported_vec ty) (fdiv _ x (splat _ y)))) 1848 (rv_vfdiv_vf x y (unmasked) ty)) 1849 1850(rule 3 (lower (has_type (ty_supported_vec ty) (fdiv _ (splat _ x) y))) 1851 (rv_vfrdiv_vf y x (unmasked) ty)) 1852 1853;;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1854 1855;; RISC-V's `fmin` instruction returns the number input if one of inputs is a 1856;; NaN. We handle this by manually checking if one of the inputs is a NaN 1857;; and selecting based on that result. 1858(rule 0 (lower (has_type (ty_supported_float_full ty) (fmin _ x y))) 1859 (let ( 1860 ;; Check if both inputs are not nan. 1861 (is_ordered FloatCompare (fcmp_to_float_compare (FloatCC.Ordered) ty x y)) 1862 ;; `fadd` returns a nan if any of the inputs is a NaN. 1863 (nan FReg (rv_fadd ty (FRM.RNE) x y)) 1864 (min FReg (rv_fmin ty x y))) 1865 (gen_select_freg is_ordered min nan))) 1866 1867;; With Zfa we can use the special `fminm` that precisely matches the expected 1868;; NaN behavior. 1869(rule 1 (lower (has_type (ty_supported_float_full ty) (fmin _ x y))) 1870 (if-let true (has_zfa)) 1871 (rv_fminm ty x y)) 1872 1873;; vfmin does almost the right thing, but it does not handle NaN's correctly. 1874;; We should return a NaN if any of the inputs is a NaN, but vfmin returns the 1875;; number input instead. 1876;; 1877;; TODO: We can improve this by using a masked `fmin` instruction that modifies 1878;; the canonical nan register. That way we could avoid the `vmerge.vv` instruction. 1879(rule 2 (lower (has_type (ty_supported_vec ty) (fmin _ x y))) 1880 (let ((is_not_nan VReg (gen_fcmp_mask ty (FloatCC.Ordered) x y)) 1881 (nan XReg (imm $I64 (canonical_nan_u64 (lane_type ty)))) 1882 (vec_nan VReg (rv_vmv_vx nan ty)) 1883 (min VReg (rv_vfmin_vv x y (unmasked) ty))) 1884 (rv_vmerge_vvm vec_nan min is_not_nan ty))) 1885 1886;;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1887 1888;; RISC-V's `fmax` instruction returns the number input if one of inputs is a 1889;; NaN. We handle this by manually checking if one of the inputs is a NaN 1890;; and selecting based on that result. 1891(rule 0 (lower (has_type (ty_supported_float_full ty) (fmax _ x y))) 1892 (let ( 1893 ;; Check if both inputs are not nan. 1894 (is_ordered FloatCompare (fcmp_to_float_compare (FloatCC.Ordered) ty x y)) 1895 ;; `fadd` returns a NaN if any of the inputs is a NaN. 1896 (nan FReg (rv_fadd ty (FRM.RNE) x y)) 1897 (max FReg (rv_fmax ty x y))) 1898 (gen_select_freg is_ordered max nan))) 1899 1900;; With Zfa we can use the special `fmaxm` that precisely matches the expected 1901;; NaN behavior. 1902(rule 1 (lower (has_type (ty_supported_float_full ty) (fmax _ x y))) 1903 (if-let true (has_zfa)) 1904 (rv_fmaxm ty x y)) 1905 1906;; vfmax does almost the right thing, but it does not handle NaN's correctly. 1907;; We should return a NaN if any of the inputs is a NaN, but vfmax returns the 1908;; number input instead. 1909;; 1910;; TODO: We can improve this by using a masked `fmax` instruction that modifies 1911;; the canonical nan register. That way we could avoid the `vmerge.vv` instruction. 1912(rule 2 (lower (has_type (ty_supported_vec ty) (fmax _ x y))) 1913 (let ((is_not_nan VReg (gen_fcmp_mask ty (FloatCC.Ordered) x y)) 1914 (nan XReg (imm $I64 (canonical_nan_u64 (lane_type ty)))) 1915 (vec_nan VReg (rv_vmv_vx nan ty)) 1916 (max VReg (rv_vfmax_vv x y (unmasked) ty))) 1917 (rv_vmerge_vvm vec_nan max is_not_nan ty))) 1918 1919;;;;; Rules for `stack_addr`;;;;;;;;; 1920(rule 1921 (lower (stack_addr _ ss offset)) 1922 (gen_stack_addr ss offset)) 1923 1924;;;;; Rules for `select`;;;;;;;;; 1925 1926;; Manually matching (iconst 0) here is a bit of a hack. We can't do that as part 1927;; of the iconst rule because that runs into regalloc issues. gen_select_xreg 1928;; has some optimizations based on the use of the zero register so we have to 1929;; manually match it here. 1930(rule 5 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c (i64_from_iconst 0) y))) 1931 (gen_select_xreg (is_nonzero_cmp c) (zero_reg) y)) 1932 1933(rule 4 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c x (i64_from_iconst 0)))) 1934 (gen_select_xreg (is_nonzero_cmp c) x (zero_reg))) 1935 1936(rule 3 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c x y))) 1937 (gen_select_xreg (is_nonzero_cmp c) x y)) 1938 1939(rule 2 (lower (has_type (ty_reg_pair _) (select _ c x y))) 1940 (gen_select_regs (is_nonzero_cmp c) x y)) 1941 1942(rule 1 (lower (has_type (ty_supported_vec _) (select _ c x y))) 1943 (gen_select_vreg (is_nonzero_cmp c) x y)) 1944 1945(rule 0 (lower (has_type (ty_supported_float_size _) (select _ c x y))) 1946 (gen_select_freg (is_nonzero_cmp c) x y)) 1947 1948;;;;; Rules for `bitselect`;;;;;;;;; 1949 1950;; Do a (c & x) | (~c & y) operation. 1951(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (bitselect _ c x y))) 1952 (let ((tmp_x XReg (rv_and c x)) 1953 (c_inverse XReg (rv_not c)) 1954 (tmp_y XReg (rv_and c_inverse y))) 1955 (rv_or tmp_x tmp_y))) 1956 1957;; For vectors, we also do the same operation. 1958;; We can technically use any type in the bitwise operations, but prefer 1959;; using the type of the inputs so that we avoid emitting unnecessary 1960;; `vsetvl` instructions. it's likely that the vector unit is already 1961;; configured for that type. 1962(rule 1 (lower (has_type (ty_supported_vec ty) (bitselect _ c x y))) 1963 (let ((tmp_x VReg (rv_vand_vv c x (unmasked) ty)) 1964 (c_inverse VReg (rv_vnot_v c (unmasked) ty)) 1965 (tmp_y VReg (rv_vand_vv c_inverse y (unmasked) ty))) 1966 (rv_vor_vv tmp_x tmp_y (unmasked) ty))) 1967 1968;; Special case for bitselects with cmp's as an input. 1969;; 1970;; This allows us to skip the mask expansion step and use the more efficient 1971;; vmerge.vvm instruction. 1972;; 1973;; We should be careful to ensure that the mask and the vmerge have the 1974;; same type. So that we don't generate a mask with length 16 (i.e. for i8x16), and then 1975;; only copy the first few lanes of the result to the destination register because 1976;; the bitselect has a different length (i.e. i64x2). 1977;; 1978;; See: https://github.com/bytecodealliance/wasmtime/issues/8131 1979 1980(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (icmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y))) 1981 (let ((mask VReg (gen_icmp_mask cmp_ty cc a b))) 1982 (rv_vmerge_vvm y x mask cmp_ty))) 1983 1984(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (fcmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y))) 1985 (let ((mask VReg (gen_fcmp_mask cmp_ty cc a b))) 1986 (rv_vmerge_vvm y x mask cmp_ty))) 1987 1988(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (bitcast _ _ (fcmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y))) 1989 (let ((mask VReg (gen_fcmp_mask cmp_ty cc a b))) 1990 (rv_vmerge_vvm y x mask cmp_ty))) 1991 1992(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (bitcast _ _ (icmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y))) 1993 (let ((mask VReg (gen_icmp_mask cmp_ty cc a b))) 1994 (rv_vmerge_vvm y x mask cmp_ty))) 1995 1996 1997;;;;; Rules for `isplit`;;;;;;;;; 1998(rule 1999 (lower (isplit _ x)) 2000 (let 2001 ((t1 XReg (value_regs_get x 0)) 2002 (t2 XReg (value_regs_get x 1))) 2003 (output_pair t1 t2))) 2004 2005;;;;; Rules for `iconcat`;;;;;;;;; 2006(rule 2007 (lower (has_type $I128 (iconcat _ x y))) 2008 (let 2009 ((t1 XReg x) 2010 (t2 XReg y)) 2011 (value_regs t1 t2))) 2012 2013;; Special-case the lowering of an `isplit` of a 128-bit multiply where the 2014;; lower bits of the result are discarded and the operands are sign or zero 2015;; extended. This maps directly to `umulh` and `smulh`. 2016(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (uextend _ x) (uextend _ y))))) 2017 (if-let (first_result lo) i) 2018 (if-let true (value_is_unused lo)) 2019 (output_pair (invalid_reg) (rv_mulhu (zext x) (zext y)))) 2020 2021(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (sextend _ x) (sextend _ y))))) 2022 (if-let (first_result lo) i) 2023 (if-let true (value_is_unused lo)) 2024 (output_pair (invalid_reg) (rv_mulh (sext x) (sext y)))) 2025 2026;;;;; Rules for `smax`;;;;;;;;; 2027 2028(rule 0 (lower (has_type (fits_in_64 ty) (smax _ x y))) 2029 (let ((x XReg (sext x)) 2030 (y XReg (sext y))) 2031 (gen_select_xreg (cmp_gt x y) x y))) 2032 2033(rule 1 (lower (has_type $I128 (smax _ x y))) 2034 (gen_select_regs (icmp_to_int_compare (IntCC.SignedGreaterThan) x y) x y)) 2035 2036(rule 2 (lower (has_type (ty_supported_vec ty) (smax _ x y))) 2037 (rv_vmax_vv x y (unmasked) ty)) 2038 2039(rule 3 (lower (has_type (ty_supported_vec ty) (smax _ x (splat _ y)))) 2040 (rv_vmax_vx x y (unmasked) ty)) 2041 2042(rule 4 (lower (has_type (ty_supported_vec ty) (smax _ (splat _ x) y))) 2043 (rv_vmax_vx y x (unmasked) ty)) 2044 2045;;;;; Rules for `smin`;;;;;;;;; 2046 2047(rule 0 (lower (has_type (fits_in_64 ty) (smin _ x y))) 2048 (let ((x XReg (sext x)) 2049 (y XReg (sext y))) 2050 (gen_select_xreg (cmp_lt x y) x y))) 2051 2052(rule 1 (lower (has_type $I128 (smin _ x y))) 2053 (gen_select_regs (icmp_to_int_compare (IntCC.SignedLessThan) x y) x y)) 2054 2055(rule 2 (lower (has_type (ty_supported_vec ty) (smin _ x y))) 2056 (rv_vmin_vv x y (unmasked) ty)) 2057 2058(rule 3 (lower (has_type (ty_supported_vec ty) (smin _ x (splat _ y)))) 2059 (rv_vmin_vx x y (unmasked) ty)) 2060 2061(rule 4 (lower (has_type (ty_supported_vec ty) (smin _ (splat _ x) y))) 2062 (rv_vmin_vx y x (unmasked) ty)) 2063 2064;;;;; Rules for `umax`;;;;;;;;; 2065 2066(rule 0 (lower (has_type (fits_in_64 ty) (umax _ x y))) 2067 (let ((x XReg (zext x)) 2068 (y XReg (zext y))) 2069 (gen_select_xreg (cmp_gtu x y) x y))) 2070 2071(rule 1 (lower (has_type $I128 (umax _ x y))) 2072 (gen_select_regs (icmp_to_int_compare (IntCC.UnsignedGreaterThan) x y) x y)) 2073 2074(rule 2 (lower (has_type (ty_supported_vec ty) (umax _ x y))) 2075 (rv_vmaxu_vv x y (unmasked) ty)) 2076 2077(rule 3 (lower (has_type (ty_supported_vec ty) (umax _ x (splat _ y)))) 2078 (rv_vmaxu_vx x y (unmasked) ty)) 2079 2080(rule 4 (lower (has_type (ty_supported_vec ty) (umax _ (splat _ x) y))) 2081 (rv_vmaxu_vx y x (unmasked) ty)) 2082 2083;;;;; Rules for `umin`;;;;;;;;; 2084 2085(rule 0 (lower (has_type (fits_in_64 ty) (umin _ x y))) 2086 (let ((x XReg (zext x)) 2087 (y XReg (zext y))) 2088 (gen_select_xreg (cmp_ltu x y) x y))) 2089 2090(rule 1 (lower (has_type $I128 (umin _ x y))) 2091 (gen_select_regs (icmp_to_int_compare (IntCC.UnsignedLessThan) x y) x y)) 2092 2093(rule 2 (lower (has_type (ty_supported_vec ty) (umin _ x y))) 2094 (rv_vminu_vv x y (unmasked) ty)) 2095 2096(rule 3 (lower (has_type (ty_supported_vec ty) (umin _ x (splat _ y)))) 2097 (rv_vminu_vx x y (unmasked) ty)) 2098 2099(rule 4 (lower (has_type (ty_supported_vec ty) (umin _ (splat _ x) y))) 2100 (rv_vminu_vx y x (unmasked) ty)) 2101 2102 2103;;;;; Rules for `debugtrap`;;;;;;;;; 2104(rule 2105 (lower (debugtrap)) 2106 (side_effect (SideEffectNoResult.Inst (MInst.EBreak)))) 2107 2108;;;;; Rules for `fence`;;;;;;;;; 2109(rule 2110 (lower (fence)) 2111 (side_effect (SideEffectNoResult.Inst (MInst.Fence 15 15)))) 2112 2113;;;;; Rules for `trap`;;;;;;;;; 2114(rule 2115 (lower (trap code)) 2116 (udf code)) 2117 2118;;;;; Rules for `trapz`;;;;;;;;; 2119(rule 2120 (lower (trapz value @ (value_type (fits_in_64 _)) code)) 2121 (gen_trapz value code)) 2122 2123(rule 1 2124 (lower (trapz value @ (value_type $I128) code)) 2125 (gen_trapif_val_i128 (ZeroCond.Zero) value code)) 2126 2127; fold icmp + trapz 2128(rule 2 (lower (trapz (icmp _ cc x @ (value_type (fits_in_64 _)) y) code)) 2129 (gen_trapif (intcc_complement cc) 2130 (put_value_in_reg_for_icmp cc x) 2131 (put_value_in_reg_for_icmp cc y) 2132 code)) 2133 2134;;;;; Rules for `trapnz`;;;;;;;;; 2135(rule 2136 (lower (trapnz value @ (value_type (fits_in_64 _)) code)) 2137 (gen_trapnz value code)) 2138 2139(rule 1 2140 (lower (trapnz value @ (value_type $I128) code)) 2141 (gen_trapif_val_i128 (ZeroCond.NonZero) value code)) 2142 2143; fold icmp + trapnz 2144(rule 2 (lower (trapnz (icmp _ cc x @ (value_type (fits_in_64 _)) y) code)) 2145 (gen_trapif cc 2146 (put_value_in_reg_for_icmp cc x) 2147 (put_value_in_reg_for_icmp cc y) 2148 code)) 2149 2150;;;;; Rules for `uload8`;;;;;;;;; 2151(rule (lower (uload8 _ (little_or_native_endian flags) addr offset)) 2152 (gen_load (amode addr offset) (LoadOP.Lbu) flags)) 2153 2154;;;;; Rules for `sload8`;;;;;;;;; 2155(rule (lower (sload8 _ (little_or_native_endian flags) addr offset)) 2156 (gen_load (amode addr offset) (LoadOP.Lb) flags)) 2157 2158;;;;; Rules for `uload16`;;;;;;;;; 2159(rule (lower (uload16 _ (little_or_native_endian flags) addr offset)) 2160 (gen_load (amode addr offset) (LoadOP.Lhu) flags)) 2161 2162;;;;; Rules for `iload16`;;;;;;;;; 2163(rule (lower (sload16 _ (little_or_native_endian flags) addr offset)) 2164 (gen_load (amode addr offset) (LoadOP.Lh) flags)) 2165 2166;;;;; Rules for `uload32`;;;;;;;;; 2167(rule (lower (uload32 _ (little_or_native_endian flags) addr offset)) 2168 (gen_load (amode addr offset) (LoadOP.Lwu) flags)) 2169 2170;;;;; Rules for `sload32`;;;;;;;;; 2171(rule (lower (sload32 _ (little_or_native_endian flags) addr offset)) 2172 (gen_load (amode addr offset) (LoadOP.Lw) flags)) 2173 2174;;;;; Rules for `load`;;;;;;;;; 2175(rule (lower (has_type ty (load _ (little_or_native_endian flags) addr offset))) 2176 (gen_load (amode addr offset) (load_op ty) flags)) 2177 2178(rule 1 (lower (has_type (ty_reg_pair _) (load _ (little_or_native_endian flags) addr offset))) 2179 (if-let offset_plus_8 (i32_checked_add offset 8)) 2180 (let ((lo XReg (gen_load (amode addr offset) (LoadOP.Ld) flags)) 2181 (hi XReg (gen_load (amode addr offset_plus_8) (LoadOP.Ld) flags))) 2182 (value_regs lo hi))) 2183 2184(rule 2 (lower (has_type (ty_supported_vec ty) (load _ (little_or_native_endian flags) addr offset))) 2185 (let ((eew VecElementWidth (element_width_from_type ty)) 2186 (amode AMode (amode addr offset))) 2187 (vec_load eew (VecAMode.UnitStride amode) flags (unmasked) ty))) 2188 2189;;;;; Rules for Load + Extend Combos ;;;;;;;;; 2190 2191;; These rules cover the special loads that load a 64bit value and do some sort of extension. 2192;; We don't have any special instructions to do this, so just load the 64 bits as a vector, and 2193;; do a SEW/2 extension. This only reads half width elements from the source vector register 2194;; extends it, and writes the back the full register. 2195 2196(decl gen_load64_extend (Type ExtendOp MemFlags AMode) VReg) 2197 2198(rule (gen_load64_extend ty (ExtendOp.Signed) flags amode) 2199 (let ((eew VecElementWidth (element_width_from_type $I64)) 2200 (load_state VState (vstate_from_type $I64)) 2201 (loaded VReg (vec_load eew (VecAMode.UnitStride amode) flags (unmasked) load_state))) 2202 (rv_vsext_vf2 loaded (unmasked) ty))) 2203 2204(rule (gen_load64_extend ty (ExtendOp.Zero) flags amode) 2205 (let ((eew VecElementWidth (element_width_from_type $I64)) 2206 (load_state VState (vstate_from_type $I64)) 2207 (loaded VReg (vec_load eew (VecAMode.UnitStride amode) flags (unmasked) load_state))) 2208 (rv_vzext_vf2 loaded (unmasked) ty))) 2209 2210;;;;; Rules for `uload8x8`;;;;;;;;;; 2211(rule (lower (has_type (ty_supported_vec ty @ $I16X8) (uload8x8 _ (little_or_native_endian flags) addr offset))) 2212 (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset))) 2213 2214;;;;; Rules for `uload16x4`;;;;;;;;; 2215(rule (lower (has_type (ty_supported_vec ty @ $I32X4) (uload16x4 _ (little_or_native_endian flags) addr offset))) 2216 (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset))) 2217 2218;;;;; Rules for `uload32x2`;;;;;;;;; 2219(rule (lower (has_type (ty_supported_vec ty @ $I64X2) (uload32x2 _ (little_or_native_endian flags) addr offset))) 2220 (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset))) 2221 2222;;;;; Rules for `sload8x8`;;;;;;;;;; 2223(rule (lower (has_type (ty_supported_vec ty @ $I16X8) (sload8x8 _ (little_or_native_endian flags) addr offset))) 2224 (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset))) 2225 2226;;;;; Rules for `sload16x4`;;;;;;;;; 2227(rule (lower (has_type (ty_supported_vec ty @ $I32X4) (sload16x4 _ (little_or_native_endian flags) addr offset))) 2228 (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset))) 2229 2230;;;;; Rules for `sload32x2`;;;;;;;;; 2231(rule (lower (has_type (ty_supported_vec ty @ $I64X2) (sload32x2 _ (little_or_native_endian flags) addr offset))) 2232 (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset))) 2233 2234;;;;; Rules for `istore8`;;;;;;;;; 2235(rule (lower (istore8 (little_or_native_endian flags) src addr offset)) 2236 (rv_store (amode addr offset) (StoreOP.Sb) flags src)) 2237 2238;;;;; Rules for `istore16`;;;;;;;;; 2239(rule (lower (istore16 (little_or_native_endian flags) src addr offset)) 2240 (rv_store (amode addr offset) (StoreOP.Sh) flags src)) 2241 2242;;;;; Rules for `istore32`;;;;;;;;; 2243(rule (lower (istore32 (little_or_native_endian flags) src addr offset)) 2244 (rv_store (amode addr offset) (StoreOP.Sw) flags src)) 2245 2246;;;;; Rules for `store`;;;;;;;;; 2247(rule (lower (store (little_or_native_endian flags) src @ (value_type ty) addr offset)) 2248 (gen_store (amode addr offset) flags src)) 2249 2250(rule 1 (lower (store (little_or_native_endian flags) src @ (value_type (ty_reg_pair _)) addr offset)) 2251 (if-let offset_plus_8 (i32_checked_add offset 8)) 2252 (let ((_ InstOutput (rv_store (amode addr offset) (StoreOP.Sd) flags (value_regs_get src 0)))) 2253 (rv_store (amode addr offset_plus_8) (StoreOP.Sd) flags (value_regs_get src 1)))) 2254 2255(rule 2 (lower (store (little_or_native_endian flags) src @ (value_type (ty_supported_vec ty)) addr offset)) 2256 (let ((eew VecElementWidth (element_width_from_type ty)) 2257 (amode AMode (amode addr offset))) 2258 (vec_store eew (VecAMode.UnitStride amode) src flags (unmasked) ty))) 2259 2260;; Avoid unnecessary moves to floating point registers for `F16` memory to memory copies when 2261;; `Zfhmin` is unavailable. 2262(rule 3 (lower (store (little_or_native_endian store_flags) 2263 (sinkable_load inst $F16 (little_or_native_endian load_flags) load_addr load_offset) store_addr store_offset)) 2264 (if-let false (has_zfhmin)) 2265 (rv_store (amode store_addr store_offset) (StoreOP.Sh) store_flags (gen_sunk_load inst (amode load_addr load_offset) (LoadOP.Lh) load_flags))) 2266 2267 2268;;;;; Rules for `icmp`;;;;;;;;; 2269 2270;; 8-64 bit comparisons. Mostly fall back onto `IntegerCompare` and then 2271;; materializing that, but before that happens try to match some 2272;; constant-related patterns 2273 2274(rule 0 (lower (icmp _ cc x @ (value_type (fits_in_64 ty)) y)) 2275 (lower_icmp cc x y)) 2276 2277; Recursion: at most once to implement >= in terms of <. 2278(decl rec lower_icmp (IntCC Value Value) XReg) 2279(rule 0 (lower_icmp cc x y) 2280 (lower_int_compare (icmp_to_int_compare cc x y))) 2281 2282;; a == $imm => seqz(xori(..)) 2283(rule 1 (lower_icmp (IntCC.Equal) x y) 2284 (if-let (i64_from_iconst (i64_extract_non_zero (imm12_from_i64 imm))) y) 2285 (rv_seqz (rv_xori (sext x) imm))) 2286(rule 2 (lower_icmp (IntCC.Equal) x y) 2287 (if-let (i64_from_iconst (i64_extract_non_zero (imm12_from_i64 imm))) x) 2288 (rv_seqz (rv_xori (sext y) imm))) 2289 2290;; a != $imm => snez(xori(..)) 2291(rule 1 (lower_icmp (IntCC.NotEqual) x y) 2292 (if-let (i64_from_iconst (i64_extract_non_zero (imm12_from_i64 imm))) y) 2293 (rv_snez (rv_xori (sext x) imm))) 2294(rule 2 (lower_icmp (IntCC.NotEqual) x y) 2295 (if-let (i64_from_iconst (i64_extract_non_zero (imm12_from_i64 imm))) x) 2296 (rv_snez (rv_xori (sext y) imm))) 2297 2298;; a < $imm => slti(..) 2299(rule 1 (lower_icmp (IntCC.SignedLessThan) x y) 2300 (if-let (i64_from_iconst (i64_extract_non_zero (imm12_from_i64 imm))) y) 2301 (rv_slti (sext x) imm)) 2302(rule 1 (lower_icmp (IntCC.SignedGreaterThan) x y) 2303 (if-let (i64_from_iconst (i64_extract_non_zero (imm12_from_i64 imm))) x) 2304 (rv_slti (sext y) imm)) 2305(rule 1 (lower_icmp (IntCC.UnsignedLessThan) x y) 2306 (if-let (u64_from_iconst (u64_extract_non_zero (imm12_from_u64 imm))) y) 2307 (rv_sltiu (zext x) imm)) 2308(rule 1 (lower_icmp (IntCC.UnsignedGreaterThan) x y) 2309 (if-let (u64_from_iconst (u64_extract_non_zero (imm12_from_u64 imm))) x) 2310 (rv_sltiu (zext y) imm)) 2311 2312;; a >= $imm => !(a < $imm) 2313(rule 2 (lower_icmp cc @ (IntCC.SignedGreaterThanOrEqual) x y) 2314 (if-let (i64_from_iconst (i64_extract_non_zero (imm12_from_i64 _))) y) 2315 (rv_xori (lower_icmp (intcc_complement cc) x y) (imm12_const 1))) 2316(rule 2 (lower_icmp cc @ (IntCC.UnsignedGreaterThanOrEqual) x y) 2317 (if-let (u64_from_iconst (u64_extract_non_zero (imm12_from_u64 _))) y) 2318 (rv_xori (lower_icmp (intcc_complement cc) x y) (imm12_const 1))) 2319 2320;; Materializes an `IntegerCompare` bundle directly into an `XReg` with a 0 2321;; or 1 value. 2322(decl lower_int_compare (IntegerCompare) XReg) 2323 2324;; x == y => x ^ y == 0 2325(rule 0 (lower_int_compare (int_compare_decompose (IntCC.Equal) x y)) 2326 (rv_seqz (rv_xor x y))) 2327(rule 1 (lower_int_compare (int_compare_decompose (IntCC.Equal) x (zero_reg))) 2328 (rv_seqz x)) 2329(rule 2 (lower_int_compare (int_compare_decompose (IntCC.Equal) (zero_reg) y)) 2330 (rv_seqz y)) 2331;; x != y => x ^ y != 0 2332(rule 0 (lower_int_compare (int_compare_decompose (IntCC.NotEqual) x y)) 2333 (rv_snez (rv_xor x y))) 2334(rule 1 (lower_int_compare (int_compare_decompose (IntCC.NotEqual) x (zero_reg))) 2335 (rv_snez x)) 2336(rule 2 (lower_int_compare (int_compare_decompose (IntCC.NotEqual) (zero_reg) x)) 2337 (rv_snez x)) 2338;; x < y => x < y 2339(rule (lower_int_compare (int_compare_decompose (IntCC.SignedLessThan) x y)) 2340 (rv_slt x y)) 2341(rule (lower_int_compare (int_compare_decompose (IntCC.UnsignedLessThan) x y)) 2342 (rv_sltu x y)) 2343;; x > y => y < x 2344(rule (lower_int_compare (int_compare_decompose (IntCC.SignedGreaterThan) x y)) 2345 (rv_slt y x)) 2346(rule (lower_int_compare (int_compare_decompose (IntCC.UnsignedGreaterThan) x y)) 2347 (rv_sltu y x)) 2348;; x <= y => !(y < x) 2349(rule (lower_int_compare (int_compare_decompose (IntCC.SignedLessThanOrEqual) x y)) 2350 (rv_xori (rv_slt y x) (imm12_const 1))) 2351(rule (lower_int_compare (int_compare_decompose (IntCC.UnsignedLessThanOrEqual) x y)) 2352 (rv_xori (rv_sltu y x) (imm12_const 1))) 2353;; x >= y => !(x < y) 2354(rule (lower_int_compare (int_compare_decompose (IntCC.SignedGreaterThanOrEqual) x y)) 2355 (rv_xori (rv_slt x y) (imm12_const 1))) 2356(rule (lower_int_compare (int_compare_decompose (IntCC.UnsignedGreaterThanOrEqual) x y)) 2357 (rv_xori (rv_sltu x y) (imm12_const 1))) 2358 2359;; 128-bit comparisons. 2360;; 2361;; Currently only `==`, `!=`, and `<` are implemented, and everything else 2362;; delegates to one of those. 2363 2364(rule 20 (lower (icmp _ cc x @ (value_type $I128) y)) 2365 (lower_icmp_i128 cc x y)) 2366 2367; Recursion: at most once to implement some conditions in terms of a smaller primitive set. 2368(decl rec lower_icmp_i128 (IntCC ValueRegs ValueRegs) XReg) 2369(rule 0 (lower_icmp_i128 (IntCC.Equal) x y) 2370 (let ((lo XReg (rv_xor (value_regs_get x 0) (value_regs_get y 0))) 2371 (hi XReg (rv_xor (value_regs_get x 1) (value_regs_get y 1)))) 2372 (rv_seqz (rv_or lo hi)))) 2373(rule 0 (lower_icmp_i128 (IntCC.NotEqual) x y) 2374 (let ((lo XReg (rv_xor (value_regs_get x 0) (value_regs_get y 0))) 2375 (hi XReg (rv_xor (value_regs_get x 1) (value_regs_get y 1)))) 2376 (rv_snez (rv_or lo hi)))) 2377 2378;; swap args for `>` to use `<` instead 2379(rule 0 (lower_icmp_i128 cc @ (IntCC.SignedGreaterThan) x y) 2380 (lower_icmp_i128 (intcc_swap_args cc) y x)) 2381(rule 0 (lower_icmp_i128 cc @ (IntCC.UnsignedGreaterThan) x y) 2382 (lower_icmp_i128 (intcc_swap_args cc) y x)) 2383 2384;; complement `=`-related conditions to get ones that don't use `=`. 2385(rule 0 (lower_icmp_i128 cc @ (IntCC.SignedLessThanOrEqual) x y) 2386 (rv_xori (lower_icmp_i128 (intcc_complement cc) x y) (imm12_const 1))) 2387(rule 0 (lower_icmp_i128 cc @ (IntCC.SignedGreaterThanOrEqual) x y) 2388 (rv_xori (lower_icmp_i128 (intcc_complement cc) x y) (imm12_const 1))) 2389(rule 0 (lower_icmp_i128 cc @ (IntCC.UnsignedLessThanOrEqual) x y) 2390 (rv_xori (lower_icmp_i128 (intcc_complement cc) x y) (imm12_const 1))) 2391(rule 0 (lower_icmp_i128 cc @ (IntCC.UnsignedGreaterThanOrEqual) x y) 2392 (rv_xori (lower_icmp_i128 (intcc_complement cc) x y) (imm12_const 1))) 2393 2394;; Compare both the bottom and upper halves of the 128-bit values. If 2395;; the top half is equal use the bottom comparison, otherwise use the upper 2396;; comparison. Note that the lower comparison is always unsigned since if it's 2397;; used the top halves are all zeros and the semantic values are positive. 2398(rule 1 (lower_icmp_i128 cc x y) 2399 (if-let (IntCC.UnsignedLessThan) (intcc_unsigned cc)) 2400 (let ((x_lo Reg (value_regs_get x 0)) 2401 (x_hi Reg (value_regs_get x 1)) 2402 (y_lo Reg (value_regs_get y 0)) 2403 (y_hi Reg (value_regs_get y 1)) 2404 (top_cmp XReg (lower_int_compare (int_compare cc x_hi y_hi))) 2405 (bottom_cmp XReg (rv_sltu x_lo y_lo))) 2406 (gen_select_xreg (cmp_eqz (rv_xor x_hi y_hi)) bottom_cmp top_cmp))) 2407 2408;; vector icmp comparisons 2409 2410(rule 30 (lower (icmp _ cc x @ (value_type (ty_supported_vec ty)) y)) 2411 (gen_expand_mask ty (gen_icmp_mask ty cc x y))) 2412 2413;;;;; Rules for `fcmp`;;;;;;;;; 2414(rule 0 (lower (fcmp _ cc x @ (value_type (ty_supported_float_full ty)) y)) 2415 (lower_float_compare (fcmp_to_float_compare cc ty x y))) 2416 2417(decl lower_float_compare (FloatCompare) XReg) 2418(rule (lower_float_compare (FloatCompare.One r)) r) 2419(rule (lower_float_compare (FloatCompare.Zero r)) (rv_seqz r)) 2420 2421(rule 1 (lower (fcmp _ cc x @ (value_type (ty_supported_vec ty)) y)) 2422 (gen_expand_mask ty (gen_fcmp_mask ty cc x y))) 2423 2424;;;;; Rules for `func_addr`;;;;;;;;; 2425(rule 2426 (lower (func_addr _ (func_ref_data _ name dist _))) 2427 (load_ext_name name 0 dist)) 2428 2429;;;;; Rules for `fcvt_to_uint`;;;;;;;;; 2430 2431;; RISC-V float-to-integer conversion does not trap, but Cranelift semantics are 2432;; to trap. This manually performs checks for NaN and out-of-bounds values and 2433;; traps in such cases. 2434;; 2435;; TODO: could this perhaps be more optimal through inspection of the `fcsr`? 2436;; Unsure whether that needs to be preserved across function calls and/or would 2437;; cause other problems. Also unsure whether it's actually more performant. 2438(rule (lower (has_type ity (fcvt_to_uint _ v @ (value_type fty)))) 2439 (let ((_ InstOutput (gen_trapz (rv_feq fty v v) (TrapCode.BAD_CONVERSION_TO_INTEGER))) 2440 (min FReg (imm fty (fcvt_umin_bound fty false))) 2441 (_ InstOutput (gen_trapnz (rv_fle fty v min) (TrapCode.INTEGER_OVERFLOW))) 2442 (max FReg (imm fty (fcvt_umax_bound fty ity false))) 2443 (_ InstOutput (gen_trapnz (rv_fge fty v max) (TrapCode.INTEGER_OVERFLOW)))) 2444 (lower_inbounds_fcvt_to_uint ity fty v))) 2445 2446(decl lower_inbounds_fcvt_to_uint (Type Type FReg) XReg) 2447(rule 0 (lower_inbounds_fcvt_to_uint (fits_in_32 _) fty v) 2448 (rv_fcvtwu fty (FRM.RTZ) v)) 2449(rule 1 (lower_inbounds_fcvt_to_uint $I64 fty v) 2450 (rv_fcvtlu fty (FRM.RTZ) v)) 2451 2452;;;;; Rules for `fcvt_to_sint`;;;;;;;;; 2453 2454;; NB: see above with `fcvt_to_uint` as this is similar 2455(rule (lower (has_type ity (fcvt_to_sint _ v @ (value_type fty)))) 2456 (let ((_ InstOutput (gen_trapz (rv_feq fty v v) (TrapCode.BAD_CONVERSION_TO_INTEGER))) 2457 (min FReg (imm fty (fcvt_smin_bound fty ity false))) 2458 (_ InstOutput (gen_trapnz (rv_fle fty v min) (TrapCode.INTEGER_OVERFLOW))) 2459 (max FReg (imm fty (fcvt_smax_bound fty ity false))) 2460 (_ InstOutput (gen_trapnz (rv_fge fty v max) (TrapCode.INTEGER_OVERFLOW)))) 2461 (lower_inbounds_fcvt_to_sint ity fty v))) 2462 2463(decl lower_inbounds_fcvt_to_sint (Type Type FReg) XReg) 2464(rule 0 (lower_inbounds_fcvt_to_sint (fits_in_32 _) fty v) 2465 (rv_fcvtw fty (FRM.RTZ) v)) 2466(rule 1 (lower_inbounds_fcvt_to_sint $I64 fty v) 2467 (rv_fcvtl fty (FRM.RTZ) v)) 2468 2469;;;;; Rules for `fcvt_to_sint_sat`;;;;;;;;; 2470 2471(rule 0 (lower (has_type to (fcvt_to_sint_sat _ v @ (value_type (ty_supported_float_full from))))) 2472 (handle_fcvt_to_int_nan from v (lower_fcvt_to_sint_sat from to v))) 2473 2474;; Lowers to a `rv_fcvt*` instruction but handles 8/16-bit cases where the 2475;; float is clamped before the conversion. 2476(decl lower_fcvt_to_sint_sat (Type Type FReg) XReg) 2477(rule 0 (lower_fcvt_to_sint_sat ty (fits_in_16 out_ty) v) 2478 (let ((max FReg (imm ty (fcvt_smax_bound ty out_ty true))) 2479 (min FReg (imm ty (fcvt_smin_bound ty out_ty true))) 2480 (clamped FReg (rv_fmin ty max (rv_fmax ty min v)))) 2481 (rv_fcvtw ty (FRM.RTZ) clamped))) 2482(rule 1 (lower_fcvt_to_sint_sat ty $I32 v) (rv_fcvtw ty (FRM.RTZ) v)) 2483(rule 1 (lower_fcvt_to_sint_sat ty $I64 v) (rv_fcvtl ty (FRM.RTZ) v)) 2484 2485(decl fcvt_smax_bound (Type Type bool) u64) 2486(extern constructor fcvt_smax_bound fcvt_smax_bound) 2487(decl fcvt_smin_bound (Type Type bool) u64) 2488(extern constructor fcvt_smin_bound fcvt_smin_bound) 2489 2490;; RISC-V float-to-int conversions generate the same output for NaN and +Inf, 2491;; but Cranelift semantics are to produce 0 for NaN instead. This helper 2492;; translates these semantics by taking the float being converted (with the type 2493;; specified) and the native RISC-V output as an `XReg`. The returned `XReg` 2494;; will be zeroed out if the float is NaN. 2495;; 2496;; This is done by comparing the float to itself, generating 0 if it's NaN. This 2497;; bit is then negated to become either all-ones or all-zeros which is then 2498;; and-ed against the native output. That'll produce all zeros if the input is 2499;; NaN or the native output otherwise. 2500(decl handle_fcvt_to_int_nan (Type FReg XReg) XReg) 2501(rule (handle_fcvt_to_int_nan ty freg xreg) 2502 (let ((is_not_nan XReg (rv_feq ty freg freg)) 2503 (not_nan_mask XReg (rv_neg is_not_nan))) 2504 (rv_and xreg not_nan_mask))) 2505 2506(rule 1 (lower (has_type (ty_supported_vec _) (fcvt_to_sint_sat _ v @ (value_type from_ty)))) 2507 (if-let zero (i8_to_imm5 0)) 2508 (let ((is_nan VReg (rv_vmfne_vv v v (unmasked) from_ty)) 2509 (cvt VReg (rv_vfcvt_rtz_x_f_v v (unmasked) from_ty))) 2510 (rv_vmerge_vim cvt zero is_nan from_ty))) 2511 2512;;;;; Rules for `fcvt_to_uint_sat`;;;;;;;;; 2513 2514(rule 0 (lower (has_type to (fcvt_to_uint_sat _ v @ (value_type (ty_supported_float_full from))))) 2515 (handle_fcvt_to_int_nan from v (lower_fcvt_to_uint_sat from to v))) 2516 2517;; Lowers to a `rv_fcvt*` instruction but handles 8/16-bit cases where the 2518;; float is clamped before the conversion. 2519(decl lower_fcvt_to_uint_sat (Type Type FReg) XReg) 2520(rule 0 (lower_fcvt_to_uint_sat ty (fits_in_16 out_ty) v) 2521 (let ((max FReg (imm ty (fcvt_umax_bound ty out_ty true))) 2522 (min FReg (rv_fmvdx (zero_reg))) 2523 (clamped FReg (rv_fmin ty max (rv_fmax ty min v)))) 2524 (rv_fcvtwu ty (FRM.RTZ) clamped))) 2525(rule 1 (lower_fcvt_to_uint_sat ty $I32 v) (rv_fcvtwu ty (FRM.RTZ) v)) 2526(rule 1 (lower_fcvt_to_uint_sat ty $I64 v) (rv_fcvtlu ty (FRM.RTZ) v)) 2527 2528(decl fcvt_umax_bound (Type Type bool) u64) 2529(extern constructor fcvt_umax_bound fcvt_umax_bound) 2530(decl fcvt_umin_bound (Type bool) u64) 2531(extern constructor fcvt_umin_bound fcvt_umin_bound) 2532 2533(rule 1 (lower (has_type (ty_supported_vec _) (fcvt_to_uint_sat _ v @ (value_type from_ty)))) 2534 (if-let zero (i8_to_imm5 0)) 2535 (let ((is_nan VReg (rv_vmfne_vv v v (unmasked) from_ty)) 2536 (cvt VReg (rv_vfcvt_rtz_xu_f_v v (unmasked) from_ty))) 2537 (rv_vmerge_vim cvt zero is_nan from_ty))) 2538 2539;;;;; Rules for `fcvt_from_sint`;;;;;;;;; 2540(rule 0 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type (fits_in_16 ty))))) 2541 (rv_fcvtsl (FRM.RNE) (sext v))) 2542 2543(rule 1 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type $I32)))) 2544 (rv_fcvtsw (FRM.RNE) v)) 2545 2546(rule 1 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type $I64)))) 2547 (rv_fcvtsl (FRM.RNE) v)) 2548 2549(rule 0 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type (fits_in_16 ty))))) 2550 (rv_fcvtdl (FRM.RNE) (sext v))) 2551 2552(rule 1 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type $I32)))) 2553 (rv_fcvtdw v)) 2554 2555(rule 1 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type $I64)))) 2556 (rv_fcvtdl (FRM.RNE) v)) 2557 2558(rule 2 (lower (has_type (ty_supported_vec _) (fcvt_from_sint _ v @ (value_type from_ty)))) 2559 (rv_vfcvt_f_x_v v (unmasked) from_ty)) 2560 2561;;;;; Rules for `fcvt_from_uint`;;;;;;;;; 2562(rule 0 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type (fits_in_16 ty))))) 2563 (rv_fcvtslu (FRM.RNE) (zext v))) 2564 2565(rule 1 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type $I32)))) 2566 (rv_fcvtswu (FRM.RNE) v)) 2567 2568(rule 1 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type $I64)))) 2569 (rv_fcvtslu (FRM.RNE) v)) 2570 2571(rule 0 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type (fits_in_16 ty))))) 2572 (rv_fcvtdlu (FRM.RNE) (zext v))) 2573 2574(rule 1 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type $I32)))) 2575 (rv_fcvtdwu v)) 2576 2577(rule 1 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type $I64)))) 2578 (rv_fcvtdlu (FRM.RNE) v)) 2579 2580(rule 2 (lower (has_type (ty_supported_vec _) (fcvt_from_uint _ v @ (value_type from_ty)))) 2581 (rv_vfcvt_f_xu_v v (unmasked) from_ty)) 2582 2583;;;;; Rules for `symbol_value`;;;;;;;;; 2584(rule 2585 (lower (symbol_value _ (symbol_value_data name dist offset))) 2586 (load_ext_name name offset dist)) 2587 2588;;;;; Rules for `tls_value` ;;;;;;;;;;;;;; 2589 2590(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value _ (symbol_value_data name _ _)))) 2591 (elf_tls_get_addr name)) 2592 2593;;;;; Rules for `bitcast`;;;;;;;;; 2594 2595;; These rules should probably be handled in `gen_bitcast`, but it's convenient to have that return 2596;; a single register, instead of a `ValueRegs` 2597(rule 3 (lower (has_type (ty_reg_pair _) (bitcast _ _ v @ (value_type (ty_supported_vec _))))) 2598 (value_regs 2599 (gen_extractlane $I64X2 v 0) 2600 (gen_extractlane $I64X2 v 1))) 2601 2602;; Move the high half into a vector register, and then use vslide1up to move it up and 2603;; insert the lower half in one instruction. 2604(rule 2 (lower (has_type (ty_supported_vec _) (bitcast _ _ v @ (value_type (ty_reg_pair _))))) 2605 (let ((lo XReg (value_regs_get v 0)) 2606 (hi XReg (value_regs_get v 1)) 2607 (vstate VState (vstate_from_type $I64X2)) 2608 (vec VReg (rv_vmv_sx hi vstate))) 2609 (rv_vslide1up_vx vec vec lo (unmasked) vstate))) 2610 2611;; `gen_bitcast` below only works with single register values, so handle I128 2612;; and F128 specially here. 2613(rule 1 (lower (has_type (ty_reg_pair _) (bitcast _ _ v @ (value_type (ty_reg_pair _))))) 2614 v) 2615 2616(rule 0 (lower (has_type out_ty (bitcast _ _ v @ (value_type in_ty)))) 2617 (gen_bitcast v in_ty out_ty)) 2618 2619;;;;; Rules for `ceil`;;;;;;;;; 2620(rule 0 (lower (has_type (ty_supported_float_full ty) (ceil _ x))) 2621 (gen_float_round (FRM.RUP) x ty)) 2622 2623(rule 1 (lower (has_type (ty_supported_vec ty) (ceil _ x))) 2624 (gen_vec_round x (FRM.RUP) ty)) 2625 2626;;;;; Rules for `floor`;;;;;;;;; 2627(rule 0 (lower (has_type (ty_supported_float_full ty) (floor _ x))) 2628 (gen_float_round (FRM.RDN) x ty)) 2629 2630(rule 1 (lower (has_type (ty_supported_vec ty) (floor _ x))) 2631 (gen_vec_round x (FRM.RDN) ty)) 2632 2633;;;;; Rules for `trunc`;;;;;;;;; 2634(rule 0 (lower (has_type (ty_supported_float_full ty) (trunc _ x))) 2635 (gen_float_round (FRM.RTZ) x ty)) 2636 2637(rule 1 (lower (has_type (ty_supported_vec ty) (trunc _ x))) 2638 (gen_vec_round x (FRM.RTZ) ty)) 2639 2640;;;;; Rules for `nearest`;;;;;;;;; 2641(rule 0 (lower (has_type (ty_supported_float_full ty) (nearest _ x))) 2642 (gen_float_round (FRM.RNE) x ty)) 2643 2644(rule 1 (lower (has_type (ty_supported_vec ty) (nearest _ x))) 2645 (gen_vec_round x (FRM.RNE) ty)) 2646 2647 2648;;;;; Rules for `select_spectre_guard`;;;;;;;;; 2649 2650;; SelectSpectreGuard is equivalent to Select, but we should not use a branch based 2651;; lowering for it. Instead we use a conditional move based lowering. 2652;; 2653;; We don't have cmov's in RISC-V either, but we can emulate those using bitwise 2654;; operations, which is what we do below. 2655 2656;; Base case: use `gen_bmask` to generate a 0 mask or -1 mask from the value of 2657;; `cmp`. This is then used with some bit twiddling to produce the final result. 2658(rule 0 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp x y))) 2659 (let ((mask XReg (gen_bmask cmp))) 2660 (rv_or (rv_and mask x) (rv_andn y mask)))) 2661(rule 1 (lower (has_type $I128 (select_spectre_guard _ cmp x y))) 2662 (let ((mask XReg (gen_bmask cmp))) 2663 (value_regs 2664 (rv_or (rv_and mask (value_regs_get x 0)) (rv_andn (value_regs_get y 0) mask)) 2665 (rv_or (rv_and mask (value_regs_get x 1)) (rv_andn (value_regs_get y 1) mask))))) 2666 2667;; Special case when an argument is the constant zero as some ands and ors 2668;; can be folded away. 2669(rule 2 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp (i64_from_iconst 0) y))) 2670 (rv_andn y (gen_bmask cmp))) 2671(rule 3 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp x (i64_from_iconst 0)))) 2672 (rv_and x (gen_bmask cmp))) 2673 2674;;;;; Rules for `bmask`;;;;;;;;; 2675(rule 2676 (lower (has_type oty (bmask _ x))) 2677 (lower_bmask x oty)) 2678 2679;; N.B.: the Ret itself is generated by the ABI. 2680(rule (lower (return args)) 2681 (lower_return args)) 2682 2683;;; Rules for `get_{frame,stack}_pointer` and `get_return_address` ;;;;;;;;;;;;; 2684 2685(rule (lower (get_frame_pointer _)) 2686 (gen_mov_from_preg (fp_reg))) 2687 2688(rule (lower (get_stack_pointer _)) 2689 (gen_mov_from_preg (sp_reg))) 2690 2691(rule (lower (get_return_address _)) 2692 (load_ra)) 2693 2694;;; Rules for `iabs` ;;;;;;;;;;;;; 2695 2696;; I64 and lower 2697;; Generate the following code: 2698;; sext.{b,h,w} a0, a0 2699;; neg a1, a0 2700;; max a0, a0, a1 2701(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (iabs _ x))) 2702 (let ((extended XReg (sext x)) 2703 (negated XReg (rv_neg extended))) 2704 (gen_select_xreg (cmp_gt extended negated) extended negated))) 2705 2706;; For vectors we generate the same code, but with vector instructions 2707;; we can skip the sign extension, since the vector unit will only process 2708;; Element Sized chunks. 2709(rule 1 (lower (has_type (ty_supported_vec ty) (iabs _ x))) 2710 (let ((negated VReg (rv_vneg_v x (unmasked) ty))) 2711 (rv_vmax_vv x negated (unmasked) ty))) 2712 2713;;;; Rules for calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2714 2715;; Direct call to an in-range function. 2716(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near) patchable) args)) 2717 (let ((output ValueRegsVec (gen_call_output sig_ref)) 2718 (abi Sig (abi_sig sig_ref)) 2719 (uses CallArgList (gen_call_args abi args)) 2720 (defs CallRetList (gen_call_rets abi output)) 2721 (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none) patchable)) 2722 (_ Unit (emit_side_effect (call_impl info)))) 2723 output)) 2724 2725;; Direct call to an out-of-range function (implicitly via pointer). 2726(rule (lower (call (func_ref_data sig_ref name dist false) args)) 2727 (let ((output ValueRegsVec (gen_call_output sig_ref)) 2728 (abi Sig (abi_sig sig_ref)) 2729 (uses CallArgList (gen_call_args abi args)) 2730 (defs CallRetList (gen_call_rets abi output)) 2731 (target Reg (load_ext_name name 0 dist)) 2732 (info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none))) 2733 (_ Unit (emit_side_effect (call_ind_impl info)))) 2734 output)) 2735 2736;; Indirect call. 2737(rule (lower (call_indirect sig_ref ptr args)) 2738 (let ((output ValueRegsVec (gen_call_output sig_ref)) 2739 (abi Sig (abi_sig sig_ref)) 2740 (target Reg (put_in_reg ptr)) 2741 (uses CallArgList (gen_call_args abi args)) 2742 (defs CallRetList (gen_call_rets abi output)) 2743 (info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none))) 2744 (_ Unit (emit_side_effect (call_ind_impl info)))) 2745 output)) 2746 2747;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2748 2749;; Direct call to an in-range function. 2750(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (RelocDistance.Near) patchable) args et) targets) 2751 (let ((abi Sig (abi_sig sig_ref)) 2752 (trycall OptionTryCallInfo (try_call_info et targets)) 2753 (uses CallArgList (gen_call_args abi args)) 2754 (defs CallRetList (gen_try_call_rets abi)) 2755 (info BoxCallInfo (gen_call_info abi name uses defs trycall patchable))) 2756 (emit_side_effect (call_impl info)))) 2757 2758;; Direct call to an out-of-range function (implicitly via pointer). 2759(rule (lower_branch (try_call (func_ref_data sig_ref name dist false) args et) targets) 2760 (let ((abi Sig (abi_sig sig_ref)) 2761 (trycall OptionTryCallInfo (try_call_info et targets)) 2762 (uses CallArgList (gen_call_args abi args)) 2763 (defs CallRetList (gen_try_call_rets abi)) 2764 (target Reg (load_ext_name name 0 dist)) 2765 (info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall))) 2766 (emit_side_effect (call_ind_impl info)))) 2767 2768;; Indirect call. 2769(rule (lower_branch (try_call_indirect ptr args et) targets) 2770 (if-let (exception_sig sig_ref) et) 2771 (let ((abi Sig (abi_sig sig_ref)) 2772 (trycall OptionTryCallInfo (try_call_info et targets)) 2773 (target Reg (put_in_reg ptr)) 2774 (uses CallArgList (gen_call_args abi args)) 2775 (defs CallRetList (gen_try_call_rets abi)) 2776 (info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall))) 2777 (emit_side_effect (call_ind_impl info)))) 2778 2779;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;; 2780 2781;; Direct call to an in-range function. 2782(rule 1 (lower (return_call (func_ref_data sig_ref name (RelocDistance.Near) false) args)) 2783 (let ((abi Sig (abi_sig sig_ref)) 2784 (uses CallArgList (gen_return_call_args abi args)) 2785 (info BoxReturnCallInfo (gen_return_call_info abi name uses))) 2786 (side_effect (return_call_impl info)))) 2787 2788;; Direct call to an out-of-range function (implicitly via pointer). 2789(rule (lower (return_call (func_ref_data sig_ref name dist false) args)) 2790 (let ((abi Sig (abi_sig sig_ref)) 2791 (uses CallArgList (gen_return_call_args abi args)) 2792 (target Reg (load_ext_name name 0 dist)) 2793 (info BoxReturnCallIndInfo (gen_return_call_ind_info abi target uses))) 2794 (side_effect (return_call_ind_impl info)))) 2795 2796;; Indirect call. 2797(rule (lower (return_call_indirect sig_ref ptr args)) 2798 (let ((abi Sig (abi_sig sig_ref)) 2799 (target Reg (put_in_reg ptr)) 2800 (uses CallArgList (gen_return_call_args abi args)) 2801 (info BoxReturnCallIndInfo (gen_return_call_ind_info abi target uses))) 2802 (side_effect (return_call_ind_impl info)))) 2803 2804;;;; Rules for `extractlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2805 2806(rule (lower (extractlane _ x @ (value_type ty) (u8_from_uimm8 idx))) 2807 (gen_extractlane ty x idx)) 2808 2809;;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2810 2811;; We can insert a lane by using a masked splat from an X register. 2812;; Build a mask that is only enabled in the lane we want to insert. 2813;; Then use a masked splat (vmerge) to insert the value. 2814(rule 0 (lower (insertlane _ vec @ (value_type (ty_supported_vec ty)) 2815 val @ (value_type (ty_int _)) 2816 (u8_from_uimm8 lane))) 2817 (let ((mask VReg (gen_vec_mask (u64_wrapping_shl 1 lane)))) 2818 (rv_vmerge_vxm vec val mask ty))) 2819 2820;; Similar to above, but using the float variants of the instructions. 2821(rule 1 (lower (insertlane _ vec @ (value_type (ty_supported_vec ty)) 2822 val @ (value_type (ty_supported_float_full _)) 2823 (u8_from_uimm8 lane))) 2824 (let ((mask VReg (gen_vec_mask (u64_wrapping_shl 1 lane)))) 2825 (rv_vfmerge_vfm vec val mask ty))) 2826 2827;; If we are inserting from an Imm5 const we can use the immediate 2828;; variant of vmerge. 2829(rule 2 (lower (insertlane _ vec @ (value_type (ty_supported_vec ty)) 2830 (i64_from_iconst (imm5_from_i64 imm)) 2831 (u8_from_uimm8 lane))) 2832 (let ((mask VReg (gen_vec_mask (u64_wrapping_shl 1 lane)))) 2833 (rv_vmerge_vim vec imm mask ty))) 2834 2835;;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2836 2837(rule 0 (lower (has_type ty (splat _ n @ (value_type (ty_supported_float_full _))))) 2838 (rv_vfmv_vf n ty)) 2839 2840(rule 1 (lower (has_type ty (splat _ n @ (value_type (ty_int_ref_scalar_64 _))))) 2841 (rv_vmv_vx n ty)) 2842 2843(rule 2 (lower (has_type ty (splat _ (iconst _ (u64_from_imm64 (imm5_from_u64 imm)))))) 2844 (rv_vmv_vi imm ty)) 2845 2846;; TODO: We can splat out more patterns by using for example a vmv.v.i i8x16 for 2847;; a i64x2 const with a compatible bit pattern. The AArch64 Backend does something 2848;; similar in its splat rules. 2849;; TODO: Look through bitcasts when splatting out registers. We can use 2850;; `vmv.v.x` in a `(splat.f32x4 (bitcast.f32 val))`. And vice versa for integers. 2851 2852;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2853 2854(rule 0 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y))) 2855 (rv_vsaddu_vv x y (unmasked) ty)) 2856 2857(rule 1 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x (splat _ y)))) 2858 (rv_vsaddu_vx x y (unmasked) ty)) 2859 2860(rule 2 (lower (has_type (ty_supported_vec ty) (uadd_sat _ (splat _ x) y))) 2861 (rv_vsaddu_vx y x (unmasked) ty)) 2862 2863(rule 3 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y))) 2864 (if-let y_imm (replicated_imm5 y)) 2865 (rv_vsaddu_vi x y_imm (unmasked) ty)) 2866 2867(rule 4 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y))) 2868 (if-let x_imm (replicated_imm5 x)) 2869 (rv_vsaddu_vi y x_imm (unmasked) ty)) 2870 2871;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2872 2873(rule 0 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y))) 2874 (rv_vsadd_vv x y (unmasked) ty)) 2875 2876(rule 1 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x (splat _ y)))) 2877 (rv_vsadd_vx x y (unmasked) ty)) 2878 2879(rule 2 (lower (has_type (ty_supported_vec ty) (sadd_sat _ (splat _ x) y))) 2880 (rv_vsadd_vx y x (unmasked) ty)) 2881 2882(rule 3 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y))) 2883 (if-let y_imm (replicated_imm5 y)) 2884 (rv_vsadd_vi x y_imm (unmasked) ty)) 2885 2886(rule 4 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y))) 2887 (if-let x_imm (replicated_imm5 x)) 2888 (rv_vsadd_vi y x_imm (unmasked) ty)) 2889 2890;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2891 2892(rule 0 (lower (has_type (ty_supported_vec ty) (usub_sat _ x y))) 2893 (rv_vssubu_vv x y (unmasked) ty)) 2894 2895(rule 1 (lower (has_type (ty_supported_vec ty) (usub_sat _ x (splat _ y)))) 2896 (rv_vssubu_vx x y (unmasked) ty)) 2897 2898;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2899 2900(rule 0 (lower (has_type (ty_supported_vec ty) (ssub_sat _ x y))) 2901 (rv_vssub_vv x y (unmasked) ty)) 2902 2903(rule 1 (lower (has_type (ty_supported_vec ty) (ssub_sat _ x (splat _ y)))) 2904 (rv_vssub_vx x y (unmasked) ty)) 2905 2906;;;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2907 2908;; Here we do a Vector Reduce operation. Get the unsigned minimum value of any 2909;; lane in the vector. The fixed input to the reduce operation is a 1. 2910;; This way, if any lane is 0, the result will be 0. Otherwise, the result will 2911;; be a 1. 2912;; The reduce operation leaves the result in the lowest lane, we then move it 2913;; into the destination X register. 2914(rule (lower (vall_true _ x @ (value_type (ty_supported_vec ty)))) 2915 (if-let one (i8_to_imm5 1)) 2916 ;; We don't need to broadcast the immediate into all lanes, only into lane 0. 2917 ;; I did it this way since it uses one less instruction than with a vmv.s.x. 2918 (let ((fixed VReg (rv_vmv_vi one ty)) 2919 (min VReg (rv_vredminu_vs x fixed (unmasked) ty))) 2920 (rv_vmv_xs min ty))) 2921 2922 2923;;;; Rules for `vany_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2924 2925;; Here we do a Vector Reduce operation. Get the unsigned maximum value of the 2926;; input vector register. Move the max to an X register, and do a `snez` on it 2927;; to ensure its either 1 or 0. 2928(rule (lower (vany_true _ x @ (value_type (ty_supported_vec ty)))) 2929 (let ((max VReg (rv_vredmaxu_vs x x (unmasked) ty)) 2930 (x_max XReg (rv_vmv_xs max ty))) 2931 (rv_snez x_max))) 2932 2933 2934;;;; Rules for `vhigh_bits` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2935 2936;; To check if the MSB of a lane is set, we do a `vmslt` with zero, this sets 2937;; the mask bit to 1 if the value is negative (MSB 1) and 0 if not. We can then 2938;; just move that mask to an X Register. 2939;; 2940;; We must ensure that the move to the X register has a SEW with enough bits 2941;; to hold the full mask. Additionally, in some cases (e.g. i64x2) we are going 2942;; to read some tail bits. These are undefined, so we need to further mask them 2943;; off. 2944(rule (lower (vhigh_bits _ x @ (value_type (ty_supported_vec ty)))) 2945 (let ((mask VReg (rv_vmslt_vx x (zero_reg) (unmasked) ty)) 2946 ;; Here we only need I64X1, but emit an AVL of 2 since it 2947 ;; saves one vector state change in the case of I64X2. 2948 ;; 2949 ;; TODO: For types that have more lanes than element bits, we can 2950 ;; use the original type as a VState and avoid a state change. 2951 (x_mask XReg (rv_vmv_xs mask (vstate_from_type $I64X2)))) 2952 (gen_andi x_mask (ty_lane_mask ty)))) 2953 2954;;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2955 2956(rule 0 (lower (has_type (ty_supported_vec ty) (swizzle _ x y))) 2957 (rv_vrgather_vv x y (unmasked) ty)) 2958 2959(rule 1 (lower (has_type (ty_supported_vec ty) (swizzle _ x (splat _ y)))) 2960 (rv_vrgather_vx x y (unmasked) ty)) 2961 2962(rule 2 (lower (has_type (ty_supported_vec ty) (swizzle _ x y))) 2963 (if-let y_imm (replicated_uimm5 y)) 2964 (rv_vrgather_vi x y_imm (unmasked) ty)) 2965 2966;;;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2967 2968;; Use a vrgather to load all 0-15 lanes from x. And then modify the mask to load all 2969;; 16-31 lanes from y. Finally, use a vor to combine the two vectors. 2970;; 2971;; vrgather will insert a 0 for lanes that are out of bounds, so we can let it load 2972;; negative and out of bounds indexes. 2973(rule (lower (has_type (ty_supported_vec ty @ $I8X16) (shuffle _ x y (vconst_from_immediate mask)))) 2974 (if-let neg16 (i8_to_imm5 -16)) 2975 (let ((x_mask VReg (gen_constant ty mask)) 2976 (x_lanes VReg (rv_vrgather_vv x x_mask (unmasked) ty)) 2977 (y_mask VReg (rv_vadd_vi x_mask neg16 (unmasked) ty)) 2978 (y_lanes VReg (rv_vrgather_vv y y_mask (unmasked) ty))) 2979 (rv_vor_vv x_lanes y_lanes (unmasked) ty))) 2980 2981;;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2982 2983;; Slide down half the vector, and do a signed extension. 2984(rule 0 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ x @ (value_type in_ty)))) 2985 (rv_vsext_vf2 (gen_slidedown_half in_ty x) (unmasked) out_ty)) 2986 2987(rule 1 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ (swiden_high _ x @ (value_type in_ty))))) 2988 (if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty))) 2989 (rv_vsext_vf4 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty)) 2990 2991(rule 2 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ (swiden_high _ (swiden_high _ x @ (value_type in_ty)))))) 2992 (if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty))) 2993 (rv_vsext_vf8 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty)) 2994 2995;;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2996 2997;; Slide down half the vector, and do a zero extension. 2998(rule 0 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ x @ (value_type in_ty)))) 2999 (rv_vzext_vf2 (gen_slidedown_half in_ty x) (unmasked) out_ty)) 3000 3001(rule 1 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ (uwiden_high _ x @ (value_type in_ty))))) 3002 (if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty))) 3003 (rv_vzext_vf4 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty)) 3004 3005(rule 2 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ (uwiden_high _ (uwiden_high _ x @ (value_type in_ty)))))) 3006 (if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty))) 3007 (rv_vzext_vf8 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty)) 3008 3009;;;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3010 3011(rule 0 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ x))) 3012 (rv_vsext_vf2 x (unmasked) out_ty)) 3013 3014(rule 1 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ (swiden_low _ x)))) 3015 (rv_vsext_vf4 x (unmasked) out_ty)) 3016 3017(rule 2 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ (swiden_low _ (swiden_low _ x))))) 3018 (rv_vsext_vf8 x (unmasked) out_ty)) 3019 3020;;;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3021 3022(rule 0 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ x))) 3023 (rv_vzext_vf2 x (unmasked) out_ty)) 3024 3025(rule 1 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ (uwiden_low _ x)))) 3026 (rv_vzext_vf4 x (unmasked) out_ty)) 3027 3028(rule 2 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ (uwiden_low _ (uwiden_low _ x))))) 3029 (rv_vzext_vf8 x (unmasked) out_ty)) 3030 3031;;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3032 3033;; We don't have a dedicated instruction for this, rearrange the register elements 3034;; and use a vadd. 3035;; 3036;; We do this by building two masks, one for the even elements and one for the odd 3037;; elements. Using vcompress we can extract the elements and group them together. 3038;; 3039;; This is likely not the optimal way of doing this. LLVM does this using a bunch 3040;; of vrgathers (See: https://godbolt.org/z/jq8Wj8WG4), that doesn't seem to be 3041;; too much better than this. 3042;; 3043;; However V8 does something better. They use 2 vcompresses using LMUL2, that means 3044;; that they can do the whole thing in 3 instructions (2 vcompress + vadd). We don't 3045;; support LMUL > 1, so we can't do that. 3046(rule (lower (has_type (ty_supported_vec ty) (iadd_pairwise _ x y))) 3047 (if-let half_size (u64_to_uimm5 (u64_checked_div (ty_lane_count ty) 2))) 3048 (let ((odd_mask VReg (gen_vec_mask 0x5555555555555555)) 3049 (lhs_lo VReg (rv_vcompress_vm x odd_mask ty)) 3050 (lhs_hi VReg (rv_vcompress_vm y odd_mask ty)) 3051 (lhs VReg (rv_vslideup_vvi lhs_lo lhs_hi half_size (unmasked) ty)) 3052 3053 (even_mask VReg (gen_vec_mask 0xAAAAAAAAAAAAAAAA)) 3054 (rhs_lo VReg (rv_vcompress_vm x even_mask ty)) 3055 (rhs_hi VReg (rv_vcompress_vm y even_mask ty)) 3056 (rhs VReg (rv_vslideup_vvi rhs_lo rhs_hi half_size (unmasked) ty))) 3057 (rv_vadd_vv lhs rhs (unmasked) ty))) 3058 3059;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3060 3061;; `avg_round` computes the unsigned average with rounding: a := (x + y + 1) // 2 3062;; 3063;; See Section "2–5 Average of Two Integers" of the Hacker's Delight book 3064;; 3065;; The floor average of two integers without overflow can be computed as: 3066;; t = (x & y) + ((x ^ y) >> 1) 3067;; 3068;; The right shift should be a logical shift if the integers are unsigned. 3069;; 3070;; We are however interested in the ceiling average (x + y + 1). For that 3071;; we use a special rounding mode in the right shift instruction. 3072;; 3073;; For the right shift instruction we use `vssrl` which is a Scaling Shift 3074;; Right Logical instruction using the `vxrm` fixed-point rounding mode. The 3075;; default rounding mode is `rnu` (round-to-nearest-up (add +0.5 LSB)). 3076;; Which is coincidentally the rounding mode we want for `avg_round`. 3077(rule (lower (has_type (ty_supported_vec ty) (avg_round _ x y))) 3078 (if-let one (u64_to_uimm5 1)) 3079 (let ((lhs VReg (rv_vand_vv x y (unmasked) ty)) 3080 (xor VReg (rv_vxor_vv x y (unmasked) ty)) 3081 (rhs VReg (rv_vssrl_vi xor one (unmasked) ty))) 3082 (rv_vadd_vv lhs rhs (unmasked) ty))) 3083 3084;;;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3085 3086(rule 0 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ x))) 3087 (if (ty_vector_float ty)) 3088 (let ((zero VReg (rv_vmv_vx (zero_reg) ty)) 3089 (elem VReg (rv_vfmv_sf x ty)) 3090 (mask VReg (gen_vec_mask 1))) 3091 (rv_vmerge_vvm zero elem mask ty))) 3092 3093(rule 1 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ x))) 3094 (if (ty_vector_not_float ty)) 3095 (let ((zero VReg (rv_vmv_vx (zero_reg) ty)) 3096 (mask VReg (gen_vec_mask 1))) 3097 (rv_vmerge_vxm zero x mask ty))) 3098 3099(rule 2 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ (imm5_from_value x)))) 3100 (let ((zero VReg (rv_vmv_vx (zero_reg) ty)) 3101 (mask VReg (gen_vec_mask 1))) 3102 (rv_vmerge_vim zero x mask ty))) 3103 3104;;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3105 3106(rule 0 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ x y))) 3107 (rv_vsmul_vv x y (unmasked) ty)) 3108 3109(rule 1 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ x (splat _ y)))) 3110 (rv_vsmul_vx x y (unmasked) ty)) 3111 3112(rule 2 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ (splat _ x) y))) 3113 (rv_vsmul_vx y x (unmasked) ty)) 3114 3115;;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3116 3117(rule (lower (has_type (ty_supported_vec out_ty) (snarrow _ x @ (value_type in_ty) y))) 3118 (if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2))) 3119 (if-let zero (u64_to_uimm5 0)) 3120 (let ((x_clip VReg (rv_vnclip_wi x zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty)))) 3121 (y_clip VReg (rv_vnclip_wi y zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty))))) 3122 (rv_vslideup_vvi x_clip y_clip lane_diff (unmasked) out_ty))) 3123 3124;;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3125 3126(rule (lower (has_type (ty_supported_vec out_ty) (uunarrow _ x @ (value_type in_ty) y))) 3127 (if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2))) 3128 (if-let zero (u64_to_uimm5 0)) 3129 (let ((x_clip VReg (rv_vnclipu_wi x zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty)))) 3130 (y_clip VReg (rv_vnclipu_wi y zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty))))) 3131 (rv_vslideup_vvi x_clip y_clip lane_diff (unmasked) out_ty))) 3132 3133;;;; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3134 3135;; We don't have a instruction that saturates a signed source into an unsigned destination. 3136;; To correct for this we just remove negative values using `vmax` and then use the normal 3137;; unsigned to unsigned narrowing instruction. 3138 3139(rule (lower (has_type (ty_supported_vec out_ty) (unarrow _ x @ (value_type in_ty) y))) 3140 (if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2))) 3141 (if-let zero (u64_to_uimm5 0)) 3142 (let ((x_pos VReg (rv_vmax_vx x (zero_reg) (unmasked) in_ty)) 3143 (y_pos VReg (rv_vmax_vx y (zero_reg) (unmasked) in_ty)) 3144 (x_clip VReg (rv_vnclipu_wi x_pos zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty)))) 3145 (y_clip VReg (rv_vnclipu_wi y_pos zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty))))) 3146 (rv_vslideup_vvi x_clip y_clip lane_diff (unmasked) out_ty))) 3147 3148 3149;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3150 3151(rule (lower (get_exception_handler_address _ (u64_from_imm64 idx) block)) 3152 (let ((succ_label MachLabel (block_exn_successor_label block idx))) 3153 (rv64_label_address succ_label))) 3154 3155;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 3156 3157(rule (lower (sequence_point)) 3158 (side_effect 3159 (rv64_sequence_point))) 3160