Merge with upstream

cranelift/codegen/build.rs

@@ -228,6 +228,7 @@ fn get_isle_compilations(
                     src_opts.join("icmp.isle"),
                     src_opts.join("remat.isle"),
                     src_opts.join("selects.isle"),
+                    src_opts.join("spaceship.isle"),
                     src_opts.join("shifts.isle"),
                     src_opts.join("vector.isle"),
                 ],

cranelift/codegen/src/opts/arithmetic.isle

@@ -20,6 +20,17 @@
                       x))
       (ineg ty x))
 
+;; x + -y == -y + x == -(y - x) == x - y
+(rule (simplify (iadd ty x (ineg ty y)))
+      (isub ty x y))
+(rule (simplify (iadd ty (ineg ty y) x))
+      (isub ty x y))
+(rule (simplify (ineg ty (isub ty y x)))
+      (isub ty x y))
+;; x - -y == x + y
+(rule (simplify (isub ty x (ineg ty y)))
+      (iadd ty x y))
+
 ;; ineg(ineg(x)) == x.
 (rule (simplify (ineg ty (ineg ty x))) (subsume x))
 
@@ -55,24 +66,16 @@
       (if-let -1 (i64_sextend_imm64 ty c))
       (ineg ty x))
 
-;; (!x) + 1 == 1 + (!x) == !(x) - (-1) == ineg(x)
+;; (!x) + 1 == ineg(x)
 (rule (simplify (iadd ty (bnot ty x) (iconst ty (u64_from_imm64 1))))
       (ineg ty x))
-(rule (simplify (iadd ty (iconst ty (u64_from_imm64 1)) (bnot ty x)))
-      (ineg ty x))
-(rule (simplify (isub ty (bnot ty x) (iconst ty c)))
-      (if-let -1 (i64_sextend_imm64 ty c))
-      (ineg ty x))
 
-;; !(x - 1) == !(x + (-1)) == !((-1) + x) == ineg(x)
+;; !(x - 1) == !(x + (-1)) == ineg(x)
 (rule (simplify (bnot ty (isub ty x (iconst ty (u64_from_imm64 1)))))
       (ineg ty x))
 (rule (simplify (bnot ty (iadd ty x (iconst ty c))))
       (if-let -1 (i64_sextend_imm64 ty c))
       (ineg ty x))
-(rule (simplify (bnot ty (iadd ty (iconst ty c) x)))
-      (if-let -1 (i64_sextend_imm64 ty c))
-      (ineg ty x))
 
 ;; x/1 == x.
 (rule (simplify (sdiv ty

cranelift/codegen/src/opts/bitops.isle

@@ -1,26 +1,18 @@
 ;; Rewrites for `band`, `bnot`, `bor`, `bxor`
 
-;; x | 0 == 0 | x == x | x == x.
+;; x | 0 == x | x == x.
 (rule (simplify (bor ty
                      x
                      (iconst ty (u64_from_imm64 0))))
       (subsume x))
-(rule (simplify (bor ty
-                     (iconst ty (u64_from_imm64 0))
-                     x))
-      (subsume x))
 (rule (simplify (bor ty x x))
       (subsume x))
 
-;; x ^ 0 == 0 ^ x == x.
+;; x ^ 0 == x.
 (rule (simplify (bxor ty
                      x
                      (iconst ty (u64_from_imm64 0))))
       (subsume x))
-(rule (simplify (bxor ty
-                     (iconst ty (u64_from_imm64 0))
-                     x))
-      (subsume x))
 
 ;; x ^ x == 0.
 (rule (simplify (bxor (fits_in_64 (ty_int ty)) x x))
@@ -34,18 +26,14 @@
 (rule (simplify (bor (fits_in_64 (ty_int ty)) x (bnot ty x))) (subsume (iconst ty (imm64 (ty_mask ty)))))
 (rule (simplify (bor (fits_in_64 (ty_int ty)) (bnot ty x) x)) (subsume (iconst ty (imm64 (ty_mask ty)))))
 
-;; x & -1 == -1 & x == x & x == x.
+;; x & x == x & -1 == x.
 (rule (simplify (band ty x x)) (subsume x))
 (rule (simplify (band ty x (iconst ty k)))
       (if-let -1 (i64_sextend_imm64 ty k))
       (subsume x))
-(rule (simplify (band ty (iconst ty k) x))
-      (if-let -1 (i64_sextend_imm64 ty k))
-      (subsume x))
 
-;; x & 0 == 0 & x == x & not(x) == not(x) & x == 0.
+;; x & 0 == x & not(x) == not(x) & x == 0.
 (rule (simplify (band ty _ zero @ (iconst ty (u64_from_imm64 0)))) (subsume zero))
-(rule (simplify (band ty zero @ (iconst ty (u64_from_imm64 0)) _)) (subsume zero))
 (rule (simplify (band (fits_in_64 (ty_int ty)) x (bnot ty x))) (subsume (iconst ty (imm64 0))))
 (rule (simplify (band (fits_in_64 (ty_int ty)) (bnot ty x) x)) (subsume (iconst ty (imm64 0))))
 

cranelift/codegen/src/opts/cprop.isle

@@ -173,6 +173,17 @@
        (select ty (iconst _ (u64_from_imm64 0)) x y))
       y)
 
+;; Replace subtraction by a "negative" constant with addition.
+;; Notably, this gives `x - (-1) == x + 1`, so other patterns don't have to
+;; match the subtract-negative-one version too.
+;; TODO: it would be nice to do this for `x + (-1) == x - 1` as well, but
+;; that needs work in lowering first to avoid regressing addressing modes.
+
+(rule (simplify (isub ty x (iconst ty k1)))
+      (if-let k2 (i64_sextend_imm64 ty k1))
+      (if-let $true (u64_lt (i64_as_u64 (i64_neg k2)) (i64_as_u64 k2)))
+      (iadd ty x (iconst ty (imm64_masked ty (i64_as_u64 (i64_neg k2))))))
+
 ;; TODO: fadd, fsub, fmul, fdiv, fneg, fabs
 
 ;; A splat of a constant can become a direct `vconst` with the appropriate bit

cranelift/codegen/src/opts/extends.isle

@@ -4,6 +4,14 @@
 (rule (simplify (sextend ty (sextend _intermediate_ty x)))
       (sextend ty x))
 
+;; Once something has be `uextend`ed, further `sextend`ing is the same as `uextend`
+(rule (simplify (sextend ty (uextend _intermediate_ty x)))
+      (uextend ty x))
+
+;; `icmp` is zero or one, so sign-extending is the same as zero-extending
+(rule (simplify (sextend ty x@(icmp _ _ _ _)))
+      (uextend ty x))
+
 ;; Masking out any of the top bits of the result of `uextend` is a no-op. (This
 ;; is like a cheap version of known-bits analysis.)
 (rule (simplify (band wide x @ (uextend _ (value_type narrow)) (iconst _ (u64_from_imm64 mask))))
@@ -28,6 +36,15 @@
              (iconst _ (u64_from_imm64 0))))
       (iconst ty (imm64 1)))
 
+;; Sign-extending can't change whether a number is zero nor how it signed-compares to zero
+(rule (simplify (eq _ (sextend _ x@(value_type ty)) (iconst _ (u64_from_imm64 0))))
+      (eq ty x (iconst ty (imm64 0))))
+(rule (simplify (ne _ (sextend _ x@(value_type ty)) (iconst _ (u64_from_imm64 0))))
+      (ne ty x (iconst ty (imm64 0))))
+(rule (simplify (icmp _ cc (sextend _ x@(value_type ty)) (iconst _ (u64_from_imm64 0))))
+      (if (signed_cond_code cc))
+      (icmp ty cc x (iconst ty (imm64 0))))
+
 ;; A reduction-of-an-extend back to the same original type is the same as not
 ;; actually doing the extend in the first place.
 (rule (simplify (ireduce ty (sextend _ x @ (value_type ty)))) x)

cranelift/codegen/src/opts/spaceship.isle

@@ -0,0 +1,237 @@
+;; Simplifications for C++20's `<=>` "spaceship" operator, aka Rust's `Ord::cmp`.
+;;
+;; There's no cranelift instruction for this, nor usually a machine instruction.
+;; Inspired by <https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign>
+;; we canonicalize the various implementations of `x <=> y` to `(x > y) - (x < y)`.
+
+;; Unfortunately, there's at least 3!×2 reasonable ways to write this as nested
+;; selects, and no broad agreement which is the best -- notably Rust 1.74 and
+;; Clang 17 use different sequences -- so we just match all of them.
+
+;; x < y ? -1 : x == y ? 0 : +1
+;; x < y ? -1 : x != y ? +1 : 0
+(rule (simplify (select ty (ult rty x y)
+                           (iconst ty neg_1)
+                           (ne rty x y)))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+(rule (simplify (select ty (ult rty x y)
+                           (iconst ty neg_1)
+                           (uextend ty (ne rty x y))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+;; x < y ? -1 : x <= y ? 0 : +1
+;; x < y ? -1 : x > y ? +1 : 0
+(rule (simplify (select ty (ult rty x y)
+                           (iconst ty neg_1)
+                           (ugt rty x y)))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+(rule (simplify (select ty (ult rty x y)
+                           (iconst ty neg_1)
+                           (uextend ty (ugt rty x y))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+
+;; x == y ? 0 : x < y ? -1 : +1
+(rule (simplify (select ty (eq rty x y)
+                           (iconst ty (u64_from_imm64 0))
+                           (select ty (ult rty x y)
+                                      (iconst ty neg_1)
+                                      (iconst ty (u64_from_imm64 1)))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+;; x == y ? 0 : x <= y ? -1 : +1
+(rule (simplify (select ty (eq rty x y)
+                           (iconst ty (u64_from_imm64 0))
+                           (select ty (ule rty x y)
+                                      (iconst ty neg_1)
+                                      (iconst ty (u64_from_imm64 1)))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+;; x == y ? 0 : x > y ? +1 : -1
+(rule (simplify (select ty (eq rty x y)
+                           (iconst ty (u64_from_imm64 0))
+                           (select ty (ugt rty x y)
+                                      (iconst ty (u64_from_imm64 1))
+                                      (iconst ty neg_1))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+;; x == y ? 0 : x >= y ? +1 : -1
+(rule (simplify (select ty (eq rty x y)
+                           (iconst ty (u64_from_imm64 0))
+                           (select ty (uge rty x y)
+                                      (iconst ty (u64_from_imm64 1))
+                                      (iconst ty neg_1))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+
+;; x > y ? 1 : x < y ? -1 : 0
+(rule (simplify (select ty (ugt rty x y)
+                           (iconst ty (u64_from_imm64 1))
+                           (select ty (ult rty x y)
+                                      (iconst ty neg_1)
+                                      (iconst ty (u64_from_imm64 0)))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+;; x > y ? 1 : x != y ? -1 : 0
+(rule (simplify (select ty (ugt rty x y)
+                           (iconst ty (u64_from_imm64 1))
+                           (select ty (ne rty x y)
+                                      (iconst ty neg_1)
+                                      (iconst ty (u64_from_imm64 0)))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+;; x > y ? 1 : x == y ? 0 : -1
+(rule (simplify (select ty (ugt rty x y)
+                           (iconst ty (u64_from_imm64 1))
+                           (select ty (eq rty x y)
+                                      (iconst ty (u64_from_imm64 0))
+                                      (iconst ty neg_1))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+;; x > y ? 1 : x >= y ? 0 : -1
+(rule (simplify (select ty (ugt rty x y)
+                           (iconst ty (u64_from_imm64 1))
+                           (select ty (uge rty x y)
+                                      (iconst ty (u64_from_imm64 0))
+                                      (iconst ty neg_1))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_u rty x y)))
+
+;; Same, but for signed comparisons this time
+
+;; x < y ? -1 : x == y ? 0 : +1
+;; x < y ? -1 : x != y ? +1 : 0
+(rule (simplify (select ty (slt rty x y)
+                           (iconst ty neg_1)
+                           (ne rty x y)))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (spaceship_s rty x y))
+(rule (simplify (select ty (slt rty x y)
+                           (iconst ty neg_1)
+                           (uextend ty (ne rty x y))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+;; x < y ? -1 : x <= y ? 0 : +1
+;; x < y ? -1 : x > y ? +1 : 0
+(rule (simplify (select ty (slt rty x y)
+                           (iconst ty neg_1)
+                           (sgt rty x y)))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (spaceship_s rty x y))
+(rule (simplify (select ty (slt rty x y)
+                           (iconst ty neg_1)
+                           (uextend ty (sgt rty x y))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+
+;; x == y ? 0 : x < y ? -1 : +1
+(rule (simplify (select ty (eq rty x y)
+                           (iconst ty (u64_from_imm64 0))
+                           (select ty (slt rty x y)
+                                      (iconst ty neg_1)
+                                      (iconst ty (u64_from_imm64 1)))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+;; x == y ? 0 : x <= y ? -1 : +1
+(rule (simplify (select ty (eq rty x y)
+                           (iconst ty (u64_from_imm64 0))
+                           (select ty (sle rty x y)
+                                      (iconst ty neg_1)
+                                      (iconst ty (u64_from_imm64 1)))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+;; x == y ? 0 : x > y ? +1 : -1
+(rule (simplify (select ty (eq rty x y)
+                           (iconst ty (u64_from_imm64 0))
+                           (select ty (sgt rty x y)
+                                      (iconst ty (u64_from_imm64 1))
+                                      (iconst ty neg_1))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+;; x == y ? 0 : x >= y ? +1 : -1
+(rule (simplify (select ty (eq rty x y)
+                           (iconst ty (u64_from_imm64 0))
+                           (select ty (sge rty x y)
+                                      (iconst ty (u64_from_imm64 1))
+                                      (iconst ty neg_1))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+
+;; x > y ? 1 : x < y ? -1 : 0
+(rule (simplify (select ty (sgt rty x y)
+                           (iconst ty (u64_from_imm64 1))
+                           (select ty (slt rty x y)
+                                      (iconst ty neg_1)
+                                      (iconst ty (u64_from_imm64 0)))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+;; x > y ? 1 : x != y ? -1 : 0
+(rule (simplify (select ty (sgt rty x y)
+                           (iconst ty (u64_from_imm64 1))
+                           (select ty (ne rty x y)
+                                      (iconst ty neg_1)
+                                      (iconst ty (u64_from_imm64 0)))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+;; x > y ? 1 : x == y ? 0 : -1
+(rule (simplify (select ty (sgt rty x y)
+                           (iconst ty (u64_from_imm64 1))
+                           (select ty (eq rty x y)
+                                      (iconst ty (u64_from_imm64 0))
+                                      (iconst ty neg_1))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+;; x > y ? 1 : x >= y ? 0 : -1
+(rule (simplify (select ty (sgt rty x y)
+                           (iconst ty (u64_from_imm64 1))
+                           (select ty (sge rty x y)
+                                      (iconst ty (u64_from_imm64 0))
+                                      (iconst ty neg_1))))
+      (if-let -1 (i64_sextend_imm64 ty neg_1))
+      (sextend_from_i8 ty (spaceship_s rty x y)))
+
+;; Then once we have it normalized, we can apply some basic simplifications.
+;; For example, a derived `PartialOrd::lt` on a newtype in Rust will essentially
+;; emit `(a <=> b) < 0`, and replacing that with `a < b` can really help.
+;; `icmp.isle` prefers comparing against zero so we don't need to worry about
+;; also matching things like `(a <=> b) < 1` or `(a <=> b) <= -1`.
+
+;; (a <=> b) == 0 --> a == b
+(rule (simplify (eq _ (spaceship_s ty x y) (iconst _ (u64_from_imm64 0))))
+      (eq ty x y))
+(rule (simplify (eq _ (spaceship_u ty x y) (iconst _ (u64_from_imm64 0))))
+      (eq ty x y))
+;; (a <=> b) != 0 --> a != b
+(rule (simplify (ne _ (spaceship_s ty x y) (iconst _ (u64_from_imm64 0))))
+      (ne ty x y))
+(rule (simplify (ne _ (spaceship_u ty x y) (iconst _ (u64_from_imm64 0))))
+      (ne ty x y))
+
+;; (a <=> b) < 0 --> a < b
+(rule (simplify (slt _ (spaceship_s ty x y) (iconst _ (u64_from_imm64 0))))
+      (slt ty x y))
+(rule (simplify (slt _ (spaceship_u ty x y) (iconst _ (u64_from_imm64 0))))
+      (ult ty x y))
+;; (a <=> b) <= 0 --> a <= b
+(rule (simplify (sle _ (spaceship_s ty x y) (iconst _ (u64_from_imm64 0))))
+      (sle ty x y))
+(rule (simplify (sle _ (spaceship_u ty x y) (iconst _ (u64_from_imm64 0))))
+      (ule ty x y))
+;; (a <=> b) > 0 --> a > b
+(rule (simplify (sgt _ (spaceship_s ty x y) (iconst _ (u64_from_imm64 0))))
+      (sgt ty x y))
+(rule (simplify (sgt _ (spaceship_u ty x y) (iconst _ (u64_from_imm64 0))))
+      (ugt ty x y))
+;; (a <=> b) >= 0 --> a >= b
+(rule (simplify (sge _ (spaceship_s ty x y) (iconst _ (u64_from_imm64 0))))
+      (sge ty x y))
+(rule (simplify (sge _ (spaceship_u ty x y) (iconst _ (u64_from_imm64 0))))
+      (uge ty x y))
+
+;; extend from i8 to i8 is invalid CLIF, so this allows fixing that in the output
+;; rather than needing to duplicate rules for the different width categories
+(decl sextend_from_i8 (Type Value) Value)
+(rule 0 (sextend_from_i8 ty val) (sextend ty val))
+(rule 1 (sextend_from_i8 $I8 val) val)