From 843549e9e87e5bfba0f269cf5b265a0aaafb23e4 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 27 Sep 2023 23:24:41 -0700
Subject: [PATCH 1/5] [RISCV][GISel] Legalize G_SMULO/G_UMULO

I'm unhappy with the XLen case when we need to make a libcall. We
get two libcalls.
---
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp |   8 +
 .../legalizer/rv32/legalize-mulo.mir          | 340 ++++++++++++++
 .../legalizer/rv64/legalize-mulo.mir          | 442 ++++++++++++++++++
 3 files changed, 790 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 313a8221ab1bd..a9d6f61541253 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -145,6 +145,10 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         .legalFor({XLenLLT})
         .lower();
     // clang-format on
+
+    getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+        .minScalar(0, XLenLLT)
+        .lower();
   } else {
     getActionDefinitionsBuilder(G_MUL)
         .libcallFor({XLenLLT, DoubleXLenLLT})
@@ -152,6 +156,10 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         .clampScalar(0, XLenLLT, DoubleXLenLLT);
 
     getActionDefinitionsBuilder({G_SMULH, G_UMULH}).lowerFor({XLenLLT});
+
+    getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+        .minScalar(0, XLenLLT)
+        .lower();
   }
 
   if (ST.hasStdExtM()) {
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
new file mode 100644
index 0000000000000..35bea08f426a3
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
@@ -0,0 +1,340 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mattr=+m -mtriple=riscv32 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mattr=+zmmul -mtriple=riscv32 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s --check-prefix=LIBCALL
+
+---
+name:            smulo_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i8
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ASHR]], [[ASHR1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[MUL]], [[C2]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[ASHR2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i8
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s32) = COPY $x10
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $x11
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s32) = G_ANYEXT %4(s8)
+    %7:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i16
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ASHR]], [[ASHR1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[MUL]], [[C2]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[ASHR2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i16
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s32) = COPY $x10
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $x11
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s32) = G_ANYEXT %4(s16)
+    %7:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i32
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[C]](s32)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[COPY1]], [[C1]](s32)
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR]](s32)
+    ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s32)
+    ; LIBCALL-NEXT: $x13 = COPY [[ASHR1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[COPY3]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY3]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32), %3:_(s1) = G_SMULO %0, %1
+    %4:_(s32) = G_ANYEXT %3(s1)
+    $x10 = COPY %2(s32)
+    $x11 = COPY %4(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i8
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i8
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s32) = COPY $x10
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $x11
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s32) = G_ANYEXT %4(s8)
+    %7:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i16
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i16
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s32) = COPY $x10
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $x11
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s32) = G_ANYEXT %4(s16)
+    %7:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[UMULH]](s32), [[C]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i32
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[C]](s32)
+    ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s32)
+    ; LIBCALL-NEXT: $x13 = COPY [[C1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[COPY1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[C2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY3]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32), %3:_(s1) = G_UMULO %0, %1
+    %4:_(s32) = G_ANYEXT %3(s1)
+    $x10 = COPY %2(s32)
+    $x11 = COPY %4(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir
new file mode 100644
index 0000000000000..219635914efdd
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir
@@ -0,0 +1,442 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mattr=+m -mtriple=riscv64 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mattr=+zmmul -mtriple=riscv64 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s --check-prefix=LIBCALL
+
+---
+name:            smulo_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i8
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ASHR]], [[ASHR1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[MUL]], [[C2]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[ASHR2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i8
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s8) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s8) = G_TRUNC %3(s64)
+    %4:_(s8), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s8)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i16
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ASHR]], [[ASHR1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[MUL]], [[C2]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[ASHR2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i16
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s16) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s16) = G_TRUNC %3(s64)
+    %4:_(s16), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s16)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
+    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s64) = G_SEXT_INREG [[MUL]], 32
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[SEXT_INREG2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i32
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
+    ; LIBCALL-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
+    ; LIBCALL-NEXT: $x10 = COPY [[SEXT_INREG]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[SEXT_INREG1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY2]], 32
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[SEXT_INREG2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s32) = G_TRUNC %3(s64)
+    %4:_(s32), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s32)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[SMULH:%[0-9]+]]:_(s64) = G_SMULH [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MUL]], [[C]](s64)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[SMULH]](s64), [[ASHR]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i64
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s64)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C1]](s64)
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR]](s64)
+    ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s64)
+    ; LIBCALL-NEXT: $x13 = COPY [[ASHR1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__multi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[COPY1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[COPY3]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY3]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64), %3:_(s1) = G_SMULO %0, %1
+    %4:_(s64) = G_ANYEXT %3(s1)
+    $x10 = COPY %2(s64)
+    $x11 = COPY %4(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i8
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i8
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s8) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s8) = G_TRUNC %3(s64)
+    %4:_(s8), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s8)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i16
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i16
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s16) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s16) = G_TRUNC %3(s64)
+    %4:_(s16), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s16)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i32
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s32) = G_TRUNC %3(s64)
+    %4:_(s32), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s32)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s64) = G_UMULH [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[UMULH]](s64), [[C]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i64
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[C]](s64)
+    ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s64)
+    ; LIBCALL-NEXT: $x13 = COPY [[C1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__multi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[COPY1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[C2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY3]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64), %3:_(s1) = G_UMULO %0, %1
+    %4:_(s64) = G_ANYEXT %3(s1)
+    $x10 = COPY %2(s64)
+    $x11 = COPY %4(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...

From 56b7ccb9e41a12965e3b3bfdaffdf5dfe5e853f2 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 29 Sep 2023 11:32:30 -0700
Subject: [PATCH 2/5] !fixup reduce code duplication

---
 llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index a9d6f61541253..103b12fde57bb 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -145,10 +145,6 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         .legalFor({XLenLLT})
         .lower();
     // clang-format on
-
-    getActionDefinitionsBuilder({G_SMULO, G_UMULO})
-        .minScalar(0, XLenLLT)
-        .lower();
   } else {
     getActionDefinitionsBuilder(G_MUL)
         .libcallFor({XLenLLT, DoubleXLenLLT})
@@ -156,12 +152,12 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         .clampScalar(0, XLenLLT, DoubleXLenLLT);
 
     getActionDefinitionsBuilder({G_SMULH, G_UMULH}).lowerFor({XLenLLT});
-
-    getActionDefinitionsBuilder({G_SMULO, G_UMULO})
-        .minScalar(0, XLenLLT)
-        .lower();
   }
 
+  getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+      .minScalar(0, XLenLLT)
+      .lower();
+
   if (ST.hasStdExtM()) {
     getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM})
         .legalFor({s32, XLenLLT})

From b2137ec3e405ba04e8e0bb320380cb41fc3918f5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 29 Sep 2023 12:49:32 -0700
Subject: [PATCH 3/5] !fixup clang-format

---
 llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 103b12fde57bb..5f4e8da3e628f 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -154,9 +154,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
     getActionDefinitionsBuilder({G_SMULH, G_UMULH}).lowerFor({XLenLLT});
   }
 
-  getActionDefinitionsBuilder({G_SMULO, G_UMULO})
-      .minScalar(0, XLenLLT)
-      .lower();
+  getActionDefinitionsBuilder({G_SMULO, G_UMULO}).minScalar(0, XLenLLT).lower();
 
   if (ST.hasStdExtM()) {
     getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM})

From 9132427918220ca995fd9852c0e53b5fffbc3f5e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 29 Sep 2023 17:36:46 -0700
Subject: [PATCH 4/5] Fixes to get a single libcall for XLenLLT.

---
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    | 20 ++++++---
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 20 ++++++++-
 .../legalizer/rv32/legalize-mulo.mir          | 42 +++++++++++--------
 .../legalizer/rv64/legalize-mulo.mir          | 42 +++++++++++--------
 4 files changed, 83 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 0c3f558ac2a64..10b115b217352 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1995,8 +1995,20 @@ LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
   auto LeftOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {LHS});
   auto RightOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {RHS});
 
-  auto Mulo = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy, OverflowTy},
-                                    {LeftOperand, RightOperand});
+  // Multiplication cannot overflow if the WideTy is >= 2 * original width,
+  // so we don't need to check the overflow result of larger type Mulo.
+  bool WideMulCanOverflow = WideTy.getScalarSizeInBits() < 2 * SrcBitWidth;
+
+  unsigned MulOpc =
+      WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL;
+
+  MachineInstrBuilder Mulo;
+  if (WideMulCanOverflow)
+    Mulo = MIRBuilder.buildInstr(MulOpc, {WideTy, OverflowTy},
+                                 {LeftOperand, RightOperand});
+  else
+    Mulo = MIRBuilder.buildInstr(MulOpc, {WideTy}, {LeftOperand, RightOperand});
+
   auto Mul = Mulo->getOperand(0);
   MIRBuilder.buildTrunc(Result, Mul);
 
@@ -2014,9 +2026,7 @@ LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
     ExtResult = MIRBuilder.buildZExtInReg(WideTy, Mul, SrcBitWidth);
   }
 
-  // Multiplication cannot overflow if the WideTy is >= 2 * original width,
-  // so we don't need to check the overflow result of larger type Mulo.
-  if (WideTy.getScalarSizeInBits() < 2 * SrcBitWidth) {
+  if (WideMulCanOverflow) {
     auto Overflow =
         MIRBuilder.buildICmp(CmpInst::ICMP_NE, OverflowTy, Mul, ExtResult);
     // Finally check if the multiplication in the larger type itself overflowed.
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 5f4e8da3e628f..c90ebf8b9fbc9 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -145,6 +145,10 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         .legalFor({XLenLLT})
         .lower();
     // clang-format on
+
+    getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+        .minScalar(0, XLenLLT)
+        .lower();
   } else {
     getActionDefinitionsBuilder(G_MUL)
         .libcallFor({XLenLLT, DoubleXLenLLT})
@@ -152,9 +156,21 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         .clampScalar(0, XLenLLT, DoubleXLenLLT);
 
     getActionDefinitionsBuilder({G_SMULH, G_UMULH}).lowerFor({XLenLLT});
-  }
 
-  getActionDefinitionsBuilder({G_SMULO, G_UMULO}).minScalar(0, XLenLLT).lower();
+    getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+        .minScalar(0, XLenLLT)
+        // Widen XLenLLT to DoubleXLenLLT so we can use a single libcall to get
+        // the low bits for the mul result and high bits to do the overflow
+        // check.
+        .widenScalarIf(
+            [=, &ST](const LegalityQuery &Query) {
+              return Query.Types[0] == XLenLLT;
+            },
+            [=](const LegalityQuery &Query) {
+              return std::make_pair(0, DoubleXLenLLT);
+            })
+        .lower();
+  }
 
   if (ST.hasStdExtM()) {
     getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM})
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
index 35bea08f426a3..43fd1f99fdcaf 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
@@ -160,15 +160,19 @@ body:             |
     ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s32)
     ; LIBCALL-NEXT: $x13 = COPY [[ASHR1]](s32)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
-    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x11
-    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
-    ; LIBCALL-NEXT: $x11 = COPY [[COPY1]](s32)
-    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[COPY3]], [[C2]](s32)
-    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[ASHR2]]
-    ; LIBCALL-NEXT: $x10 = COPY [[COPY3]](s32)
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; LIBCALL-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[ASHR2]], [[C3]](s32)
+    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[ASHR2]]
+    ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY3]], [[ASHR3]]
+    ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR]], [[XOR1]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C4]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
     ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
     %0:_(s32) = COPY $x10
@@ -319,14 +323,18 @@ body:             |
     ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s32)
     ; LIBCALL-NEXT: $x13 = COPY [[C1]](s32)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
-    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x11
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
-    ; LIBCALL-NEXT: $x11 = COPY [[COPY1]](s32)
-    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x10
-    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[C2]]
-    ; LIBCALL-NEXT: $x10 = COPY [[COPY3]](s32)
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[AND]]
+    ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY3]], [[AND1]]
+    ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR]], [[XOR1]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C4]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
     ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
     %0:_(s32) = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir
index 219635914efdd..7e1ec1e0961d7 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir
@@ -208,15 +208,19 @@ body:             |
     ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s64)
     ; LIBCALL-NEXT: $x13 = COPY [[ASHR1]](s64)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__multi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
-    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x11
-    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
-    ; LIBCALL-NEXT: $x11 = COPY [[COPY1]](s64)
-    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
-    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[COPY3]], [[C2]](s64)
-    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[ASHR2]]
-    ; LIBCALL-NEXT: $x10 = COPY [[COPY3]](s64)
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; LIBCALL-NEXT: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[ASHR2]], [[C3]](s64)
+    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[ASHR2]]
+    ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[COPY3]], [[ASHR3]]
+    ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[XOR]], [[XOR1]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C4]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
     ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
     %0:_(s64) = COPY $x10
@@ -421,14 +425,18 @@ body:             |
     ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s64)
     ; LIBCALL-NEXT: $x13 = COPY [[C1]](s64)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__multi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
-    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x11
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
-    ; LIBCALL-NEXT: $x11 = COPY [[COPY1]](s64)
-    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x10
-    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[C2]]
-    ; LIBCALL-NEXT: $x10 = COPY [[COPY3]](s64)
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C3]]
+    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[AND]]
+    ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[COPY3]], [[AND1]]
+    ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[XOR]], [[XOR1]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C4]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
     ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
     %0:_(s64) = COPY $x10

From 2ca370db280b4d720d3907ce4ef7e394de1a7786 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 13 Oct 2023 17:18:18 -0700
Subject: [PATCH 5/5] !fixup remove unused lambda capture

---
 llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index c90ebf8b9fbc9..2f889833b6907 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -163,7 +163,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         // the low bits for the mul result and high bits to do the overflow
         // check.
         .widenScalarIf(
-            [=, &ST](const LegalityQuery &Query) {
+            [=](const LegalityQuery &Query) {
               return Query.Types[0] == XLenLLT;
             },
             [=](const LegalityQuery &Query) {