[InstCombine] Formation of fmax ignores SNaN

[nunoplopes]

According to LangRef, fmax SNaN, num -> QNaN

; Transforms/InstCombine/fcmp-select.ll

define double @test_fcmp_select_maxnum(double %x) {
  %cmp1 = fcmp ogt double %x, 1.000000
  %sel1 = select nnan nsz i1 %cmp1, double %x, double 1.000000
  %cmp2 = fcmp olt double %sel1, 255.000000
  %sel2 = select nnan nsz i1 %cmp2, double %sel1, double 255.000000
  ret double %sel2
}
=>
define double @test_fcmp_select_maxnum(double %x) {
  %sel1 = fmax nsz double %x, 1.000000
  %sel2 = fmin nsz double %sel1, 255.000000
  ret double %sel2
}
Transformation doesn't verify! (unsound)
ERROR: Value mismatch

Example:
double %x = #x7ff4000000000000 (SNaN)

Source:
i1 %cmp1 = #x0 (0)
double %sel1 = #x3ff0000000000000 (1)
i1 %cmp2 = #x1 (1)
double %sel2 = #x3ff0000000000000 (1)

Target:
double %sel1 = #x7ffc000000000000 (QNaN)
double %sel2 = #x406fe00000000000 (255)
Source value: #x3ff0000000000000 (1)
Target value: #x406fe00000000000 (255)

[nunoplopes]

Another case:

; Transforms/InstCombine/fcmp-fadd-select.ll

define float @test_fcmp_ogt_fadd_select_constant(float %in) {
  %cmp1 = fcmp ogt float %in, 0.000000
  %add = fadd float %in, 1.000000
  %sel = select nnan nsz i1 %cmp1, float %add, float 1.000000
  ret float %sel
}
=>
define float @test_fcmp_ogt_fadd_select_constant(float %in) {
  %sel = fmax nsz float %in, 0.000000
  %add = fadd nnan nsz float %sel, 1.000000
  ret float %add
}
Transformation doesn't verify! (unsound)
ERROR: Target is more poisonous than source

Example:
float %in = #x7f801000 (SNaN)

Source:
i1 %cmp1 = #x0 (0)
float %add = #xffc01000 (QNaN)
float %sel = #x3f800000 (1)

Target:
float %sel = #x7fc01000 (QNaN)
float %add = poison
Source value: #x3f800000 (1)
Target value: poison

[nunoplopes]

Another one:

; Transforms/InstSimplify/fminmax-folds.ll

define float @test_minnum_const_nan(float %x) {
  %r = fmin float %x, 0x7ff80000
  ret float %r
}
=>
define float @test_minnum_const_nan(float %x) {
  ret float %x
}

[nunoplopes]

Another one:

; Transforms/InstSimplify/floating-point-compare.ll

define i1 @minnum_non_nan(float %x) {
  %min = fmin float 0.500000, %x
  %cmp = fcmp ord float %min, 1.000000
  ret i1 %cmp
}
=>
define i1 @minnum_non_nan(float %x) {
  ret i1 1
}
Transformation doesn't verify! (unsound)
ERROR: Value mismatch

Example:
float %x = #x7f800080 (SNaN)

Source:
float %min = #xffc00080 (QNaN)
i1 %cmp = #x0 (0)

Target:
Source value: #x0 (0)
Target value: #x1 (1)

[nunoplopes]

Fallout when trying to fix AliveToolkit/alive2#1201
cc @regehr @arsenm @wzssyqa

[regehr]

also cc @dtcxzyw

[wzssyqa]

It's due to expandFMINNUM_FMAXNUM
In fact FCANONICALIZE is not needed if FMINNUM_IEEE available.