fix powermod correctness on unsigned power of half typemax (#59680)

the comment claims:

> When the concrete type of p is signed and has the lowest value,
>   `p != 0 && p == -p` is equivalent to `p == typemin(typeof(p))`

this is true, but fails to consider that `p == -p` is also true when `p
= div(typemax(UInt), 2) + 1`, and that `p` is not necessarily signed

leading to incorrect results on inputs like

```
julia> powermod(0x03, 0x80, 0x07)
0x01
```

which should be `0x02`

Author: adienes

base/intfuncs.jl

@@ -489,18 +489,20 @@ julia> powermod(5, 3, 19)
 function powermod(x::Integer, p::Integer, m::T) where T<:Integer
     p == 0 && return mod(one(m),m)
     # When the concrete type of p is signed and has the lowest value,
-    # `p != 0 && p == -p` is equivalent to `p == typemin(typeof(p))` for 2's complement representation.
+    # `p < 0 && p == -p` is equivalent to `p == typemin(typeof(p))` for 2's complement representation.
     # but will work for integer types like `BigInt` that don't have `typemin` defined
     # It needs special handling otherwise will cause overflow problem.
-    if p == -p
-        imod = invmod(x, m)
-        rhalf = powermod(imod, -(p÷2), m)
-        r::T = mod(widemul(rhalf, rhalf), m)
-        isodd(p) && (r = mod(widemul(r, imod), m))
-        #else odd
-        return r
-    elseif p < 0
-        return powermod(invmod(x, m), -p, m)
+    if p < 0
+        if p == -p
+            imod = invmod(x, m)
+            rhalf = powermod(imod, -(p÷2), m)
+            r::T = mod(widemul(rhalf, rhalf), m)
+            isodd(p) && (r = mod(widemul(r, imod), m))
+            #else odd
+            return r
+        else
+            return powermod(invmod(x, m), -p, m)
+        end
     end
     (m == 1 || m == -1) && return zero(m)
     b = oftype(m,mod(x,m))  # this also checks for divide by zero

test/intfuncs.jl

@@ -359,6 +359,9 @@ end
     @test powermod(2, big(3), -5) == -2
     @inferred  powermod(2, -2, -5)
     @inferred  powermod(big(2), -2, UInt(5))
+
+    @test powermod(-3, 0x80, 7) === 2
+    @test powermod(0x03, 0x80, 0x07) === 0x02
 end
 
 @testset "nextpow/prevpow" begin