positive and negative relu activation function 'linear' operators

src/JetPack.jl

@@ -31,6 +31,8 @@ include("jop_permutedims.jl")
 include("jop_pow.jl")
 include("jop_projection.jl")
 include("jop_real.jl")
+include("jop_relumin.jl")
+include("jop_relumax.jl")
 include("jop_removedc.jl")
 include("jop_reghost.jl")
 include("jop_reshape.jl")

src/jop_relumax.jl

@@ -0,0 +1,15 @@
+"""
+    F = JopReluMax(spc)
+
+where `F` is the relu positive activation function operator with domain and range given by `spc::JetSpace`.
+
+Note that `F` is not a liner function in the sense that for general vectors `x` and `y` 
+```
+    F(α x + y) != α F(x) + F(y)
+```
+However if `x` and `y` are either both strictly positive, or both strictly negative, F is linear.
+"""
+JopReluMax(spc::JetSpace) = JopLn(dom = spc, rng = spc, df! = JopReluMax_df!)
+export JopReluMax
+
+JopReluMax_df!(d::AbstractArray{T}, m::AbstractArray{T}; kwargs...) where {T <: Real} = d .= map(x -> x > 0 ? x : 0, m)

src/jop_relumin.jl

@@ -0,0 +1,15 @@
+"""
+    F = JopReluMin(spc)
+
+where `F` is the relu negative activation function operator with domain and range given by `spc::JetSpace`.
+
+Note that `F` is not a liner function in the sense that for general vectors `x` and `y` 
+```
+    F(α x + y) != α F(x) + F(y)
+```
+However if `x` and `y` are either both strictly positive, or both strictly negative, F is linear.
+"""
+JopReluMin(spc::JetSpace) = JopLn(dom = spc, rng = spc, df! = JopReluMin_df!)
+export JopReluMin
+
+JopReluMin_df!(d::AbstractArray{T}, m::AbstractArray{T}; kwargs...) where {T <: Real} = d .= map(x -> x < 0 ? x : 0, m)

test/jop_relumax.jl

@@ -0,0 +1,40 @@
+using LinearAlgebra, Jets, JetPack, Test
+
+# note no need for a linearity test as function is "linear" for x > 0 and constant for x <= 0 
+# you do need special handling: strictly positive or negative domain and range vectors for tests
+
+n1,n2 = 33,44
+
+@testset "JopReluMax, correctness T=$(T)" for T in (Float64,Float32)
+    F = JopReluMax(JetSpace(T,n1,n2))
+    x1 = rand(domain(F))
+    @test F*x1 ≈ max.(x1,0)
+end
+
+@testset "JopReluMax, linearity test, T=$(T)" for T in (Float64,Float32)
+    F = JopReluMax(JetSpace(T,n1,n2))
+    m1 = rand(domain(F)) .+ T(0.1)
+    m2 = rand(domain(F)) .+ T(0.1)
+    lhs,rhs = linearity_test(F, m1, m2)
+    @test lhs ≈ rhs
+
+    m1 = - rand(domain(F)) .- T(0.1)
+    m2 = - rand(domain(F)) .- T(0.1)
+    lhs,rhs = linearity_test(F, m1, m2)
+    @test lhs ≈ rhs
+end
+
+@testset "JopReluMax, dot product test, T=$(T)" for T in (Float64,Float32)
+    F = JopReluMax(JetSpace(T,n1,n2))
+    δm = rand(domain(F)) .+ T(0.1)
+    δd = rand(domain(F)) .+ T(0.1)
+    lhs, rhs = dot_product_test(F, δd, δm)
+    @test lhs ≈ rhs
+
+    F = JopReluMax(JetSpace(T,n1,n2))
+    δm = - rand(domain(F)) .- T(0.1)
+    δd = - rand(domain(F)) .- T(0.1)
+    lhs, rhs = dot_product_test(F, δd, δm)
+    @test lhs ≈ rhs
+end
+

test/jop_relumin.jl

@@ -0,0 +1,40 @@
+using LinearAlgebra, Jets, JetPack, Test
+
+# note no need for a linearity test as function is "linear" for x > 0 and constant for x <= 0 
+# you do need special handling: strictly positive or negative domain and range vectors for tests
+
+n1,n2 = 33,44
+
+@testset "JopReluMin, correctness T=$(T)" for T in (Float64,Float32)
+    F = JopReluMin(JetSpace(T,n1,n2))
+    x1 = rand(domain(F))
+    @test F*x1 ≈ min.(x1,0)
+end
+
+@testset "JopReluMin, linearity test, T=$(T)" for T in (Float64,Float32)
+    F = JopReluMin(JetSpace(T,n1,n2))
+    m1 = rand(domain(F)) .+ T(0.1)
+    m2 = rand(domain(F)) .+ T(0.1)
+    lhs,rhs = linearity_test(F, m1, m2)
+    @test lhs ≈ rhs
+
+    m1 = - rand(domain(F)) .- T(0.1)
+    m2 = - rand(domain(F)) .- T(0.1)
+    lhs,rhs = linearity_test(F, m1, m2)
+    @test lhs ≈ rhs
+end
+
+@testset "JopReluMin, dot product test, T=$(T)" for T in (Float64,Float32)
+    F = JopReluMin(JetSpace(T,n1,n2))
+    δm = rand(domain(F)) .+ T(0.1)
+    δd = rand(domain(F)) .+ T(0.1)
+    lhs, rhs = dot_product_test(F, δd, δm)
+    @test lhs ≈ rhs
+
+    F = JopReluMin(JetSpace(T,n1,n2))
+    δm = - rand(domain(F)) .- T(0.1)
+    δd = - rand(domain(F)) .- T(0.1)
+    lhs, rhs = dot_product_test(F, δd, δm)
+    @test lhs ≈ rhs
+end
+

test/runtests.jl

@@ -29,6 +29,8 @@ for filename in (
         "jop_projection.jl",
         "jop_real.jl",
         "jop_reghost.jl",
+        "jop_relumin.jl",
+        "jop_relumax.jl",
         "jop_removedc.jl",
         "jop_reshape.jl",
         "jop_restriction.jl",