From dbf2d5b4f4a97646826e4bb0ab2ddb53e84c2c15 Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Thu, 25 Mar 2021 09:52:48 +0100 Subject: [PATCH] add zygote#cl/cufunc; remove entirely cufunc --- .ci/develop_nnlibcuda.jl | 1 + lib/NNlibCUDA/Manifest.toml | 67 ++++++++++++++++++++++++++++++++ lib/NNlibCUDA/Project.toml | 1 + lib/NNlibCUDA/src/NNlibCUDA.jl | 2 - lib/NNlibCUDA/src/activations.jl | 24 ------------ 5 files changed, 69 insertions(+), 26 deletions(-) delete mode 100644 lib/NNlibCUDA/src/activations.jl diff --git a/.ci/develop_nnlibcuda.jl b/.ci/develop_nnlibcuda.jl index 5ac475f43..e27142c94 100644 --- a/.ci/develop_nnlibcuda.jl +++ b/.ci/develop_nnlibcuda.jl @@ -14,5 +14,6 @@ Pkg.add(url="https://github.com/JuliaGPU/GPUCompiler.jl.git", rev="master") Pkg.add(url="https://github.com/JuliaGPU/GPUArrays.jl.git", rev="master") Pkg.add(url="https://github.com/maleadt/LLVM.jl.git", rev="master") Pkg.add(url="https://github.com/JuliaGPU/CUDA.jl.git", rev="master") +Pkg.add(url="https://github.com/FluxML/Zygote.jl.git", rev="cl/cufunc") Pkg.precompile() diff --git a/lib/NNlibCUDA/Manifest.toml b/lib/NNlibCUDA/Manifest.toml index aa0f30892..2b0ccefd9 100644 --- a/lib/NNlibCUDA/Manifest.toml +++ b/lib/NNlibCUDA/Manifest.toml @@ -40,12 +40,24 @@ repo-url = "https://github.com/JuliaGPU/CUDA.jl.git" uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" version = "2.6.0" +[[ChainRules]] +deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Reexport", "Requires", "Statistics"] +git-tree-sha1 = "e01f521443e3700f40ad3c7c1c6aa3a6940aaea1" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "0.7.54" + [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] git-tree-sha1 = "a8a75576d509f1996fe3046ce62e5685b0a8bd3a" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" version = "0.9.31" +[[CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] git-tree-sha1 = "919c7f3151e79ff196add81d7f4e45d91bbf420b" @@ -70,6 +82,18 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" deps = ["Mmap"] uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +[[DiffResults]] +deps = ["StaticArrays"] +git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.0.3" + +[[DiffRules]] +deps = ["NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "214c3fcac57755cfda163d91c58893a8723f93e9" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.0.2" + [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" @@ -83,6 +107,18 @@ git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" version = "0.1.3" +[[FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays"] +git-tree-sha1 = "31939159aeb8ffad1d4d8ee44d07f8558273120a" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "0.11.7" + +[[ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "c68fb7481b71519d313114dca639b35262ff105f" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.17" + [[GPUArrays]] deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"] git-tree-sha1 = "0e8eec7dc7c3c68fc032c0f6cb580fa86ace445f" @@ -99,6 +135,12 @@ repo-url = "https://github.com/JuliaGPU/GPUCompiler.jl.git" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" version = "0.10.1" +[[IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "c67e7515a11f726f44083e74f218d134396d6510" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.2" + [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -178,6 +220,11 @@ git-tree-sha1 = "ab1d43fead2ecb9aa5ae460d3d547c2cf8d89461" uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" version = "0.7.17" +[[NaNMath]] +git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "0.3.5" + [[NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -254,6 +301,12 @@ git-tree-sha1 = "5919936c0e92cff40e57d0ddf0ceb667d42e5902" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" version = "1.3.0" +[[StaticArrays]] +deps = ["LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "9da72ed50e94dbff92036da395275ed114e04d49" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.0.1" + [[Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" @@ -287,6 +340,20 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +[[Zygote]] +deps = ["AbstractFFTs", "ChainRules", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "baa1e40ff1bd0fe70a93bf7871acb2f5653971f0" +repo-rev = "cl/cufunc" +repo-url = "https://github.com/FluxML/Zygote.jl.git" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.6.5" + +[[ZygoteRules]] +deps = ["MacroTools"] +git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.1" + [[nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" diff --git a/lib/NNlibCUDA/Project.toml b/lib/NNlibCUDA/Project.toml index 2148104cb..e5ec254c1 100644 --- a/lib/NNlibCUDA/Project.toml +++ b/lib/NNlibCUDA/Project.toml @@ -11,6 +11,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] CUDA = "2.6, 3" diff --git a/lib/NNlibCUDA/src/NNlibCUDA.jl b/lib/NNlibCUDA/src/NNlibCUDA.jl index 42708a3af..48c11dfa3 100644 --- a/lib/NNlibCUDA/src/NNlibCUDA.jl +++ b/lib/NNlibCUDA/src/NNlibCUDA.jl @@ -2,11 +2,9 @@ module NNlibCUDA using NNlib using CUDA -# using CUDA: @cufunc # doesn't exist anymore in CUDA using Random, Statistics include("upsample.jl") -include("activations.jl") include("batchedmul.jl") include("cudnn/cudnn.jl") include("cudnn/conv.jl") diff --git a/lib/NNlibCUDA/src/activations.jl b/lib/NNlibCUDA/src/activations.jl deleted file mode 100644 index 6409423c3..000000000 --- a/lib/NNlibCUDA/src/activations.jl +++ /dev/null @@ -1,24 +0,0 @@ -# Activation functions - -# Some of activation functions need a wrapper for GPU support -# https://github.com/JuliaGPU/CuArrays.jl/issues/614 - -# @cufunc softplus(x::Real) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x))) - -# @cufunc logσ(x::Real) = -softplus(-x) - -# @cufunc function gelu(x::Real) -# p = oftype(x / 1, π) -# λ = oftype(x / 1, √(2 / p)) -# α = oftype(x / 1, 0.044715) -# h = oftype(x / 1, 0.5) -# h * x * (one(x) + tanh(λ * (x + α * x^3))) -# end - -# @cufunc lisht(x::Real) = x * tanh(x) - -# @cufunc logcosh(x::Real) = x + softplus(-2x) - log(oftype(x, 2)) - -# @cufunc mish(x::Real) = x * tanh(softplus(x)) - -# @cufunc tanhshrink(x::Real) = x - tanh(x)