From fab6f3f02979bff725f182bd65cdbccefc5ad3c2 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 15 Mar 2021 14:52:18 +0530 Subject: [PATCH 01/14] Manifest.toml --- .gitignore | 2 +- Project.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2a59a5d0e5..b042b95f75 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ docs/build/ docs/site/ deps .vscode -Manifest.toml +# Manifest.toml diff --git a/Project.toml b/Project.toml index 1a2fefcc4f..51f723c74e 100644 --- a/Project.toml +++ b/Project.toml @@ -14,6 +14,7 @@ Juno = "e5e0dc1b-0480-54bc-9374-aad01c23163d" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +NNlibCUDA = "a00861dc-f156-4864-bf3c-e6376f28a68d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" From 1693a5cbbc72e993713a95b657eb78e3a9c15d24 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 15 Mar 2021 14:54:02 +0530 Subject: [PATCH 02/14] add Manifest --- Manifest.toml | 443 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 443 insertions(+) create mode 100644 Manifest.toml diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000000..105a243021 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,443 @@ +# This file is machine-generated - editing it directly is not advised + +[[AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.0.1" + +[[AbstractTrees]] +git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.3.4" + +[[Adapt]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ffcfa2d345aaee0ef3d8346a073d5dd03c983ebe" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.2.0" + +[[ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" + +[[Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[BFloat16s]] +deps = ["LinearAlgebra", "Test"] +git-tree-sha1 = "4af69e205efc343068dc8722b8dfec1ade89254a" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.1.0" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[CEnum]] +git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.4.1" + +[[CUDA]] +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "Memoize", "Printf", "Random", "Reexport", "Requires", "SparseArrays", "Statistics", "TimerOutputs"] +git-tree-sha1 = "381ba7285e069ba5a7817fe50d2d845888056d6a" +repo-rev = "master" +repo-url = "https://github.com/JuliaGPU/CUDA.jl.git" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "2.6.0" + +[[ChainRules]] +deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Reexport", "Requires", "Statistics"] +git-tree-sha1 = "e01f521443e3700f40ad3c7c1c6aa3a6940aaea1" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "0.7.54" + +[[ChainRulesCore]] +deps = ["Compat", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "de4f08843c332d355852721adb1592bce7924da3" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "0.9.29" + +[[CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.0" + +[[ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "32a2b8af383f11cbb65803883837a149d10dfe8a" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.10.12" + +[[Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Reexport"] +git-tree-sha1 = "ac5f2213e56ed8a34a3dd2f681f4df1166b34929" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.6" + +[[CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "919c7f3151e79ff196add81d7f4e45d91bbf420b" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "3.25.0" + +[[CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" + +[[DataAPI]] +git-tree-sha1 = "dfb3b7e89e395be1e25c2ad6d7690dc29cc53b1d" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.6.0" + +[[DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.9" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[DiffResults]] +deps = ["StaticArrays"] +git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.0.3" + +[[DiffRules]] +deps = ["NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "214c3fcac57755cfda163d91c58893a8723f93e9" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.0.2" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[Downloads]] +deps = ["ArgTools", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" + +[[ExprTools]] +git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.3" + +[[FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays"] +git-tree-sha1 = "dd4ab4257c257532003eb9836eea07473fcc732e" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "0.11.6" + +[[FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.4" + +[[ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "c68fb7481b71519d313114dca639b35262ff105f" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.17" + +[[Functors]] +deps = ["MacroTools"] +git-tree-sha1 = "a7bb2af991c43dcf5c3455d276dd83976799634f" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.2.1" + +[[GPUArrays]] +deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"] +git-tree-sha1 = "f99a25fe0313121f2f9627002734c7d63b4dd3bd" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "6.2.0" + +[[GPUCompiler]] +deps = ["DataStructures", "ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "Serialization", "TimerOutputs", "UUIDs"] +git-tree-sha1 = "ef2839b063e158672583b9c09d2cf4876a8d3d55" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "0.10.0" + +[[IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "c67e7515a11f726f44083e74f218d134396d6510" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.2" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[JLLWrappers]] +git-tree-sha1 = "a431f5f2ca3f4feef3bd7a5e94b8b8d4f2f647a0" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.2.0" + +[[Juno]] +deps = ["Base64", "Logging", "Media", "Profile"] +git-tree-sha1 = "07cb43290a840908a771552911a6274bc6c072c7" +uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d" +version = "0.8.4" + +[[LLVM]] +deps = ["CEnum", "Libdl", "Printf", "Unicode"] +git-tree-sha1 = "b616937c31337576360cb9fb872ec7633af7b194" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "3.6.0" + +[[LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" + +[[LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" + +[[LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "6a8a2a625ab0dea913aba95c11370589e0239ff0" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.6" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" + +[[Media]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "75a54abd10709c01f1b86b84ec225d26e840ed58" +uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27" +version = "0.5.0" + +[[Memoize]] +deps = ["MacroTools"] +git-tree-sha1 = "2b1dfcba103de714d31c033b5dacc2e4a12c7caa" +uuid = "c03570c3-d221-55d1-a50c-7939bbd78826" +version = "0.4.4" + +[[Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "f8c673ccc215eb50fcadb285f522420e29e69e1c" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "0.4.5" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" + +[[NNlib]] +deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"] +git-tree-sha1 = "ab1d43fead2ecb9aa5ae460d3d547c2cf8d89461" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.7.17" + +[[NNlibCUDA]] +deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"] +git-tree-sha1 = "919a71e04789f802cac3d01cae4ca52d33955133" +repo-rev = "master" +repo-url = "https://github.com/FluxML/NNlibCUDA.jl.git" +uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" +version = "0.1.0" + +[[NaNMath]] +git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "0.3.5" + +[[NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" + +[[OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9db77584158d0ab52307f8c04f8e7c08ca76b5b3" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.3+4" + +[[OrderedCollections]] +git-tree-sha1 = "4fa2ba51070ec13fcc7517db714445b4ab986bdf" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.4.0" + +[[Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[Profile]] +deps = ["Printf"] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Reexport]] +git-tree-sha1 = "57d8440b0c7d98fc4f889e478e80f268d534c9d5" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.0.0" + +[[Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.1.3" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Scratch]] +deps = ["Dates"] +git-tree-sha1 = "ad4b278adb62d185bbcb6864dc24959ab0627bf6" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.0.3" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SortingAlgorithms]] +deps = ["DataStructures", "Random", "Test"] +git-tree-sha1 = "03f5898c9959f8115e30bc7226ada7d0df554ddd" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "0.3.1" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[SpecialFunctions]] +deps = ["ChainRulesCore", "OpenSpecFun_jll"] +git-tree-sha1 = "5919936c0e92cff40e57d0ddf0ceb667d42e5902" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "1.3.0" + +[[StaticArrays]] +deps = ["LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "9da72ed50e94dbff92036da395275ed114e04d49" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.0.1" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"] +git-tree-sha1 = "400aa43f7de43aeccc5b2e39a76a79d262202b76" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.33.3" + +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[[Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" + +[[Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[TimerOutputs]] +deps = ["Printf"] +git-tree-sha1 = "32cdbe6cd2d214c25a0b88f985c9e0092877c236" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.8" + +[[TranscodingStreams]] +deps = ["Random", "Test"] +git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.9.5" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[ZipFile]] +deps = ["Libdl", "Printf", "Zlib_jll"] +git-tree-sha1 = "c3a5637e27e914a7a445b8d0ad063d701931e9f7" +uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" +version = "0.9.3" + +[[Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" + +[[Zygote]] +deps = ["AbstractFFTs", "ChainRules", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "47dd50e831959f8785f8a9854783a2eebac8e1bd" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.6.4" + +[[ZygoteRules]] +deps = ["MacroTools"] +git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.1" + +[[nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" From 0acf6c15d422440da98605cafce23edbc5416a9d Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 15 Mar 2021 15:05:50 +0530 Subject: [PATCH 03/14] rm CUDA CUDNN; use NNlibCUDA --- src/cuda/cuda.jl | 3 ++- src/cuda/cudnn.jl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cuda/cuda.jl b/src/cuda/cuda.jl index 6c8096f978..62d3195032 100644 --- a/src/cuda/cuda.jl +++ b/src/cuda/cuda.jl @@ -1,11 +1,12 @@ module CUDAint using ..CUDA -using CUDA: CUDNN +# using CUDA: CUDNN import ..Flux: Flux import Zygote using Zygote: @adjoint +import NNlib, NNlibCUDA include("cudnn.jl") diff --git a/src/cuda/cudnn.jl b/src/cuda/cudnn.jl index 0494672791..4a3b2618c8 100644 --- a/src/cuda/cudnn.jl +++ b/src/cuda/cudnn.jl @@ -1,4 +1,4 @@ -import CUDA.CUDNN: batchnorm, ∇batchnorm +import NNlibCUDA: batchnorm, ∇batchnorm function (BN::Flux.BatchNorm)(x::Union{CuArray{T,2},CuArray{T,4},CuArray{T,5}}, cache=nothing) where T<:Union{Float32, Float64} From 5dcb6d437877140d2dbac798a35531d9155f8110 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Sat, 10 Apr 2021 22:19:07 +0530 Subject: [PATCH 04/14] rm cufunc --- src/losses/functions.jl | 5 ----- src/losses/utils.jl | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/src/losses/functions.jl b/src/losses/functions.jl index 7863c77586..dac80c9607 100644 --- a/src/losses/functions.jl +++ b/src/losses/functions.jl @@ -291,8 +291,6 @@ julia> Flux.crossentropy(y_prob, y_hot) function binarycrossentropy(ŷ, y; agg = mean, ϵ = epseltype(ŷ)) agg(@.(-xlogy(y, ŷ + ϵ) - xlogy(1 - y, 1 - ŷ + ϵ))) end -# Re-definition to fix interaction with CuArrays. -# CUDA.@cufunc binarycrossentropy(ŷ, y; ϵ = eps(ŷ)) = -y * log(ŷ + ϵ) - (1 - y) * log(1 - ŷ + ϵ) """ logitbinarycrossentropy(ŷ, y; agg = mean) @@ -322,9 +320,6 @@ julia> Flux.binarycrossentropy(sigmoid.(y_model), y_bin) function logitbinarycrossentropy(ŷ, y; agg = mean) agg(@.((1 - y) * ŷ - logσ(ŷ))) end -# Re-definition to fix interaction with CuArrays. -# CUDA.@cufunc logitbinarycrossentropy(ŷ, y) = (1 - y) * ŷ - logσ(ŷ) - """ kldivergence(ŷ, y; agg = mean, ϵ = eps(ŷ)) diff --git a/src/losses/utils.jl b/src/losses/utils.jl index 8d4852d007..b9871ecfc9 100644 --- a/src/losses/utils.jl +++ b/src/losses/utils.jl @@ -8,11 +8,6 @@ function xlogx(x) ifelse(iszero(x), zero(result), result) end -CUDA.@cufunc function xlogx(x) - result = x * log(x) - ifelse(iszero(x), zero(result), result) -end - """ xlogy(x, y) @@ -23,11 +18,6 @@ function xlogy(x, y) ifelse(iszero(x), zero(result), result) end -CUDA.@cufunc function xlogy(x, y) - result = x * log(y) - ifelse(iszero(x), zero(result), result) -end - @adjoint function broadcasted(::typeof(xlogy), x::Zygote.Numeric, y::Zygote.Numeric) res = xlogy.(x, y) res, Δ -> (nothing, Zygote.unbroadcast(x, xlogy.(Δ, y)), Zygote.unbroadcast(y, Δ .* x ./ y)) From cba517748c4a72edc27694abcb2d579aea2e5318 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Sat, 10 Apr 2021 22:21:01 +0530 Subject: [PATCH 05/14] pkg up --- Manifest.toml | 78 +++++++++++++++++++++++++++------------------------ Project.toml | 5 ++-- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 105a243021..056dc89843 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -13,9 +13,9 @@ version = "0.3.4" [[Adapt]] deps = ["LinearAlgebra"] -git-tree-sha1 = "ffcfa2d345aaee0ef3d8346a073d5dd03c983ebe" +git-tree-sha1 = "f1b523983a58802c4695851926203b36e28f09db" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "3.2.0" +version = "3.3.0" [[ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -38,24 +38,22 @@ uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" version = "0.4.1" [[CUDA]] -deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "Memoize", "Printf", "Random", "Reexport", "Requires", "SparseArrays", "Statistics", "TimerOutputs"] -git-tree-sha1 = "381ba7285e069ba5a7817fe50d2d845888056d6a" -repo-rev = "master" -repo-url = "https://github.com/JuliaGPU/CUDA.jl.git" +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "Memoize", "Printf", "Random", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "TimerOutputs"] +git-tree-sha1 = "e837ae29695082ce4bfd754ae188e72dc2c5604c" uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "2.6.0" +version = "3.0.0" [[ChainRules]] deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Reexport", "Requires", "Statistics"] -git-tree-sha1 = "e01f521443e3700f40ad3c7c1c6aa3a6940aaea1" +git-tree-sha1 = "2b20bcac801e5aa0d12412205ffa533aa240af78" uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" -version = "0.7.54" +version = "0.7.59" [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "de4f08843c332d355852721adb1592bce7924da3" +git-tree-sha1 = "44e9f638aa9ed1ad58885defc568c133010140aa" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "0.9.29" +version = "0.9.37" [[CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] @@ -70,10 +68,10 @@ uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" version = "0.10.12" [[Colors]] -deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Reexport"] -git-tree-sha1 = "ac5f2213e56ed8a34a3dd2f681f4df1166b34929" +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "82f4e6ff9f847eca3e5ebc666ea2cd7b48e8b47e" uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" -version = "0.12.6" +version = "0.12.7" [[CommonSubexpressions]] deps = ["MacroTools", "Test"] @@ -83,9 +81,9 @@ version = "0.3.0" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "919c7f3151e79ff196add81d7f4e45d91bbf420b" +git-tree-sha1 = "ac4132ad78082518ec2037ae5770b6e796f7f956" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.25.0" +version = "3.27.0" [[CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] @@ -137,9 +135,9 @@ version = "0.1.3" [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays"] -git-tree-sha1 = "dd4ab4257c257532003eb9836eea07473fcc732e" +git-tree-sha1 = "31939159aeb8ffad1d4d8ee44d07f8558273120a" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "0.11.6" +version = "0.11.7" [[FixedPointNumbers]] deps = ["Statistics"] @@ -148,10 +146,10 @@ uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" version = "0.8.4" [[ForwardDiff]] -deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"] -git-tree-sha1 = "c68fb7481b71519d313114dca639b35262ff105f" +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "e2af66012e08966366a43251e1fd421522908be6" uuid = "f6369f11-7733-5829-9624-2563aa707210" -version = "0.10.17" +version = "0.10.18" [[Functors]] deps = ["MacroTools"] @@ -161,15 +159,15 @@ version = "0.2.1" [[GPUArrays]] deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"] -git-tree-sha1 = "f99a25fe0313121f2f9627002734c7d63b4dd3bd" +git-tree-sha1 = "9c95b2fd5c16bc7f97371e9f92f0fef77e0f5957" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "6.2.0" +version = "6.2.2" [[GPUCompiler]] deps = ["DataStructures", "ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "Serialization", "TimerOutputs", "UUIDs"] -git-tree-sha1 = "ef2839b063e158672583b9c09d2cf4876a8d3d55" +git-tree-sha1 = "a5a239b8c688f59872eb689edd75395e97cc6641" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "0.10.0" +version = "0.11.2" [[IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] @@ -267,16 +265,16 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" [[NNlib]] -deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"] -git-tree-sha1 = "ab1d43fead2ecb9aa5ae460d3d547c2cf8d89461" +deps = ["Adapt", "ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"] +git-tree-sha1 = "723c0d5252bf95808f934b2384519dd325869f40" uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" -version = "0.7.17" +version = "0.7.18" [[NNlibCUDA]] deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"] -git-tree-sha1 = "919a71e04789f802cac3d01cae4ca52d33955133" +git-tree-sha1 = "a38b154adc86b1637b7c4a69ac765f7499daf18c" repo-rev = "master" -repo-url = "https://github.com/FluxML/NNlibCUDA.jl.git" +repo-url = "https://github.com/FluxML/NNlibCUDA.jl" uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" version = "0.1.0" @@ -319,6 +317,12 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +[[RandomNumbers]] +deps = ["Random", "Requires"] +git-tree-sha1 = "441e6fc35597524ada7f85e13df1f4e10137d16f" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.4.0" + [[Reexport]] git-tree-sha1 = "57d8440b0c7d98fc4f889e478e80f268d534c9d5" uuid = "189a3867-3050-52da-a836-e630ba90ab69" @@ -367,9 +371,9 @@ version = "1.3.0" [[StaticArrays]] deps = ["LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "9da72ed50e94dbff92036da395275ed114e04d49" +git-tree-sha1 = "2f01a51c23eed210ff4a1be102c4cc8236b66e5b" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.0.1" +version = "1.1.0" [[Statistics]] deps = ["LinearAlgebra", "SparseArrays"] @@ -377,9 +381,9 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [[StatsBase]] deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"] -git-tree-sha1 = "400aa43f7de43aeccc5b2e39a76a79d262202b76" +git-tree-sha1 = "4bc58880426274277a066de306ef19ecc22a6863" uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.33.3" +version = "0.33.5" [[TOML]] deps = ["Dates"] @@ -423,10 +427,10 @@ deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" [[Zygote]] -deps = ["AbstractFFTs", "ChainRules", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "47dd50e831959f8785f8a9854783a2eebac8e1bd" +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "de86b4c5ff8e161c37bde0b5ecf6d201721373f8" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.4" +version = "0.6.9" [[ZygoteRules]] deps = ["MacroTools"] diff --git a/Project.toml b/Project.toml index 51f723c74e..c95432871f 100644 --- a/Project.toml +++ b/Project.toml @@ -28,8 +28,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] AbstractTrees = "0.3" -Adapt = "2.0, 3.0" -CUDA = "2.1" +Adapt = "3.0" CodecZlib = "0.7" Colors = "0.12" Functors = "0.2.1" @@ -40,7 +39,7 @@ Reexport = "0.2, 1.0" StatsBase = "0.33" ZipFile = "0.9" Zygote = "0.6" -julia = "1.5" +julia = "1.6" [extras] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" From 0f082494cce85015857e64957b9237114afaf3de Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 12 Apr 2021 15:29:01 +0530 Subject: [PATCH 06/14] rm 1.5 ci --- .buildkite/pipeline.yml | 16 +++------------- .github/workflows/ci.yml | 14 +++----------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d8c5d822d3..4dd12e2b57 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,18 +1,8 @@ steps: - - label: "GPU v1.5" + - label: "GPU integeration with julia v1.6" # change to "v1" when 1.6 is out plugins: - JuliaCI/julia#v1: - version: "1.5" - - JuliaCI/julia-test#v1: ~ - agents: - queue: "juliagpu" - cuda: "*" - timeout_in_minutes: 60 - - - label: "GPU v1.6-nightly" # change to "v1" when 1.6 is out - plugins: - - JuliaCI/julia#v1: - version: "1.6-nightly" # change to "1" when 1.6 is out + version: "1" - JuliaCI/julia-test#v1: ~ agents: queue: "juliagpu" @@ -27,4 +17,4 @@ steps: # agents: # queue: "juliagpu" # cuda: "*" - # timeout_in_minutes: 60 \ No newline at end of file + # timeout_in_minutes: 60 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2511bcc9e..8e8f32126f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,22 +17,14 @@ jobs: fail-fast: false matrix: version: - - '1.5' # Replace this with the minimum Julia version that your package supports. - - '1.6-nightly' # change to '1' when v1.6 is out + - '1' # Replace this with the minimum Julia version that your package supports. - 'nightly' os: - ubuntu-latest - # - macOS-latest - # - windows-latest + - macOS-latest + - windows-latest arch: - x64 - include: - - os: windows-latest - version: '1' - arch: x64 - - os: macOS-latest - version: '1' - arch: x64 steps: - uses: actions/checkout@v2 - uses: julia-actions/setup-julia@v1 From 6978d1c7c1af175cd91fe3bba84f8cf67b74a085 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 12 Apr 2021 16:16:02 +0530 Subject: [PATCH 07/14] em some CUDA.exp --- src/layers/normalise.jl | 9 ++++++++- src/losses/ctc-gpu.jl | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index ceac19c45c..8b2cf053dc 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -173,8 +173,10 @@ function _norm_layer_forward(l, x::AbstractArray{T,N}; reduce_dims, affine_shape μ = reshape(l.μ, stats_shape) σ² = reshape(l.σ², stats_shape) else # trainmode or testmode without tracked stats + # @show size(μ), size(σ²) μ = mean(x; dims=reduce_dims) σ² = mean((x .- μ).^2; dims=reduce_dims) + @show size(μ), size(σ²) if l.track_stats ## update moving mean/std Zygote.ignore() do @@ -182,14 +184,18 @@ function _norm_layer_forward(l, x::AbstractArray{T,N}; reduce_dims, affine_shape m = prod(size(x, i) for i in reduce_dims) # needed for computing corrected var μnew = vec(N ∈ reduce_dims ? μ : mean(μ, dims=N)) σ²new = vec(N ∈ reduce_dims ? σ² : mean(σ², dims=N)) + @show reduce_dims, N + @show size(μnew), size(σ²new) l.μ = (1-mtm) .* l.μ .+ mtm .* μnew l.σ² = (1-mtm) .* l.σ² .+ mtm .* (m / (m - one(eltype(l.σ²)))) .* σ²new end end end if hasaffine(l) + @show affine_shape γ = reshape(l.γ, affine_shape) β = reshape(l.β, affine_shape) + @show size.((μ,σ²,x, γ, β)) return l.λ.(γ .* (x .- μ) ./ sqrt.(σ² .+ l.ϵ) .+ β) else return l.λ.((x .- μ) ./ sqrt.(σ² .+ l.ϵ)) @@ -431,6 +437,7 @@ function (gn::GroupNorm)(x) x = reshape(x, sz[1:N-2]..., sz[N-1]÷gn.G, gn.G, sz[N]) N = ndims(x) reduce_dims = 1:N-2 + @show reduce_dims affine_shape = ntuple(i -> i ∈ (N-1, N-2) ? size(x, i) : 1, N) x = _norm_layer_forward(gn, x; reduce_dims, affine_shape) return reshape(x, sz) @@ -454,4 +461,4 @@ scale parameters, `false` otherwise. See [`BatchNorm`](@ref), [`InstanceNorm`](@ref), [`GroupNorm`](@ref), and [`LayerNorm`](@ref). """ -hasaffine(l::Union{BatchNorm, InstanceNorm, LayerNorm, GroupNorm}) = l.affine \ No newline at end of file +hasaffine(l::Union{BatchNorm, InstanceNorm, LayerNorm, GroupNorm}) = l.affine diff --git a/src/losses/ctc-gpu.jl b/src/losses/ctc-gpu.jl index 48934d1cae..b856215b61 100644 --- a/src/losses/ctc-gpu.jl +++ b/src/losses/ctc-gpu.jl @@ -21,7 +21,7 @@ function log_plus_f(p1, p2) if p1 < p2 p1, p2 = p2, p1 end - return p1 + CUDA.log(1+CUDA.exp(p2 - p1)) + return p1 + log(1+exp(p2 - p1)) end function count_repeats(A) @@ -139,7 +139,7 @@ function compute_beta_and_grad_kernel(probs, labelSize, uttLength, end # ∂L/∂a (where a is activation before logsoftmax) - grad[idx, T] = CUDA.exp(probs[idx, T]) - CUDA.exp(accum[idx, T] - s) + grad[idx, T] = exp(probs[idx, T]) - exp(accum[idx, T] - s) idx += blockDim().x end sync_threads() @@ -191,7 +191,7 @@ function compute_beta_and_grad_kernel(probs, labelSize, uttLength, while idx <= size(grad, 1) # ∂L/∂a (where a is activation before logsoftmax) - grad[idx, t] = CUDA.exp(probs[idx, t]) - CUDA.exp(accum[idx, t] + loss) + grad[idx, t] = exp(probs[idx, t]) - exp(accum[idx, t] + loss) idx += blockDim().x end sync_threads() From ac0c9e90325b68ff24991949c677e96d4df06880 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 12 Apr 2021 16:19:33 +0530 Subject: [PATCH 08/14] rm some CUDA.exp/log --- src/losses/ctc-gpu.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/losses/ctc-gpu.jl b/src/losses/ctc-gpu.jl index 48934d1cae..b856215b61 100644 --- a/src/losses/ctc-gpu.jl +++ b/src/losses/ctc-gpu.jl @@ -21,7 +21,7 @@ function log_plus_f(p1, p2) if p1 < p2 p1, p2 = p2, p1 end - return p1 + CUDA.log(1+CUDA.exp(p2 - p1)) + return p1 + log(1+exp(p2 - p1)) end function count_repeats(A) @@ -139,7 +139,7 @@ function compute_beta_and_grad_kernel(probs, labelSize, uttLength, end # ∂L/∂a (where a is activation before logsoftmax) - grad[idx, T] = CUDA.exp(probs[idx, T]) - CUDA.exp(accum[idx, T] - s) + grad[idx, T] = exp(probs[idx, T]) - exp(accum[idx, T] - s) idx += blockDim().x end sync_threads() @@ -191,7 +191,7 @@ function compute_beta_and_grad_kernel(probs, labelSize, uttLength, while idx <= size(grad, 1) # ∂L/∂a (where a is activation before logsoftmax) - grad[idx, t] = CUDA.exp(probs[idx, t]) - CUDA.exp(accum[idx, t] + loss) + grad[idx, t] = exp(probs[idx, t]) - exp(accum[idx, t] + loss) idx += blockDim().x end sync_threads() From be9bbbfee3c614e2c3f73ad08aba09714c04fe80 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 12 Apr 2021 16:20:53 +0530 Subject: [PATCH 09/14] revert work from other prs --- src/layers/normalise.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index 8b2cf053dc..877a1a58c8 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -173,10 +173,8 @@ function _norm_layer_forward(l, x::AbstractArray{T,N}; reduce_dims, affine_shape μ = reshape(l.μ, stats_shape) σ² = reshape(l.σ², stats_shape) else # trainmode or testmode without tracked stats - # @show size(μ), size(σ²) μ = mean(x; dims=reduce_dims) σ² = mean((x .- μ).^2; dims=reduce_dims) - @show size(μ), size(σ²) if l.track_stats ## update moving mean/std Zygote.ignore() do @@ -184,18 +182,14 @@ function _norm_layer_forward(l, x::AbstractArray{T,N}; reduce_dims, affine_shape m = prod(size(x, i) for i in reduce_dims) # needed for computing corrected var μnew = vec(N ∈ reduce_dims ? μ : mean(μ, dims=N)) σ²new = vec(N ∈ reduce_dims ? σ² : mean(σ², dims=N)) - @show reduce_dims, N - @show size(μnew), size(σ²new) l.μ = (1-mtm) .* l.μ .+ mtm .* μnew l.σ² = (1-mtm) .* l.σ² .+ mtm .* (m / (m - one(eltype(l.σ²)))) .* σ²new end end end if hasaffine(l) - @show affine_shape γ = reshape(l.γ, affine_shape) β = reshape(l.β, affine_shape) - @show size.((μ,σ²,x, γ, β)) return l.λ.(γ .* (x .- μ) ./ sqrt.(σ² .+ l.ϵ) .+ β) else return l.λ.((x .- μ) ./ sqrt.(σ² .+ l.ϵ)) @@ -437,7 +431,6 @@ function (gn::GroupNorm)(x) x = reshape(x, sz[1:N-2]..., sz[N-1]÷gn.G, gn.G, sz[N]) N = ndims(x) reduce_dims = 1:N-2 - @show reduce_dims affine_shape = ntuple(i -> i ∈ (N-1, N-2) ? size(x, i) : 1, N) x = _norm_layer_forward(gn, x; reduce_dims, affine_shape) return reshape(x, sz) From 98e3a563f4bc39209749a07811a45737eeda0303 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Sun, 18 Apr 2021 09:05:28 +0530 Subject: [PATCH 10/14] bump pkg --- Manifest.toml | 27 ++++++++++++++++----------- Project.toml | 1 + 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 056dc89843..0b030f4e8b 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -39,15 +39,15 @@ version = "0.4.1" [[CUDA]] deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "Memoize", "Printf", "Random", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "TimerOutputs"] -git-tree-sha1 = "e837ae29695082ce4bfd754ae188e72dc2c5604c" +git-tree-sha1 = "a6ce96dcf22fc4f1bfdfac02d54f0b77ecf2a4cc" uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "3.0.0" +version = "3.0.3" [[ChainRules]] deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Reexport", "Requires", "Statistics"] -git-tree-sha1 = "2b20bcac801e5aa0d12412205ffa533aa240af78" +git-tree-sha1 = "a426b3526dffff05ef3eaab35d6dc2869ec5846a" uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" -version = "0.7.59" +version = "0.7.60" [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] @@ -165,9 +165,9 @@ version = "6.2.2" [[GPUCompiler]] deps = ["DataStructures", "ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "Serialization", "TimerOutputs", "UUIDs"] -git-tree-sha1 = "a5a239b8c688f59872eb689edd75395e97cc6641" +git-tree-sha1 = "6eadd2321dc3ac0fc9d530ab01c2caa7fe5d74c6" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "0.11.2" +version = "0.11.4" [[IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] @@ -180,9 +180,10 @@ deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[JLLWrappers]] -git-tree-sha1 = "a431f5f2ca3f4feef3bd7a5e94b8b8d4f2f647a0" +deps = ["Preferences"] +git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.2.0" +version = "1.3.0" [[Juno]] deps = ["Base64", "Logging", "Media", "Profile"] @@ -272,9 +273,7 @@ version = "0.7.18" [[NNlibCUDA]] deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"] -git-tree-sha1 = "a38b154adc86b1637b7c4a69ac765f7499daf18c" -repo-rev = "master" -repo-url = "https://github.com/FluxML/NNlibCUDA.jl" +git-tree-sha1 = "4b368b466bcdd25d448a5b20de4b7e481d68b88e" uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" version = "0.1.0" @@ -301,6 +300,12 @@ version = "1.4.0" deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +[[Preferences]] +deps = ["TOML"] +git-tree-sha1 = "ea79e4c9077208cd3bc5d29631a26bc0cff78902" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.2.1" + [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" diff --git a/Project.toml b/Project.toml index c95432871f..aee8857f8c 100644 --- a/Project.toml +++ b/Project.toml @@ -29,6 +29,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] AbstractTrees = "0.3" Adapt = "3.0" +CUDA = "3" CodecZlib = "0.7" Colors = "0.12" Functors = "0.2.1" From 768aecdddb95798b00f6f6d72736047d7146774c Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 19 Apr 2021 15:33:59 +0530 Subject: [PATCH 11/14] bump version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index ee90d9c7dc..5331b19346 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Flux" uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" -version = "0.12.1" +version = "0.12.2" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" From 7fd98f048bdd542fac6d6b42d886e7057440c9d9 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 19 Apr 2021 19:03:28 +0530 Subject: [PATCH 12/14] rm comment --- src/cuda/cuda.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cuda/cuda.jl b/src/cuda/cuda.jl index 62d3195032..0ef3c65308 100644 --- a/src/cuda/cuda.jl +++ b/src/cuda/cuda.jl @@ -1,7 +1,6 @@ module CUDAint using ..CUDA -# using CUDA: CUDNN import ..Flux: Flux import Zygote From f600fb78678a067d21ee0a1a218f62e1828068bd Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 19 Apr 2021 21:17:06 +0530 Subject: [PATCH 13/14] add NNlibCUDA compat --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 5331b19346..8e2f70db19 100644 --- a/Project.toml +++ b/Project.toml @@ -36,6 +36,7 @@ Functors = "0.2.1" Juno = "0.8" MacroTools = "0.5" NNlib = "0.7.14" +NNlibCUDA = "0.1" Reexport = "0.2, 1.0" StatsBase = "0.33" ZipFile = "0.9" From e5325c041f4f96b41bb92c3f307428fb4292058b Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Fri, 23 Apr 2021 17:44:12 +0530 Subject: [PATCH 14/14] pkg up --- Manifest.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 0b030f4e8b..455ea5bddf 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -45,15 +45,15 @@ version = "3.0.3" [[ChainRules]] deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Reexport", "Requires", "Statistics"] -git-tree-sha1 = "a426b3526dffff05ef3eaab35d6dc2869ec5846a" +git-tree-sha1 = "1f410fba5c04d03ab712f348f1542e6059376547" uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" -version = "0.7.60" +version = "0.7.61" [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "44e9f638aa9ed1ad58885defc568c133010140aa" +git-tree-sha1 = "42e3c181483fbd2c416087a0a93838803e358358" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "0.9.37" +version = "0.9.38" [[CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] @@ -267,9 +267,9 @@ uuid = "14a3606d-f60d-562e-9121-12d972cd8159" [[NNlib]] deps = ["Adapt", "ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"] -git-tree-sha1 = "723c0d5252bf95808f934b2384519dd325869f40" +git-tree-sha1 = "80b8360670f445d88b3475e88b33bbcc92f7866e" uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" -version = "0.7.18" +version = "0.7.19" [[NNlibCUDA]] deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"] @@ -376,9 +376,9 @@ version = "1.3.0" [[StaticArrays]] deps = ["LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "2f01a51c23eed210ff4a1be102c4cc8236b66e5b" +git-tree-sha1 = "e8cd1b100d37f5b4cfd2c83f45becf61c762eaf7" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.1.0" +version = "1.1.1" [[Statistics]] deps = ["LinearAlgebra", "SparseArrays"] @@ -433,9 +433,9 @@ uuid = "83775a58-1f1d-513f-b197-d71354ab007a" [[Zygote]] deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "de86b4c5ff8e161c37bde0b5ecf6d201721373f8" +git-tree-sha1 = "927209c83efa62256788a9880c191774c07c5b51" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.9" +version = "0.6.10" [[ZygoteRules]] deps = ["MacroTools"]