Merge #1299

1299: Fix ADADelta calculations and broken tests not catching the problems r=DhairyaLGandhi a=scimas

29832ac broke ADADelta, simply reversing its change and adding comments to not move around the epsilons. Fixes #1158 

The testset probably needs more work than specifically for this case, so I'm not adding it here.
### PR Checklist

- [x] Tests are ~added~ fixed
- [ ] Entry in NEWS.md
- [x] Documentation, if applicable
- [ ] Final review from `@dhairyagandhi96` (for API changes).


Co-authored-by: Mihir Gadgil <16473290+scimas@users.noreply.github.com>

Author: bors[bot]

src/optimise/optimisers.jl

@@ -283,7 +283,7 @@ mutable struct OADAM
   state::IdDict
 end
 
-OADAM(η = 0.0001, β = (0.5, 0.9)) = OADAM(η, β, IdDict())
+OADAM(η = 0.001, β = (0.5, 0.9)) = OADAM(η, β, IdDict())
 
 function apply!(o::OADAM, x, Δ)
   η, β = o.eta, o.beta
@@ -357,7 +357,9 @@ function apply!(o::ADADelta, x, Δ)
   ρ = o.rho
   acc, Δacc = get!(o.state, x, (zero(x), zero(x)))
   @. acc = ρ * acc + (1 - ρ) * Δ^2
-  @. Δ *= √Δacc/ (√acc + ϵ)
+  # DON'T remove epsilon from numerator
+  # or even out of the square roots
+  @. Δ *= √(Δacc + ϵ) / √(acc + ϵ)
   @. Δacc = ρ * Δacc + (1 - ρ) * Δ^2
   return Δ
 end
@@ -599,4 +601,4 @@ function apply!(o::ClipNorm, x, Δ)
         rmul!(Δ, o.thresh / Δnrm)
     end
     return Δ
-end
+end

test/optimise.jl

@@ -2,12 +2,17 @@ using Flux.Optimise
 using Flux.Optimise: runall
 using Flux: Params, gradient
 using Test
+using Random
 
 @testset "Optimise" begin
+  # Ensure rng has different state inside and outside the inner @testset
+  # so that w and w' are different
+  Random.seed!(84)
   w = randn(10, 10)
   @testset for opt in [ADAMW(), ADAGrad(0.1), AdaMax(), ADADelta(0.9), AMSGrad(),
                        NADAM(), RADAM(), Descent(0.1), ADAM(), OADAM(), Nesterov(), RMSProp(),
                        Momentum()]
+    Random.seed!(42)
     w′ = randn(10, 10)
     loss(x) = Flux.Losses.mse(w*x, w′*x)
     for t = 1: 10^5
@@ -21,8 +26,10 @@ using Test
 end
 
 @testset "Optimiser" begin
+  Random.seed!(84)
   w = randn(10, 10)
   @testset for Opt in [InvDecay, WeightDecay, ExpDecay]
+    Random.seed!(42)
     w′ = randn(10, 10)
     loss(x) = Flux.Losses.mse(w*x, w′*x)
     opt = Optimiser(Opt(), ADAM(0.001))