Linesearch + Merit functions

.gitignore

@@ -1,2 +1,4 @@
 docs/build
 docs/Manifest.toml
+
+Manifest.toml

Project.toml

@@ -19,10 +19,11 @@ NLPModels = "0.13"
 julia = "^1.3.0"
 
 [extras]
+Krylov = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["NLPModels", "LinearAlgebra", "Test", "Logging"]
+test = ["Krylov", "NLPModels", "LinearAlgebra", "Test", "Logging"]

docs/src/api.md

@@ -18,13 +18,23 @@ project_step!
 ## Line-Search
 
 ```@docs
-LineModel
+linesearch
+LineSearchOutput
+armijo!
+armijo_wolfe!
+```
+
+## Merit
+
+```@docs
+AbstractMeritModel
 obj
-grad
-grad!
-hess
-redirect!
-armijo_wolfe
+dualobj
+primalobj
+derivative
+L1Merit
+AugLagMerit
+UncMerit
 ```
 
 ## Stats
@@ -38,13 +48,7 @@ show_statuses
 
 ```@docs
 TrustRegionException
-SolverTools.AbstractTrustRegion
-aredpred
-acceptable
-reset!
-get_property
-set_property!
-update!
-TrustRegion
-TRONTrustRegion
+trust_region
+basic_trust_region!
+tron_trust_region!
 ```

docs/src/reference.md

@@ -1,4 +1,12 @@
 # Reference
 
-```@index
+## Contents
+
+```@contents
+Pages = ["reference.md"]
 ```
+
+## Index
+
+```@index
+```

src/SolverTools.jl

@@ -16,6 +16,7 @@ include("auxiliary/logger.jl")
 include("stats/stats.jl")
 
 # Algorithmic components.
+include("merit/merit.jl")
 include("linesearch/linesearch.jl")
 include("trust-region/trust-region.jl")
 

src/linesearch/armijo.jl

@@ -0,0 +1,57 @@
+export armijo!
+
+"""
+    lso = armijo!(ϕ, x, d, xt; kwargs...)
+
+Performs a line search from `x` along the direction `d` for `AbstractMeritModel` `ϕ`.
+The steplength is chosen trying to satisfy the Armijo condition
+```math
+ϕ(x + t d; η) ≤ ϕ(x) + τ₀ t Dϕ(x; η),
+```
+using backtracking.
+
+The keyword aguments are:
+- ...
+
+The output is a `LineSearchOutput`. The following specific information are also available in
+`lso.specific`:
+- nbk: the number of times the steplength was decreased to satisfy the Armijo condition, i.e.,
+  number of backtracks;
+
+The following keyword arguments can be provided:
+- `update_obj_at_x`: Whether to call `obj(ϕ, x; update=true)` to update `ϕ.fx` and `ϕ.cx` (default: `false`);
+- `update_derivative_at_x`: Whether to call `derivative(ϕ, x, d; update=true)` to update `ϕ.gx` and `ϕ.Ad` (default: `false`);
+- `t`: starting steplength (default `1`);
+- `τ₀`: slope factor in the Armijo condition (default `max(1e-4, √ϵₘ)`);
+- `σ`: backtracking decrease factor (default `0.4`);
+- `bk_max`: maximum number of backtracks (default `10`).
+"""
+function armijo!(
+  ϕ :: AbstractMeritModel{M,T,V},
+  x :: V,
+  d :: V,
+  xt :: V;
+  update_obj_at_x :: Bool = false,
+  update_derivative_at_x :: Bool = false,
+  t :: T=one(T),
+  τ₀ :: T=max(T(1.0e-4), sqrt(eps(T))),
+  σ :: T=T(0.4),
+  bk_max :: Int=10
+) where {M <: AbstractNLPModel, T <: Real, V <: AbstractVector{<: T}}
+
+  nbk = 0
+  ϕx = obj(ϕ, x; update=update_obj_at_x)
+  slope = derivative(ϕ, x, d; update=update_derivative_at_x)
+
+  xt .= x .+ t .* d
+  ϕt = obj(ϕ, xt)
+  while !(ϕt ≤ ϕx + τ₀ * t * slope) && (nbk < bk_max)
+    t *= σ
+    xt .= x .+ t .* d
+    ϕt = obj(ϕ, xt)
+
+    nbk += 1
+  end
+
+  return LineSearchOutput(t, xt, ϕt, specific=(nbk=nbk,))
+end

src/linesearch/armijo_wolfe.jl

@@ -1,84 +1,91 @@
-export armijo_wolfe
+export armijo_wolfe!
 
 """
-    t, good_grad, ht, nbk, nbW = armijo_wolfe(h, h₀, slope, g)
+    lso = armijo_wolfe!(ϕ, x, d, xt; kwargs...)
 
-Performs a line search from `x` along the direction `d` as defined by the
-`LineModel` ``h(t) = f(x + t d)``, where
-`h₀ = h(0) = f(x)`, `slope = h'(0) = ∇f(x)ᵀd` and `g` is a vector that will be
-overwritten with the gradient at various points. On exit, if `good_grad=true`,
-`g` contains the gradient at the final step length.
-The steplength is chosen trying to satisfy the Armijo and Wolfe conditions. The Armijo condition is
+Performs a line search from `x` along the direction `d` for `AbstractMeritModel` `ϕ`.
+The steplength is chosen trying to satisfy the Armijo and Wolfe conditions.
+The Armijo condition is
 ```math
-h(t) ≤ h₀ + τ₀ t h'(0)
+ϕ(x + t d; η) ≤ ϕ(x) + τ₀ t Dϕ(x; d, η)
 ```
 and the Wolfe condition is
 ```math
-h'(t) ≤ τ₁ h'(0).
+Dϕ(x + t d; d, η) ≤ τ₁ Dϕ(x; d, η).
 ```
-Initially the step is increased trying to satisfy the Wolfe condition.
-Afterwards, only backtracking is performed in order to try to satisfy the Armijo condition.
-The final steplength may only satisfy Armijo's condition.
+Initially the step is increased trying to satisfy the Wolfe condition. Afterwards, only backtracking
+is performed in order to try to satisfy the Armijo condition. The final steplength may only satisfy
+Armijo's condition.
 
-The output is the following:
-- t: the step length;
-- good_grad: whether `g` is the gradient at `x + t * d`;
-- ht: the model value at `t`, i.e., `f(x + t * d)`;
-- nbk: the number of times the steplength was decreased to satisfy the Armijo condition, i.e., number of backtracks;
+The output is a `LineSearchOutput`. The following specific information are also available in
+`lso.specific`:
+- nbk: the number of times the steplength was decreased to satisfy the Armijo condition, i.e.,
+  number of backtracks;
 - nbW: the number of times the steplength was increased to satisfy the Wolfe condition.
 
 The following keyword arguments can be provided:
+- `update_obj_at_x`: Whether to call `obj(ϕ, x; update=true)` to update `ϕ.fx` and `ϕ.cx` (default: `false`);
+- `update_derivative_at_x`: Whether to call `derivative(ϕ, x, d; update=true)` to update `ϕ.gx` and `ϕ.Ad` (default: `false`);
 - `t`: starting steplength (default `1`);
 - `τ₀`: slope factor in the Armijo condition (default `max(1e-4, √ϵₘ)`);
 - `τ₁`: slope factor in the Wolfe condition. It should satisfy `τ₁ > τ₀` (default `0.9999`);
+- `σ`: backtracking decrease factor (default `0.4`);
 - `bk_max`: maximum number of backtracks (default `10`);
-- `bW_max`: maximum number of increases (default `5`);
-- `verbose`: whether to print information (default `false`).
+- `bW_max`: maximum number of increases (default `5`).
 """
-function armijo_wolfe(h :: LineModel,
-                      h₀ :: T,
-                      slope :: T,
-                      g :: Array{T,1};
-                      t :: T=one(T),
-                      τ₀ :: T=max(T(1.0e-4), sqrt(eps(T))),
-                      τ₁ :: T=T(0.9999),
-                      bk_max :: Int=10,
-                      bW_max :: Int=5,
-                      verbose :: Bool=false) where T <: AbstractFloat
+function armijo_wolfe!(
+  ϕ :: AbstractMeritModel{M,T,V},
+  x :: V,
+  d :: V,
+  xt :: V;
+  update_obj_at_x :: Bool = false,
+  update_derivative_at_x :: Bool = false,
+  t :: T=one(T),
+  τ₀ :: T=max(T(1.0e-4), sqrt(eps(T))),
+  τ₁ :: T=T(0.9999),
+  σ :: T=T(0.4),
+  bk_max :: Int=10,
+  bW_max :: Int=5
+) where {M <: AbstractNLPModel, T <: Real, V <: AbstractVector{<: T}}
 
   # Perform improved Armijo linesearch.
   nbk = 0
   nbW = 0
+  ϕx = obj(ϕ, x; update=update_obj_at_x)
+  slope = derivative(ϕ, x, d; update=update_derivative_at_x)
 
   # First try to increase t to satisfy loose Wolfe condition
-  ht = obj(h, t)
-  slope_t = grad!(h, t, g)
-  while (slope_t < τ₁*slope) && (ht <= h₀ + τ₀ * t * slope) && (nbW < bW_max)
+  xt .= x .+ t .* d
+  ϕt = obj(ϕ, xt)
+  slope_t = derivative(ϕ, xt, d)
+  while (slope_t < τ₁*slope) && (ϕt <= ϕx + τ₀ * t * slope) && (nbW < bW_max)
     t *= 5
-    ht = obj(h, t)
-    slope_t = grad!(h, t, g)
+    xt .= x .+ t .* d
+    ϕt = obj(ϕ, xt)
+    slope_t = derivative(ϕ, xt, d)
     nbW += 1
   end
 
-  hgoal = h₀ + slope * t * τ₀;
+  ϕgoal = ϕx + slope * t * τ₀;
   fact = -T(0.8)
   ϵ = eps(T)^T(3/5)
 
   # Enrich Armijo's condition with Hager & Zhang numerical trick
-  Armijo = (ht <= hgoal) || ((ht <= h₀ + ϵ * abs(h₀)) && (slope_t <= fact * slope))
+  Armijo = (ϕt <= ϕgoal) || ((ϕt <= ϕx + ϵ * abs(ϕx)) && (slope_t <= fact * slope))
   good_grad = true
   while !Armijo && (nbk < bk_max)
-    t *= T(0.4)
-    ht = obj(h, t)
-    hgoal = h₀ + slope * t * τ₀;
+    t *= σ
+    xt .= x .+ t .* d
+    ϕt = obj(ϕ, xt)
+    ϕgoal = ϕx + slope * t * τ₀;
 
     # avoids unused grad! calls
     Armijo = false
     good_grad = false
-    if ht <= hgoal
+    if ϕt <= ϕgoal
       Armijo = true
-    elseif ht <= h₀ + ϵ * abs(h₀)
-      slope_t = grad!(h, t, g)
+    elseif ϕt <= ϕx + ϵ * abs(ϕx)
+      slope_t = derivative!(ϕ, xt, d)
       good_grad = true
       if slope_t <= fact * slope
         Armijo = true
@@ -88,7 +95,5 @@ function armijo_wolfe(h :: LineModel,
     nbk += 1
   end
 
-  verbose && @printf("  %4d %4d\n", nbk, nbW);
-
-  return (t, good_grad, ht, nbk, nbW)
+  return LineSearchOutput(t, xt, ϕt, good_grad=good_grad, gt=ϕ.gx, specific=(nbk=nbk, nbW=nbW))
 end