Skip to content

Commit 67b1514

Browse files
alystalyst
committed
BlackBoxOptim.jl backend support
1 parent 58623c3 commit 67b1514

File tree

6 files changed

+441
-0
lines changed

6 files changed

+441
-0
lines changed

‎Project.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,12 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
4848
test = ["Test"]
4949

5050
[weakdeps]
51+
BlackBoxOptim = "a134a8b2-14d6-55f6-9291-3336d3ab0209"
5152
NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd"
53+
Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
5254
ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9"
5355

5456
[extensions]
5557
SEMNLOptExt = "NLopt"
5658
SEMProximalOptExt = "ProximalAlgorithms"
59+
SEMBlackBoxOptimExt = ["BlackBoxOptim", "Optimisers"]

‎ext/SEMBlackBoxOptimExt/AdamMutation.jl‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# mutate by moving in the gradient direction
2+
mutable struct AdamMutation{M <: AbstractSem, O, S} <: MutationOperator
3+
model::M
4+
optim::O
5+
opt_state::S
6+
params_fraction::Float64
7+
8+
function AdamMutation(model::AbstractSem, params::AbstractDict)
9+
optim = RAdam(params[:AdamMutation_eta], params[:AdamMutation_beta])
10+
params_fraction = params[:AdamMutation_params_fraction]
11+
opt_state = Optimisers.init(optim, Vector{Float64}(undef, nparams(model)))
12+
13+
new{typeof(model), typeof(optim), typeof(opt_state)}(
14+
model,
15+
optim,
16+
opt_state,
17+
params_fraction,
18+
)
19+
end
20+
end
21+
22+
Base.show(io::IO, op::AdamMutation) =
23+
print(io, "AdamMutation(", op.optim, " state[3]=", op.opt_state[3], ")")
24+
25+
"""
26+
Default parameters for `AdamMutation`.
27+
"""
28+
const AdamMutation_DefaultOptions = ParamsDict(
29+
:AdamMutation_eta => 1E-1,
30+
:AdamMutation_beta => (0.99, 0.999),
31+
:AdamMutation_params_fraction => 0.25,
32+
)
33+
34+
function BlackBoxOptim.apply!(m::AdamMutation, v::AbstractVector{<:Real}, target_index::Int)
35+
grad = similar(v)
36+
obj = SEM.evaluate!(0.0, grad, nothing, m.model, v)
37+
@inbounds for i in eachindex(grad)
38+
(rand() > m.params_fraction) && (grad[i] = 0.0)
39+
end
40+
41+
m.opt_state, dv = Optimisers.apply!(m.optim, m.opt_state, v, grad)
42+
if (m.opt_state[3][1] <= 1E-20) || !isfinite(obj) || any(!isfinite, dv)
43+
m.opt_state = Optimisers.init(m.optim, v)
44+
else
45+
v .-= dv
46+
end
47+
48+
return v
49+
end

‎ext/SEMBlackBoxOptimExt/BlackBoxOptim.jl‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
############################################################################################
2+
### connect to BlackBoxOptim.jl as backend
3+
############################################################################################
4+
5+
"""
6+
"""
7+
struct SemOptimizerBlackBoxOptim <: SemOptimizer{:BlackBoxOptim}
8+
lower_bound::Float64 # default lower bound
9+
variance_lower_bound::Float64 # default variance lower bound
10+
lower_bounds::Union{Dict{Symbol, Float64}, Nothing}
11+
12+
upper_bound::Float64 # default upper bound
13+
upper_bounds::Union{Dict{Symbol, Float64}, Nothing}
14+
end
15+
16+
function SemOptimizerBlackBoxOptim(;
17+
lower_bound::Float64 = -1000.0,
18+
lower_bounds::Union{AbstractDict{Symbol, Float64}, Nothing} = nothing,
19+
variance_lower_bound::Float64 = 0.001,
20+
upper_bound::Float64 = 1000.0,
21+
upper_bounds::Union{AbstractDict{Symbol, Float64}, Nothing} = nothing,
22+
kwargs...,
23+
)
24+
if variance_lower_bound < 0.0
25+
throw(ArgumentError("variance_lower_bound must be non-negative"))
26+
end
27+
return SemOptimizerBlackBoxOptim(
28+
lower_bound,
29+
variance_lower_bound,
30+
lower_bounds,
31+
upper_bound,
32+
upper_bounds,
33+
)
34+
end
35+
36+
SEM.SemOptimizer{:BlackBoxOptim}(args...; kwargs...) =
37+
SemOptimizerBlackBoxOptim(args...; kwargs...)
38+
39+
SEM.algorithm(optimizer::SemOptimizerBlackBoxOptim) = optimizer.algorithm
40+
SEM.options(optimizer::SemOptimizerBlackBoxOptim) = optimizer.options
41+
42+
struct SemModelBlackBoxOptimProblem{M <: AbstractSem} <:
43+
OptimizationProblem{ScalarFitnessScheme{true}}
44+
model::M
45+
fitness_scheme::ScalarFitnessScheme{true}
46+
search_space::ContinuousRectSearchSpace
47+
end
48+
49+
function BlackBoxOptim.search_space(model::AbstractSem)
50+
optim = model.optimizer::SemOptimizerBlackBoxOptim
51+
varparams = Set(SEM.variance_params(model.implied.ram_matrices))
52+
return ContinuousRectSearchSpace(
53+
[
54+
begin
55+
def = in(p, varparams) ? optim.variance_lower_bound : optim.lower_bound
56+
isnothing(optim.lower_bounds) ? def : get(optim.lower_bounds, p, def)
57+
end for p in SEM.params(model)
58+
],
59+
[
60+
begin
61+
def = optim.upper_bound
62+
isnothing(optim.upper_bounds) ? def : get(optim.upper_bounds, p, def)
63+
end for p in SEM.params(model)
64+
],
65+
)
66+
end
67+
68+
function SemModelBlackBoxOptimProblem(
69+
model::AbstractSem,
70+
optimizer::SemOptimizerBlackBoxOptim,
71+
)
72+
SemModelBlackBoxOptimProblem(model, ScalarFitnessScheme{true}(), search_space(model))
73+
end
74+
75+
BlackBoxOptim.fitness(params::AbstractVector, wrapper::SemModelBlackBoxOptimProblem) =
76+
return SEM.evaluate!(0.0, nothing, nothing, wrapper.model, params)
77+
78+
# sem_fit method
79+
function SEM.sem_fit(
80+
optimizer::SemOptimizerBlackBoxOptim,
81+
model::AbstractSem,
82+
start_params::AbstractVector;
83+
MaxSteps::Integer = 50000,
84+
kwargs...,
85+
)
86+
problem = SemModelBlackBoxOptimProblem(model, optimizer)
87+
res = bboptimize(problem; MaxSteps, kwargs...)
88+
return SemFit(best_fitness(res), best_candidate(res), nothing, model, res)
89+
end

‎ext/SEMBlackBoxOptimExt/DiffEvoFactory.jl‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
"""
2+
Base class for factories of optimizers for a specific problem.
3+
"""
4+
abstract type OptimizerFactory{P <: OptimizationProblem} end
5+
6+
problem(factory::OptimizerFactory) = factory.problem
7+
8+
const OptController_DefaultParameters = ParamsDict(
9+
:MaxTime => 60.0,
10+
:MaxSteps => 10^8,
11+
:TraceMode => :compact,
12+
:TraceInterval => 5.0,
13+
:RecoverResults => false,
14+
:SaveTrace => false,
15+
)
16+
17+
function generate_opt_controller(alg::Optimizer, optim_factory::OptimizerFactory, params)
18+
return BlackBoxOptim.OptController(
19+
alg,
20+
problem(optim_factory),
21+
BlackBoxOptim.chain(
22+
BlackBoxOptim.DefaultParameters,
23+
OptController_DefaultParameters,
24+
params,
25+
),
26+
)
27+
end
28+
29+
function check_population(
30+
factory::OptimizerFactory,
31+
popmatrix::BlackBoxOptim.PopulationMatrix,
32+
)
33+
ssp = factory |> problem |> search_space
34+
for i in 1:popsize(popmatrix)
35+
@assert popmatrix[:, i] ∈ ssp "Individual $i is out of space: $(popmatrix[:,i])" # fitness: $(fitness(population, i))"
36+
end
37+
end
38+
39+
initial_search_space(factory::OptimizerFactory, id::Int) = search_space(factory.problem)
40+
41+
function initial_population_matrix(factory::OptimizerFactory, id::Int)
42+
#@info "Standard initial_population_matrix()"
43+
ini_ss = initial_search_space(factory, id)
44+
if !isempty(factory.initial_population)
45+
numdims(factory.initial_population) == numdims(factory.problem) || throw(
46+
DimensionMismatch(
47+
"Dimensions of :Population ($(numdims(factory.initial_population))) " *
48+
"are different from the problem dimensions ($(numdims(factory.problem)))",
49+
),
50+
)
51+
res = factory.initial_population[
52+
:,
53+
StatsBase.sample(
54+
1:popsize(factory.initial_population),
55+
factory.population_size,
56+
),
57+
]
58+
else
59+
res = rand_individuals(ini_ss, factory.population_size, method = :latin_hypercube)
60+
end
61+
prj = RandomBound(ini_ss)
62+
if size(res, 2) > 1
63+
apply!(prj, view(res, :, 1), SEM.start_fabin3(factory.problem.model))
64+
end
65+
if size(res, 2) > 2
66+
apply!(prj, view(res, :, 2), SEM.start_simple(factory.problem.model))
67+
end
68+
return res
69+
end
70+
71+
# convert individuals in the archive into population matrix
72+
population_matrix(archive::Any) = population_matrix!(
73+
Matrix{Float64}(undef, length(BlackBoxOptim.params(first(archive))), length(archive)),
74+
archive,
75+
)
76+
77+
function population_matrix!(pop::AbstractMatrix{<:Real}, archive::Any)
78+
npars = length(BlackBoxOptim.params(first(archive)))
79+
size(pop, 1) == npars || throw(
80+
DimensionMismatch(
81+
"Matrix rows count ($(size(pop, 1))) doesn't match the number of problem dimensions ($(npars))",
82+
),
83+
)
84+
@inbounds for (i, indi) in enumerate(archive)
85+
(i <= size(pop, 2)) || break
86+
pop[:, i] .= BlackBoxOptim.params(indi)
87+
end
88+
if size(pop, 2) > length(archive)
89+
@warn "Matrix columns count ($(size(pop, 2))) is bigger than population size ($(length(archive))), last columns not set"
90+
end
91+
return pop
92+
end
93+
94+
generate_embedder(factory::OptimizerFactory, id::Int, problem::OptimizationProblem) =
95+
RandomBound(search_space(problem))
96+
97+
abstract type DiffEvoFactory{P <: OptimizationProblem} <: OptimizerFactory{P} end
98+
99+
generate_selector(
100+
factory::DiffEvoFactory,
101+
id::Int,
102+
problem::OptimizationProblem,
103+
population,
104+
) = RadiusLimitedSelector(get(factory.params, :selector_radius, popsize(population) ÷ 5))
105+
106+
function generate_modifier(factory::DiffEvoFactory, id::Int, problem::OptimizationProblem)
107+
ops = GeneticOperator[
108+
MutationClock(UniformMutation(search_space(problem)), 1 / numdims(problem)),
109+
BlackBoxOptim.AdaptiveDiffEvoRandBin1(
110+
BlackBoxOptim.AdaptiveDiffEvoParameters(
111+
factory.params[:fdistr],
112+
factory.params[:crdistr],
113+
),
114+
),
115+
SimplexCrossover{3}(1.05),
116+
SimplexCrossover{2}(1.1),
117+
#SimulatedBinaryCrossover(0.05, 16.0),
118+
#SimulatedBinaryCrossover(0.05, 3.0),
119+
#SimulatedBinaryCrossover(0.1, 5.0),
120+
#SimulatedBinaryCrossover(0.2, 16.0),
121+
UnimodalNormalDistributionCrossover{2}(
122+
chain(BlackBoxOptim.UNDX_DefaultOptions, factory.params),
123+
),
124+
UnimodalNormalDistributionCrossover{3}(
125+
chain(BlackBoxOptim.UNDX_DefaultOptions, factory.params),
126+
),
127+
ParentCentricCrossover{2}(chain(BlackBoxOptim.PCX_DefaultOptions, factory.params)),
128+
ParentCentricCrossover{3}(chain(BlackBoxOptim.PCX_DefaultOptions, factory.params)),
129+
]
130+
if problem isa SemModelBlackBoxOptimProblem
131+
push!(
132+
ops,
133+
AdamMutation(problem.model, chain(AdamMutation_DefaultOptions, factory.params)),
134+
)
135+
end
136+
FAGeneticOperatorsMixture(ops)
137+
end
138+
139+
function generate_optimizer(
140+
factory::DiffEvoFactory,
141+
id::Int,
142+
problem::OptimizationProblem,
143+
popmatrix,
144+
)
145+
population = FitPopulation(popmatrix, nafitness(fitness_scheme(problem)))
146+
BlackBoxOptim.DiffEvoOpt(
147+
"AdaptiveDE/rand/1/bin/gradient",
148+
population,
149+
generate_selector(factory, id, problem, population),
150+
generate_modifier(factory, id, problem),
151+
generate_embedder(factory, id, problem),
152+
)
153+
end
154+
155+
const Population_DefaultParameters = ParamsDict(
156+
:Population => BlackBoxOptim.PopulationMatrix(undef, 0, 0),
157+
:PopulationSize => 100,
158+
)
159+
160+
const DE_DefaultParameters = chain(
161+
ParamsDict(
162+
:SelectorRadius => 0,
163+
:fdistr =>
164+
BlackBoxOptim.BimodalCauchy(0.65, 0.1, 1.0, 0.1, clampBelow0 = false),
165+
:crdistr =>
166+
BlackBoxOptim.BimodalCauchy(0.1, 0.1, 0.95, 0.1, clampBelow0 = false),
167+
),
168+
Population_DefaultParameters,
169+
)
170+
171+
struct DefaultDiffEvoFactory{P <: OptimizationProblem} <: DiffEvoFactory{P}
172+
problem::P
173+
initial_population::BlackBoxOptim.PopulationMatrix
174+
population_size::Int
175+
params::ParamsDictChain
176+
end
177+
178+
DefaultDiffEvoFactory(problem::OptimizationProblem; kwargs...) =
179+
DefaultDiffEvoFactory(problem, BlackBoxOptim.kwargs2dict(kwargs))
180+
181+
function DefaultDiffEvoFactory(problem::OptimizationProblem, params::AbstractDict)
182+
params = chain(DE_DefaultParameters, params)
183+
DefaultDiffEvoFactory{typeof(problem)}(
184+
problem,
185+
params[:Population],
186+
params[:PopulationSize],
187+
params,
188+
)
189+
end
190+
191+
function BlackBoxOptim.bbsetup(factory::OptimizerFactory; kwargs...)
192+
popmatrix = initial_population_matrix(factory, 1)
193+
check_population(factory, popmatrix)
194+
alg = generate_optimizer(factory, 1, problem(factory), popmatrix)
195+
return generate_opt_controller(alg, factory, BlackBoxOptim.kwargs2dict(kwargs))
196+
end

‎ext/SEMBlackBoxOptimExt/SEMBlackBoxOptimExt.jl‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
module SEMBlackBoxOptimExt
2+
3+
using StructuralEquationModels, BlackBoxOptim, Optimisers
4+
5+
SEM = StructuralEquationModels
6+
7+
export SemOptimizerBlackBoxOptim
8+
9+
include("AdamMutation.jl")
10+
include("DiffEvoFactory.jl")
11+
include("SemOptimizerBlackBoxOptim.jl")
12+
13+
end

0 commit comments

Comments
 (0)