editing SVM analysis example

docs/src/sensitivity-analysis-svm.md

@@ -21,84 +21,55 @@ where
 
 Import the libraries.
 
-```@example 1
+```julia
 import Random
-using Test
 import SCS
 import Plots
 using DiffOpt
+using JuMP
 using LinearAlgebra
-import MathOptInterface
-const MOI = MathOptInterface
-nothing # hide
 ```
 
 Construct separatable, non-trivial data points.
-```@example 1
-N = 100
+```julia
+N = 50
 D = 2
-Random.seed!(6)
+Random.seed!(rand(1:100))
 X = vcat(randn(N, D), randn(N,D) .+ [4.0,1.5]')
 y = append!(ones(N), -ones(N))
-nothing # hide
+N = 2*N;
 ```
 
 Let's define the variables.
 ```@example 1
-model = diff_optimizer(SCS.Optimizer)
+model = Model(() -> diff_optimizer(SCS.Optimizer))
 MOI.set(model, MOI.Silent(), true)
 
 # add variables
-l = MOI.add_variables(model, N)
-w = MOI.add_variables(model, D)
-b = MOI.add_variable(model)
-nothing # hide
+@variable(model, l[1:N])
+@variable(model, w[1:D])
+@variable(model, b);
 ```
 
 Add the constraints.
 ```@example 1
-MOI.add_constraint(
-    model,
-    MOI.VectorAffineFunction(
-        MOI.VectorAffineTerm.(1:N, MOI.ScalarAffineTerm.(1.0, l)), zeros(N),
-    ),
-    MOI.Nonnegatives(N),
-)
-
-# define the whole matrix Ax, it'll be easier then
-# refer https://discourse.julialang.org/t/solve-minimization-problem-where-constraint-is-the-system-of-linear-inequation-with-mathoptinterface-efficiently/23571/4
-Ax = Matrix{MOI.ScalarAffineTerm{Float64}}(undef, N, D+2)
-for i in 1:N
-    Ax[i, :] = MOI.ScalarAffineTerm.([1.0; y[i]*X[i,:]; y[i]], [l[i]; w; b])
-end
-terms = MOI.VectorAffineTerm.(1:N, Ax)
-f = MOI.VectorAffineFunction(
-    vec(terms),
-    -ones(N),
-)
-cons = MOI.add_constraint(
-    model,
-    f,
-    MOI.Nonnegatives(N),
-)
-nothing # hide
+@constraint(model, cons, y.*(X*w .+ b) + l.-1 ∈ MOI.Nonnegatives(N))
+@constraint(model, 1.0*l ∈ MOI.Nonnegatives(N));
 ```
 
 Define the linear objective function and solve the SVM model.
 ```@example 1
-objective_function = MOI.ScalarAffineFunction(
-                        MOI.ScalarAffineTerm.(ones(N), l),
-                        0.0,
-                    )
-MOI.set(model, MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}(), objective_function)
-MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE)
-
-MOI.optimize!(model)
-
-loss = MOI.get(model, MOI.ObjectiveValue())
-wv = MOI.get(model, MOI.VariablePrimal(), w)
-bv = MOI.get(model, MOI.VariablePrimal(), b)
-nothing # hide
+@objective(
+    model,
+    Min,
+    sum(l),
+)
+
+optimize!(model)
+
+loss = objective_value(model)
+wv = value.(w)
+bv = value(b);
 ```
 
 We can visualize the separating hyperplane.
@@ -111,12 +82,9 @@ svm_y = (-bv .- wv[1] * svm_x )/wv[2]
 p = Plots.scatter(X[:,1], X[:,2], color = [yi > 0 ? :red : :blue for yi in y], label = "")
 Plots.yaxis!(p, (-2, 4.5))
 Plots.plot!(p, svm_x, svm_y, label = "loss = $(round(loss, digits=2))", width=3)
-Plots.savefig("svm_separating.svg")
-nothing # hide
+p
 ```
 
-![svg](svm_separating.svg)
-
 # Experiments
 Now that we've solved the SVM, we can compute the sensitivity of optimal values -- the separating hyperplane in our case -- with respect to perturbations of the problem data -- the data points -- using DiffOpt. For illustration, we've explored two questions:
 
@@ -194,7 +162,6 @@ for Xi in 1:N
     dy[Xi] = 0.0  # reset the change made above
 end
 LinearAlgebra.normalize!(∇)
-nothing # hide
 ```
 
 Visualize point sensitivities with respect to separating hyperplane. Note that the gradients are normalized.
@@ -206,17 +173,13 @@ p2 = Plots.scatter(
 )
 Plots.yaxis!(p2, (-2, 4.5))
 Plots.plot!(p2, svm_x, svm_y, label = "loss = $(round(loss, digits=2))", width=3)
-Plots.savefig("sensitivity2.svg")
-nothing # hide
+p2
 ```
 
-![](sensitivity2.svg)
-
-
 ## Experiment 2: Gradient of hyperplane wrt the data point coordinates
 
 Similar to previous example, construct perturbations in data points coordinates `X`.
-```julia
+```@example 1
 ∇ = Float64[]
 dX = zeros(N, D)
 
@@ -261,8 +224,5 @@ p3 = Plots.scatter(
 )
 Plots.yaxis!(p3, (-2, 4.5))
 Plots.plot!(p3, svm_x, svm_y, label = "loss = $(round(loss, digits=2))", width=3)
-Plots.savefig(p3, "sensitivity3.svg")
-nothing # hide
+p3
 ```
-
-![](sensitivity3.svg)

examples/sensitivity-SVM.jl

@@ -3,7 +3,6 @@
 """
 
 import Random
-using Test
 import SCS
 using DiffOpt
 using LinearAlgebra
@@ -22,7 +21,8 @@ N = 50
 D = 2
 Random.seed!(rand(1:100))
 X = vcat(randn(N, D), randn(N,D) .+ [4.0,1.5]')
-y = append!(ones(N), -ones(N));
+y = append!(ones(N), -ones(N))
+N = 2*N;
 
 model = diff_optimizer(SCS.Optimizer)