Multithreaded support for apply_forest

src/classification/main.jl

@@ -268,11 +268,21 @@ function apply_forest(forest::Ensemble{S, T}, features::AbstractVector{S}) where
     end
 end
 
-function apply_forest(forest::Ensemble{S, T}, features::AbstractMatrix{S}) where {S, T}
+function apply_forest(
+        forest::Ensemble{S, T},
+        features::AbstractMatrix{S};
+        use_multithreading = false
+    ) where {S, T}
     N = size(features,1)
     predictions = Array{T}(undef, N)
-    for i in 1:N
-        predictions[i] = apply_forest(forest, features[i, :])
+    if use_multithreading
+        Threads.@threads for i in 1:N
+            predictions[i] = apply_forest(forest, @view(features[i, :]))
+        end
+    else
+        for i in 1:N
+            predictions[i] = apply_forest(forest, @view(features[i, :]))
+        end
     end
     return predictions
 end

test/classification/adult.jl

@@ -20,6 +20,10 @@ preds = apply_forest(model, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9
 
+preds_MT = apply_forest(model, features, use_multithreading = true)
+cm_MT = confusion_matrix(labels, preds_MT)
+@test cm_MT.accuracy > 0.9
+
 n_iterations = 15
 model, coeffs = build_adaboost_stumps(labels, features, n_iterations);
 preds = apply_adaboost_stumps(model, coeffs, features);

test/classification/digits.jl

@@ -74,6 +74,10 @@ preds = apply_forest(model, X)
 cm = confusion_matrix(Y, preds)
 @test cm.accuracy > 0.95
 
+preds_MT = apply_forest(model, X, use_multithreading = true)
+cm_MT = confusion_matrix(Y, preds_MT)
+@test cm_MT.accuracy > 0.95
+
 n_iterations        = 100
 model, coeffs = DecisionTree.build_adaboost_stumps(
         Y, X,

test/classification/heterogeneous.jl

@@ -26,6 +26,10 @@ preds = apply_forest(model, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9
 
+preds_MT = apply_forest(model, features, use_multithreading = true)
+cm_MT = confusion_matrix(labels, preds_MT)
+@test cm_MT.accuracy > 0.9
+
 n_subfeatures = 7
 model, coeffs = build_adaboost_stumps(labels, features, n_subfeatures)
 preds = apply_adaboost_stumps(model, coeffs, features)

test/classification/iris.jl

@@ -74,6 +74,12 @@ cm = confusion_matrix(labels, preds)
 probs = apply_forest_proba(model, features, classes)
 @test reshape(sum(probs, dims=2), n) ≈ ones(n)
 
+preds_MT = apply_forest(model, features, use_multithreading = true)
+cm_MT = confusion_matrix(labels, preds_MT)
+@test cm_MT.accuracy > 0.95
+@test typeof(preds_MT) == Vector{String}
+@test sum(preds .!= preds_MT) == 0
+
 # run n-fold cross validation for forests
 println("\n##### nfoldCV Classification Forest #####")
 n_subfeatures = 2

test/classification/low_precision.jl

@@ -46,6 +46,11 @@ cm = confusion_matrix(labels, preds)
 @test typeof(preds) == Vector{Int32}
 @test cm.accuracy > 0.9
 
+preds_MT = apply_forest(model, features, use_multithreading = true)
+cm_MT = confusion_matrix(labels, preds_MT)
+@test typeof(preds_MT) == Vector{Int32}
+@test cm_MT.accuracy > 0.9
+
 n_iterations        = Int32(25)
 model, coeffs = build_adaboost_stumps(labels, features, n_iterations);
 preds = apply_adaboost_stumps(model, coeffs, features);
@@ -112,6 +117,10 @@ model = build_forest(labels, features)
 preds = apply_forest(model, features)
 @test typeof(preds) == Vector{Int8}
 
+preds_MT = apply_forest(model, features, use_multithreading = true)
+@test typeof(preds_MT) == Vector{Int8}
+@test sum(abs.(preds .- preds_MT)) == zero(Int8)
+
 model = build_tree(labels, features)
 preds = apply_tree(model, features)
 @test typeof(preds) == Vector{Int8}

test/classification/random.jl

@@ -43,6 +43,12 @@ cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9
 @test typeof(preds) == Vector{Int}
 
+preds_MT = apply_forest(model, features, use_multithreading = true)
+cm_MT = confusion_matrix(labels, preds_MT)
+@test cm_MT.accuracy > 0.9
+@test typeof(preds_MT) == Vector{Int}
+@test sum(abs.(preds .- preds_MT)) == zero(Int)
+
 n_subfeatures       = 3
 n_trees             = 9
 partial_sampling    = 0.7
@@ -64,6 +70,10 @@ cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9
 @test length(model) == n_trees
 
+preds_MT = apply_forest(model, features, use_multithreading = true)
+cm_MT = confusion_matrix(labels, preds_MT)
+@test cm_MT.accuracy > 0.9
+
 # test n_subfeatures
 n_subfeatures       = 0
 m_partial = build_forest(labels, features) # default sqrt(n_features)

test/regression/digits.jl

@@ -79,6 +79,10 @@ model = build_forest(
 preds = apply_forest(model, X)
 @test R2(Y, preds) > 0.8
 
+preds_MT = apply_forest(model, X, use_multithreading = true)
+@test R2(Y, preds_MT) > 0.8
+@test sum(abs.(preds .- preds_MT)) < 1e-8
+
 println("\n##### 3 foldCV Regression Tree #####")
 n_folds = 5
 r2 = nfoldCV_tree(Y, X, n_folds; verbose=false);

test/regression/low_precision.jl

@@ -45,6 +45,11 @@ preds = apply_forest(model, features)
 @test R2(labels, preds) > 0.9
 @test typeof(preds) <: Vector{Float64}
 
+preds_MT = apply_forest(model, features, use_multithreading=true)
+@test R2(labels, preds_MT) > 0.9
+@test typeof(preds_MT) <: Vector{Float64}
+@test sum(abs.(preds .- preds_MT)) < 1.0e-8
+
 println("\n##### nfoldCV Regression Tree #####")
 n_folds             = Int32(3)
 pruning_purity      = 1.0
@@ -99,6 +104,10 @@ model = build_forest(labels, features)
 preds = apply_forest(model, features)
 @test typeof(preds) == Vector{Float16}
 
+preds_MT = apply_forest(model, features, use_multithreading = true)
+@test typeof(preds_MT) == Vector{Float16}
+@test sum(abs.(preds .- preds_MT)) < 1.0e-8
+
 model = build_tree(labels, features)
 preds = apply_tree(model, features)
 @test typeof(preds) == Vector{Float16}

test/regression/random.jl

@@ -1,200 +1,38 @@
-@testset "random.jl" begin
-
-Random.seed!(5)
-
-n, m = 10^3, 5 ;
-features = Array{Any}(undef, n, m);
-features[:,:] = randn(n, m);
-features[:,1] = round.(Integer, features[:,1]); # convert a column of integers
-weights = rand(-2:2,m);
-labels = float.(features * weights);            # cast to Array{Float64,1}
-
-model = build_stump(labels, features)
-@test depth(model) == 1
-
-# over-fitting
-min_samples_leaf    = 1
-max_depth           = -1
-n_subfeatures       = 0
-model = build_tree(
-        labels, features,
-        n_subfeatures,
-        max_depth,
-        min_samples_leaf)
-preds = apply_tree(model, features);
-@test R2(labels, preds) > 0.99      # R2: coeff of determination
-@test typeof(preds) <: Vector{Float64}
-### @test length(model) == n        # can / should this be enforced ???
-
-# under-fitting
-min_samples_leaf    = 100
-model = build_tree(
-        labels, round.(Int, features),
-        n_subfeatures,
-        max_depth,
-        min_samples_leaf)
-preds = apply_tree(model, round.(Int, features));
-@test R2(labels, preds) < 0.8
-
-min_samples_leaf    = 5
-max_depth           = 3
-n_subfeatures       = 0
-model = build_tree(
-        labels, features,
-        n_subfeatures,
-        max_depth,
-        min_samples_leaf)
+@testset "scikitlearn.jl" begin
+
+n, m = 10^3, 5;
+features = rand(StableRNG(1), n, m);
+weights = rand(StableRNG(1), -1:1, m);
+labels = features * weights;
+
+model = fit!(DecisionTreeRegressor(; rng=StableRNG(1), min_samples_split=5), features, labels)
+@test R2(labels, predict(model, features)) > 0.8
+@test impurity_importance(model) == impurity_importance(model.root)
+@test isapprox(permutation_importance(model, features, labels, rng=StableRNG(1)).mean, permutation_importance(model.root, labels, features, (model, y, X) -> R2(y, apply_tree(model, X)), rng=StableRNG(1)).mean)
+
+let
+    regressor = RandomForestRegressor(; rng=StableRNG(1), n_trees=10, min_samples_leaf=5, n_subfeatures=2)
+    model = fit!(regressor, features, labels)
+    @test R2(labels, predict(model, features)) > 0.8
+    @test impurity_importance(model) == impurity_importance(model.ensemble)
+    @test split_importance(model) == split_importance(model.ensemble)
+    @test isapprox(permutation_importance(model, features, labels, rng=StableRNG(1)).mean, permutation_importance(model.ensemble, labels, features, (model, y, X) -> R2(y, apply_forest(model, X)), rng=StableRNG(1)).mean)
+end
+
+Random.seed!(2)
+N = 3000
+X = randn(StableRNG(1), N, 10)
+y = randn(StableRNG(1), N)
+max_depth = 5
+model = fit!(DecisionTreeRegressor(; rng=StableRNG(1), max_depth=max_depth), X, y)
 @test depth(model) == max_depth
 
-min_samples_leaf    = 1
-n_subfeatures       = 0
-max_depth           = -1
-min_samples_split   = 300
-model = build_tree(
-        labels, features,
-        n_subfeatures,
-        max_depth,
-        min_samples_leaf,
-        min_samples_split)
-preds = apply_tree(model, features);
-@test R2(labels, preds) < 0.8
-
-n_subfeatures       = 0
-max_depth           = -1
-min_samples_leaf    = 1
-min_samples_split   = 2
-min_purity_increase = 0.5
-model = build_tree(
-        labels, features,
-        n_subfeatures,
-        max_depth,
-        min_samples_leaf,
-        min_samples_split,
-        min_purity_increase)
-preds = apply_tree(model, features);
-@test R2(labels, preds) < 0.95
-
-# test RNG param of trees
-n_subfeatures       = 2
-t1 = build_tree(labels, features, n_subfeatures; rng=10)
-t2 = build_tree(labels, features, n_subfeatures; rng=10)
-t3 = build_tree(labels, features, n_subfeatures; rng=5)
-@test (length(t1) == length(t2)) && (depth(t1) == depth(t2))
-@test (length(t1) != length(t3)) || (depth(t1) != depth(t3))
-
-mt = Random.MersenneTwister(1)
-t1 = build_tree(labels, features, n_subfeatures; rng=mt)
-t3 = build_tree(labels, features, n_subfeatures; rng=mt)
-@test (length(t1) != length(t3)) || (depth(t1) != depth(t3))
-
-
-model = build_forest(labels, features)
-preds = apply_forest(model, features)
-@test R2(labels, preds) > 0.9
-@test typeof(preds) <: Vector{Float64}
-
-n_subfeatures       = 3
-n_trees             = 9
-partial_sampling    = 0.7
-max_depth           = -1
-min_samples_leaf    = 5
-min_samples_split   = 2
-min_purity_increase = 0.0
-model = build_forest(
-        labels, features,
-        n_subfeatures,
-        n_trees,
-        partial_sampling,
-        max_depth,
-        min_samples_leaf,
-        min_samples_split,
-        min_purity_increase)
-preds = apply_forest(model, features)
-@test R2(labels, preds) > 0.9
-@test length(model) == n_trees
-
-# test n_subfeatures
-n_trees             = 10
-partial_sampling    = 1.0
-max_depth           = -1
-min_samples_leaf    = 10
-n_subfeatures       = 1
-m_partial = build_forest(
-        labels, features,
-        n_subfeatures,
-        n_trees,
-        partial_sampling,
-        max_depth,
-        min_samples_leaf)
-n_subfeatures       = 0
-m_full = build_forest(
-        labels, features,
-        n_subfeatures,
-        n_trees,
-        partial_sampling,
-        max_depth,
-        min_samples_leaf)
-@test mean(depth.(m_full.trees)) < mean(depth.(m_partial.trees))
-
-# test partial_sampling parameter, train on single sample
-partial_sampling    = 1 / n
-n_subfeatures       = 0
-n_trees             = 1         # single tree test
-max_depth           = -1
-min_samples_leaf    = 1
-min_samples_split   = 2
-min_purity_increase = 0.0
-partial = build_forest(
-            labels, features,
-            n_subfeatures,
-            n_trees,
-            partial_sampling,
-            max_depth,
-            min_samples_leaf,
-            min_samples_split,
-            min_purity_increase)
-@test typeof(partial.trees[1]) <: Leaf
-
-# test RNG parameter
-n_subfeatures       = 2
-n_trees             = 5
-m1 = build_forest(labels, features,
-        n_subfeatures,
-        n_trees;
-        rng=10)
-m2 = build_forest(labels, features,
-        n_subfeatures,
-        n_trees;
-        rng=10)
-m3 = build_forest(labels, features,
-        n_subfeatures,
-        n_trees;
-        rng=5)
-@test length.(m1.trees) == length.(m2.trees)
-@test depth.(m1.trees)  == depth.(m2.trees)
-@test length.(m1.trees) != length.(m3.trees)
-
-
-println("\n##### nfoldCV Classification Tree #####")
-nfolds          = 3
-pruning_purity  = 1.0
-max_depth       = 4
-r2_1 = nfoldCV_tree(labels, features, nfolds, pruning_purity, max_depth; rng=10, verbose=false)
-r2_2 = nfoldCV_tree(labels, features, nfolds, pruning_purity, max_depth; rng=10)
-r2_3 = nfoldCV_tree(labels, features, nfolds, pruning_purity, max_depth; rng=5)
-@test mean(r2_1) > 0.5
-@test r2_1 == r2_2
-@test r2_1 != r2_3
-
-println("\n##### nfoldCV Regression Forest #####")
-nfolds          = 3
-n_subfeatures   = 2
-n_trees         = 10
-r2_1  = nfoldCV_forest(labels, features, nfolds, n_subfeatures, n_trees; rng=10, verbose=false)
-r2_2 = nfoldCV_forest(labels, features, nfolds, n_subfeatures, n_trees; rng=10)
-r2_3 = nfoldCV_forest(labels, features, nfolds, n_subfeatures, n_trees; rng=5)
-@test mean(r2_1) > 0.8
-@test r2_1 == r2_2
-@test r2_1 != r2_3
+## Test that the RNG arguments work as expected
+X = randn(StableRNG(1), 100, 10)
+y = randn(StableRNG(1), 100)
+@test fit_predict!(RandomForestRegressor(; rng=10), X, y) ==
+    fit_predict!(RandomForestRegressor(; rng=10), X, y)
+@test fit_predict!(RandomForestRegressor(; rng=10), X, y) !=
+    fit_predict!(RandomForestRegressor(; rng=22), X, y)
 
 end # @testset