Support OnlineStats

Project.toml

@@ -12,6 +12,7 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 InitialValues = "22cec73e-a1b8-11e9-2c92-598750a2cf9c"
 MicroCollections = "128add7d-3638-4c79-886c-908ea0c25c34"
 Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e"
+Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Transducers = "28d57a85-8fef-5791-bfe6-a80928e7c999"
 

README.md

@@ -20,6 +20,8 @@ julia> Folds.sum(1:10, DistributedEx())
 55
 ```
 
+### Iterator transforms and transducers
+
 Most of the functions can be used with iterator comprehensions:
 
 ```julia
@@ -36,6 +38,39 @@ julia> 1:10 |> Filter(isodd) |> MapCat(x -> 1:x^2) |> Folds.sum
 4917
 ```
 
+### Package interop
+
+Folds.jl interoperates with various packages. For example,
+[StructArrays.jl](https://github.com/JuliaArrays/StructArrays.jl) can be used as
+an input and/or output:
+
+```julia
+julia> using StructArrays
+
+julia> table = StructVector(
+           x = [:a, :a, :b, :a, :b],
+           y = [1, 2, 3, 4, 5],
+       );
+
+julia> Folds.copy(StructVector, (row for row in table if row.x === :a))
+3-element StructArray(::Array{Symbol,1}, ::Array{Int64,1}) with eltype NamedTuple{(:x, :y),Tuple{Symbol,Int64}}:
+ (x = :a, y = 1)
+ (x = :a, y = 2)
+ (x = :a, y = 4)
+```
+
+It also works with [OnlineStats.jl](https://github.com/joshday/OnlineStats.jl)
+by treating it as a reducing function (or more precisely a monoid):
+
+```julia
+julia> using OnlineStats
+
+julia> Folds.reduce(Mean(), 1:10)
+Mean: n=10 | value=5.5
+```
+
+### Extensible execution mechanism
+
 Folds.jl decouples the implementation and the execution mechanism
 ("executor"). Additional executors can be installed from
 [FoldsThreads.jl](https://github.com/JuliaFolds/FoldsThreads.jl),

docs/Manifest.toml

@@ -1,5 +1,10 @@
 # This file is machine-generated - editing it directly is not advised
 
+[[AbstractTrees]]
+git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5"
+uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
+version = "0.3.4"
+
 [[Accessors]]
 deps = ["Compat", "CompositionsBase", "ConstructionBase", "Future", "MacroTools", "Requires", "Test"]
 git-tree-sha1 = "6061581e28cf6fbbb6af983022c2589cbc801007"
@@ -52,6 +57,12 @@ git-tree-sha1 = "25ccd31003243d2ce83e474cf11663dddf48035f"
 uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
 version = "1.4.1"
 
+[[DataStructures]]
+deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
+git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677"
+uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+version = "0.18.9"
+
 [[DataValueInterfaces]]
 git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
 uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
@@ -87,10 +98,10 @@ uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 version = "0.26.1"
 
 [[Folds]]
-deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "InitialValues", "MicroCollections", "Referenceables", "Test", "Transducers"]
+deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "InitialValues", "MicroCollections", "Referenceables", "Requires", "Test", "Transducers"]
 path = ".."
 uuid = "41a02a25-b8f0-4f67-bc48-60067656b558"
-version = "0.2.2-DEV"
+version = "0.2.5-DEV"
 
 [[Future]]
 deps = ["Random"]
@@ -152,9 +163,32 @@ git-tree-sha1 = "e991b6a9d38091c4a0d7cd051fcb57c05f98ac03"
 uuid = "128add7d-3638-4c79-886c-908ea0c25c34"
 version = "0.1.0"
 
+[[Missings]]
+deps = ["DataAPI"]
+git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7"
+uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
+version = "1.0.0"
+
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[OnlineStats]]
+deps = ["AbstractTrees", "Dates", "LinearAlgebra", "OnlineStatsBase", "OrderedCollections", "Random", "RecipesBase", "Statistics", "StatsBase", "SweepOperator"]
+git-tree-sha1 = "042eb860cc994aa73b517f326922b03827bbc159"
+uuid = "a15396b6-48d5-5d58-9928-6d29437db91e"
+version = "1.5.9"
+
+[[OnlineStatsBase]]
+deps = ["AbstractTrees", "Dates", "LinearAlgebra", "OrderedCollections", "Statistics", "StatsBase"]
+git-tree-sha1 = "ed076aedbd3cb85731730268d60a7acfa4ac92f3"
+uuid = "925886fa-5bf2-5e8e-b522-a9147a512338"
+version = "1.4.4"
+
+[[OrderedCollections]]
+git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.4.1"
+
 [[Parsers]]
 deps = ["Dates"]
 git-tree-sha1 = "50c9a9ed8c714945e01cd53a21007ed3865ed714"
@@ -177,6 +211,11 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 deps = ["Serialization"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
+[[RecipesBase]]
+git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae"
+uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
+version = "1.1.1"
+
 [[Referenceables]]
 deps = ["Adapt"]
 git-tree-sha1 = "e681d3bfa49cd46c3c161505caddf20f0e62aaa9"
@@ -208,6 +247,12 @@ uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
 [[Sockets]]
 uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 
+[[SortingAlgorithms]]
+deps = ["DataStructures"]
+git-tree-sha1 = "2ec1962eba973f383239da22e75218565c390a96"
+uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
+version = "1.0.0"
+
 [[SparseArrays]]
 deps = ["LinearAlgebra", "Random"]
 uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
@@ -222,12 +267,29 @@ version = "0.1.12"
 deps = ["LinearAlgebra", "SparseArrays"]
 uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
+[[StatsAPI]]
+git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
+uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
+version = "1.0.0"
+
+[[StatsBase]]
+deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "2f6792d523d7448bbe2fec99eca9218f06cc746d"
+uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+version = "0.33.8"
+
 [[StructArrays]]
 deps = ["Adapt", "DataAPI", "Tables"]
 git-tree-sha1 = "8099ed9fb90b6e754d6ba8c6ed8670f010eadca0"
 uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
 version = "0.4.4"
 
+[[SweepOperator]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "20da2784e79fc0bfdd70592d1b47d7a6034e82d1"
+uuid = "7522ee7d-7047-56d0-94d9-4bc626e7058d"
+version = "0.3.0"
+
 [[TableTraits]]
 deps = ["IteratorInterfaceExtensions"]
 git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e"

docs/Project.toml

@@ -1,5 +1,6 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Folds = "41a02a25-b8f0-4f67-bc48-60067656b558"
+OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e"
 StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
 Transducers = "28d57a85-8fef-5791-bfe6-a80928e7c999"

src/Folds.jl

@@ -51,10 +51,11 @@ using BangBang:
     setdiff!!
 using Base: HasShape, IteratorSize, add_sum, mapreduce_empty, mul_prod, reduce_empty
 using Baselet: Baselet
-using MicroCollections: emptyshim, singletonshim
 using DefineSingletons: @def_singleton
 using InitialValues: @def_monoid, SpecificInitialValue, asmonoid
+using MicroCollections: emptyshim, singletonshim
 using Referenceables: isreferenceable, referenceable
+using Requires: @require
 using Transducers:
     Consecutive,
     DefaultInit,
@@ -75,6 +76,7 @@ using Transducers:
     executor_for,
     extract_transducer,
     maybe_set_simd,
+    reducingfunction,
     right,
     tcollect,
     transduce,
@@ -88,6 +90,13 @@ include("utils.jl")
 include("reduce.jl")
 include("collect.jl")
 include("misc.jl")
+
+function __init__()
+    @require OnlineStatsBase="925886fa-5bf2-5e8e-b522-a9147a512338" begin
+        include("interop/onlinestats.jl")
+    end
+end
+
 end
 
 module Testing

src/interop/onlinestats.jl

@@ -0,0 +1,16 @@
+function Folds.reduce(stat::OnlineStatsBase.OnlineStat, itr, ex::Executor; init = FoldsInit)
+    ex isa SequentialEx || validate_reduce_ostat(stat)
+    return Folds.reduce(reducingfunction(stat), itr, ex; init = init)
+end
+
+const OSNonZeroNObsError = ArgumentError(
+    "An `OnlineStat` with one or more observations cannot be used with " *
+    "non-`SequentialEx` executor.",
+)
+
+function validate_reduce_ostat(stat)
+    if OnlineStatsBase.nobs(stat) != 0
+        throw(OSNonZeroNObsError)
+    end
+    return stat
+end

test/Project.toml

@@ -7,6 +7,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Folds = "41a02a25-b8f0-4f67-bc48-60067656b558"
 InitialValues = "22cec73e-a1b8-11e9-2c92-598750a2cf9c"
 LoadAllPackages = "b37bcd2d-1570-475d-a8c6-9b4fae6d0ba9"
+OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e"
 PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe"
 Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e"
 StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"

test/environments/main/Manifest.toml

@@ -1,5 +1,10 @@
 # This file is machine-generated - editing it directly is not advised
 
+[[AbstractTrees]]
+git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5"
+uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
+version = "0.3.4"
+
 [[Accessors]]
 deps = ["Compat", "CompositionsBase", "ConstructionBase", "Future", "MacroTools", "Requires", "Test"]
 git-tree-sha1 = "6061581e28cf6fbbb6af983022c2589cbc801007"
@@ -118,10 +123,10 @@ uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 version = "0.24.11"
 
 [[Folds]]
-deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "InitialValues", "MicroCollections", "Referenceables", "Test", "Transducers"]
+deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "InitialValues", "MicroCollections", "Referenceables", "Requires", "Test", "Transducers"]
 path = "../../.."
 uuid = "41a02a25-b8f0-4f67-bc48-60067656b558"
-version = "0.2.4-DEV"
+version = "0.2.5-DEV"
 
 [[Formatting]]
 deps = ["Printf"]
@@ -204,6 +209,18 @@ version = "0.4.4"
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[OnlineStats]]
+deps = ["AbstractTrees", "Dates", "LinearAlgebra", "OnlineStatsBase", "OrderedCollections", "Random", "RecipesBase", "Statistics", "StatsBase", "SweepOperator"]
+git-tree-sha1 = "042eb860cc994aa73b517f326922b03827bbc159"
+uuid = "a15396b6-48d5-5d58-9928-6d29437db91e"
+version = "1.5.9"
+
+[[OnlineStatsBase]]
+deps = ["AbstractTrees", "Dates", "LinearAlgebra", "OrderedCollections", "Statistics", "StatsBase"]
+git-tree-sha1 = "ed076aedbd3cb85731730268d60a7acfa4ac92f3"
+uuid = "925886fa-5bf2-5e8e-b522-a9147a512338"
+version = "1.4.4"
+
 [[OrderedCollections]]
 git-tree-sha1 = "cf59cfed2e2c12e8a2ff0a4f1e9b2cd8650da6db"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
@@ -249,6 +266,11 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 deps = ["Serialization"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
+[[RecipesBase]]
+git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae"
+uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
+version = "1.1.1"
+
 [[Reexport]]
 deps = ["Pkg"]
 git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0"
@@ -306,6 +328,17 @@ version = "0.1.11"
 deps = ["LinearAlgebra", "SparseArrays"]
 uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
+[[StatsAPI]]
+git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
+uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
+version = "1.0.0"
+
+[[StatsBase]]
+deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "2f6792d523d7448bbe2fec99eca9218f06cc746d"
+uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+version = "0.33.8"
+
 [[StructArrays]]
 deps = ["Adapt", "DataAPI", "Tables"]
 git-tree-sha1 = "8099ed9fb90b6e754d6ba8c6ed8670f010eadca0"
@@ -318,6 +351,12 @@ git-tree-sha1 = "d94235fcdc4a09649f263365c5f7e4ed4ba6ed34"
 uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 version = "1.2.1"
 
+[[SweepOperator]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "20da2784e79fc0bfdd70592d1b47d7a6034e82d1"
+uuid = "7522ee7d-7047-56d0-94d9-4bc626e7058d"
+version = "0.3.0"
+
 [[TableTraits]]
 deps = ["IteratorInterfaceExtensions"]
 git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e"

test/environments/main/Project.toml

@@ -7,6 +7,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Folds = "41a02a25-b8f0-4f67-bc48-60067656b558"
 InitialValues = "22cec73e-a1b8-11e9-2c92-598750a2cf9c"
 LoadAllPackages = "b37bcd2d-1570-475d-a8c6-9b4fae6d0ba9"
+OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e"
 PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe"
 Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e"
 StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"

test/test_with_sequential.jl

@@ -1,14 +1,31 @@
 module TestWithSequential
 
 using Transducers: DistributedEx, SequentialEx, ThreadedEx
-using Folds.Testing: test_with_sequential
+using Folds.Testing: test_with_sequential, eval_call_data
 
-test_with_sequential([
+executors = [
     SequentialEx(),
     ThreadedEx(),
     ThreadedEx(basesize = 1),
     DistributedEx(),
     DistributedEx(basesize = 1),
-])
+]
+
+test_with_sequential(executors)
+
+
+raw_extra_testdata = """
+reduce(Mean(), 1:10)
+reduce(Mean(), (x for x in 1:10 if isodd(x)))
+reduce(KHist(25), 1.0:10.0)
+"""
+
+preamble = quote
+    using OnlineStats
+end
+
+extra_testdata = eval_call_data(raw_extra_testdata, preamble)
+
+test_with_sequential(extra_testdata, executors)
 
 end