diff --git a/Project.toml b/Project.toml index 9b7738b..1619490 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,7 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" InitialValues = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" MicroCollections = "128add7d-3638-4c79-886c-908ea0c25c34" Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" +Requires = "ae029012-a4dd-5104-9daa-d747884805df" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Transducers = "28d57a85-8fef-5791-bfe6-a80928e7c999" diff --git a/README.md b/README.md index 9c80ead..dbb2fa3 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ julia> Folds.sum(1:10, DistributedEx()) 55 ``` +### Iterator transforms and transducers + Most of the functions can be used with iterator comprehensions: ```julia @@ -36,6 +38,39 @@ julia> 1:10 |> Filter(isodd) |> MapCat(x -> 1:x^2) |> Folds.sum 4917 ``` +### Package interop + +Folds.jl interoperates with various packages. For example, +[StructArrays.jl](https://github.com/JuliaArrays/StructArrays.jl) can be used as +an input and/or output: + +```julia +julia> using StructArrays + +julia> table = StructVector( + x = [:a, :a, :b, :a, :b], + y = [1, 2, 3, 4, 5], + ); + +julia> Folds.copy(StructVector, (row for row in table if row.x === :a)) +3-element StructArray(::Array{Symbol,1}, ::Array{Int64,1}) with eltype NamedTuple{(:x, :y),Tuple{Symbol,Int64}}: + (x = :a, y = 1) + (x = :a, y = 2) + (x = :a, y = 4) +``` + +It also works with [OnlineStats.jl](https://github.com/joshday/OnlineStats.jl) +by treating it as a reducing function (or more precisely a monoid): + +```julia +julia> using OnlineStats + +julia> Folds.reduce(Mean(), 1:10) +Mean: n=10 | value=5.5 +``` + +### Extensible execution mechanism + Folds.jl decouples the implementation and the execution mechanism ("executor"). Additional executors can be installed from [FoldsThreads.jl](https://github.com/JuliaFolds/FoldsThreads.jl), diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 25b2f3b..bafc83e 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,5 +1,10 @@ # This file is machine-generated - editing it directly is not advised +[[AbstractTrees]] +git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.3.4" + [[Accessors]] deps = ["Compat", "CompositionsBase", "ConstructionBase", "Future", "MacroTools", "Requires", "Test"] git-tree-sha1 = "6061581e28cf6fbbb6af983022c2589cbc801007" @@ -52,6 +57,12 @@ git-tree-sha1 = "25ccd31003243d2ce83e474cf11663dddf48035f" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" version = "1.4.1" +[[DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.9" + [[DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" @@ -87,10 +98,10 @@ uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" version = "0.26.1" [[Folds]] -deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "InitialValues", "MicroCollections", "Referenceables", "Test", "Transducers"] +deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "InitialValues", "MicroCollections", "Referenceables", "Requires", "Test", "Transducers"] path = ".." uuid = "41a02a25-b8f0-4f67-bc48-60067656b558" -version = "0.2.2-DEV" +version = "0.2.5-DEV" [[Future]] deps = ["Random"] @@ -152,9 +163,32 @@ git-tree-sha1 = "e991b6a9d38091c4a0d7cd051fcb57c05f98ac03" uuid = "128add7d-3638-4c79-886c-908ea0c25c34" version = "0.1.0" +[[Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.0.0" + [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +[[OnlineStats]] +deps = ["AbstractTrees", "Dates", "LinearAlgebra", "OnlineStatsBase", "OrderedCollections", "Random", "RecipesBase", "Statistics", "StatsBase", "SweepOperator"] +git-tree-sha1 = "042eb860cc994aa73b517f326922b03827bbc159" +uuid = "a15396b6-48d5-5d58-9928-6d29437db91e" +version = "1.5.9" + +[[OnlineStatsBase]] +deps = ["AbstractTrees", "Dates", "LinearAlgebra", "OrderedCollections", "Statistics", "StatsBase"] +git-tree-sha1 = "ed076aedbd3cb85731730268d60a7acfa4ac92f3" +uuid = "925886fa-5bf2-5e8e-b522-a9147a512338" +version = "1.4.4" + +[[OrderedCollections]] +git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.4.1" + [[Parsers]] deps = ["Dates"] git-tree-sha1 = "50c9a9ed8c714945e01cd53a21007ed3865ed714" @@ -177,6 +211,11 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +[[RecipesBase]] +git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.1.1" + [[Referenceables]] deps = ["Adapt"] git-tree-sha1 = "e681d3bfa49cd46c3c161505caddf20f0e62aaa9" @@ -208,6 +247,12 @@ uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +[[SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "2ec1962eba973f383239da22e75218565c390a96" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.0.0" + [[SparseArrays]] deps = ["LinearAlgebra", "Random"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" @@ -222,12 +267,29 @@ version = "0.1.12" deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +[[StatsAPI]] +git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.0.0" + +[[StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "2f6792d523d7448bbe2fec99eca9218f06cc746d" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.33.8" + [[StructArrays]] deps = ["Adapt", "DataAPI", "Tables"] git-tree-sha1 = "8099ed9fb90b6e754d6ba8c6ed8670f010eadca0" uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" version = "0.4.4" +[[SweepOperator]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "20da2784e79fc0bfdd70592d1b47d7a6034e82d1" +uuid = "7522ee7d-7047-56d0-94d9-4bc626e7058d" +version = "0.3.0" + [[TableTraits]] deps = ["IteratorInterfaceExtensions"] git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e" diff --git a/docs/Project.toml b/docs/Project.toml index f09e361..035ffda 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,5 +1,6 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Folds = "41a02a25-b8f0-4f67-bc48-60067656b558" +OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" Transducers = "28d57a85-8fef-5791-bfe6-a80928e7c999" diff --git a/src/Folds.jl b/src/Folds.jl index 252ee0b..32d6137 100644 --- a/src/Folds.jl +++ b/src/Folds.jl @@ -51,10 +51,11 @@ using BangBang: setdiff!! using Base: HasShape, IteratorSize, add_sum, mapreduce_empty, mul_prod, reduce_empty using Baselet: Baselet -using MicroCollections: emptyshim, singletonshim using DefineSingletons: @def_singleton using InitialValues: @def_monoid, SpecificInitialValue, asmonoid +using MicroCollections: emptyshim, singletonshim using Referenceables: isreferenceable, referenceable +using Requires: @require using Transducers: Consecutive, DefaultInit, @@ -75,6 +76,7 @@ using Transducers: executor_for, extract_transducer, maybe_set_simd, + reducingfunction, right, tcollect, transduce, @@ -88,6 +90,13 @@ include("utils.jl") include("reduce.jl") include("collect.jl") include("misc.jl") + +function __init__() + @require OnlineStatsBase="925886fa-5bf2-5e8e-b522-a9147a512338" begin + include("interop/onlinestats.jl") + end +end + end module Testing diff --git a/src/interop/onlinestats.jl b/src/interop/onlinestats.jl new file mode 100644 index 0000000..6022bf1 --- /dev/null +++ b/src/interop/onlinestats.jl @@ -0,0 +1,16 @@ +function Folds.reduce(stat::OnlineStatsBase.OnlineStat, itr, ex::Executor; init = FoldsInit) + ex isa SequentialEx || validate_reduce_ostat(stat) + return Folds.reduce(reducingfunction(stat), itr, ex; init = init) +end + +const OSNonZeroNObsError = ArgumentError( + "An `OnlineStat` with one or more observations cannot be used with " * + "non-`SequentialEx` executor.", +) + +function validate_reduce_ostat(stat) + if OnlineStatsBase.nobs(stat) != 0 + throw(OSNonZeroNObsError) + end + return stat +end diff --git a/test/Project.toml b/test/Project.toml index f2f5095..28284cf 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -7,6 +7,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Folds = "41a02a25-b8f0-4f67-bc48-60067656b558" InitialValues = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" LoadAllPackages = "b37bcd2d-1570-475d-a8c6-9b4fae6d0ba9" +OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe" Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" diff --git a/test/environments/main/Manifest.toml b/test/environments/main/Manifest.toml index ae9e14e..04556d9 100644 --- a/test/environments/main/Manifest.toml +++ b/test/environments/main/Manifest.toml @@ -1,5 +1,10 @@ # This file is machine-generated - editing it directly is not advised +[[AbstractTrees]] +git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.3.4" + [[Accessors]] deps = ["Compat", "CompositionsBase", "ConstructionBase", "Future", "MacroTools", "Requires", "Test"] git-tree-sha1 = "6061581e28cf6fbbb6af983022c2589cbc801007" @@ -118,10 +123,10 @@ uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" version = "0.24.11" [[Folds]] -deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "InitialValues", "MicroCollections", "Referenceables", "Test", "Transducers"] +deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "InitialValues", "MicroCollections", "Referenceables", "Requires", "Test", "Transducers"] path = "../../.." uuid = "41a02a25-b8f0-4f67-bc48-60067656b558" -version = "0.2.4-DEV" +version = "0.2.5-DEV" [[Formatting]] deps = ["Printf"] @@ -204,6 +209,18 @@ version = "0.4.4" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +[[OnlineStats]] +deps = ["AbstractTrees", "Dates", "LinearAlgebra", "OnlineStatsBase", "OrderedCollections", "Random", "RecipesBase", "Statistics", "StatsBase", "SweepOperator"] +git-tree-sha1 = "042eb860cc994aa73b517f326922b03827bbc159" +uuid = "a15396b6-48d5-5d58-9928-6d29437db91e" +version = "1.5.9" + +[[OnlineStatsBase]] +deps = ["AbstractTrees", "Dates", "LinearAlgebra", "OrderedCollections", "Statistics", "StatsBase"] +git-tree-sha1 = "ed076aedbd3cb85731730268d60a7acfa4ac92f3" +uuid = "925886fa-5bf2-5e8e-b522-a9147a512338" +version = "1.4.4" + [[OrderedCollections]] git-tree-sha1 = "cf59cfed2e2c12e8a2ff0a4f1e9b2cd8650da6db" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" @@ -249,6 +266,11 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +[[RecipesBase]] +git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.1.1" + [[Reexport]] deps = ["Pkg"] git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0" @@ -306,6 +328,17 @@ version = "0.1.11" deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +[[StatsAPI]] +git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.0.0" + +[[StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "2f6792d523d7448bbe2fec99eca9218f06cc746d" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.33.8" + [[StructArrays]] deps = ["Adapt", "DataAPI", "Tables"] git-tree-sha1 = "8099ed9fb90b6e754d6ba8c6ed8670f010eadca0" @@ -318,6 +351,12 @@ git-tree-sha1 = "d94235fcdc4a09649f263365c5f7e4ed4ba6ed34" uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" version = "1.2.1" +[[SweepOperator]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "20da2784e79fc0bfdd70592d1b47d7a6034e82d1" +uuid = "7522ee7d-7047-56d0-94d9-4bc626e7058d" +version = "0.3.0" + [[TableTraits]] deps = ["IteratorInterfaceExtensions"] git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e" diff --git a/test/environments/main/Project.toml b/test/environments/main/Project.toml index f2f5095..28284cf 100644 --- a/test/environments/main/Project.toml +++ b/test/environments/main/Project.toml @@ -7,6 +7,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Folds = "41a02a25-b8f0-4f67-bc48-60067656b558" InitialValues = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" LoadAllPackages = "b37bcd2d-1570-475d-a8c6-9b4fae6d0ba9" +OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe" Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" diff --git a/test/test_with_sequential.jl b/test/test_with_sequential.jl index 77c63e8..dcf5383 100644 --- a/test/test_with_sequential.jl +++ b/test/test_with_sequential.jl @@ -1,14 +1,31 @@ module TestWithSequential using Transducers: DistributedEx, SequentialEx, ThreadedEx -using Folds.Testing: test_with_sequential +using Folds.Testing: test_with_sequential, eval_call_data -test_with_sequential([ +executors = [ SequentialEx(), ThreadedEx(), ThreadedEx(basesize = 1), DistributedEx(), DistributedEx(basesize = 1), -]) +] + +test_with_sequential(executors) + + +raw_extra_testdata = """ +reduce(Mean(), 1:10) +reduce(Mean(), (x for x in 1:10 if isodd(x))) +reduce(KHist(25), 1.0:10.0) +""" + +preamble = quote + using OnlineStats +end + +extra_testdata = eval_call_data(raw_extra_testdata, preamble) + +test_with_sequential(extra_testdata, executors) end