diff --git a/Project.toml b/Project.toml index a2afd5165..fba22462e 100644 --- a/Project.toml +++ b/Project.toml @@ -19,7 +19,8 @@ JLD = "4138dd39-2aa7-5051-a626-17a0bb65d9c8" MatLang = "05b439c0-bb3c-11e9-1d8d-1f0a9ebca87a" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Pkg", "ColorTypes", "Test", "FixedPointNumbers", "JLD", "SparseArrays", "MatLang"] +test = ["Pkg", "ColorTypes", "Test", "FixedPointNumbers", "JLD", "SparseArrays", "MatLang", "Suppressor"] diff --git a/docs/make.jl b/docs/make.jl index 23a5e6188..e11c70012 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -7,7 +7,7 @@ makedocs( prettyurls = get(ENV, "CI", nothing) == "true" ), modules = [SnoopCompile], - pages = ["index.md", "snoopi.md", "snoopc.md", "userimg.md", "reference.md"] + pages = ["index.md", "snoopi.md", "snoopc.md", "userimg.md", "bot.md", "reference.md"] ) deploydocs( diff --git a/docs/src/bot.md b/docs/src/bot.md new file mode 100644 index 000000000..f9916c2e8 --- /dev/null +++ b/docs/src/bot.md @@ -0,0 +1,158 @@ +# SnoopCompile Bot (EXPERIMENTAL) + +You can use SnoopCompile bot to automatically and continuously create precompile files. + +One should add 3 things to a package to make the bot work: + +---------------------------------- + + +- Workflow file: + +create a workflow file with this path in your repository `.github/workflows/SnoopCompile.yml` and use the following content: + +```yaml +name: SnoopCompile + +on: + - push + + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: ['nightly'] + julia-arch: [x64] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.julia-version }} + - name: Install dependencies + run: julia --project -e 'using Pkg; Pkg.instantiate();' + - name : Add SnoopCompile and current package + run: julia -e 'using Pkg; Pkg.add("SnoopCompile"); Pkg.develop(PackageSpec(; path=pwd()));' + - name: Install Test dependencies + run: julia -e 'using SnoopCompile; SnoopCompile.addtestdep()' + - name: Generating precompile files + run: julia --project=@. -e 'include("deps/SnoopCompile/snoopCompile.jl")' + - name: Running Benchmark + run: julia --project=@. -e 'include("deps/SnoopCompile/snoopBenchmark.jl")' + + # https://github.com/marketplace/actions/create-pull-request + - name: Create Pull Request + uses: peter-evans/create-pull-request@v2.1.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Update precompile_*.jl file + committer: YOUR NAME # Change `committer` to your name and your email. + title: '[AUTO] Update precompile_*.jl file' + labels: SnoopCompile + branch: create-pull-request/SnoopCompile + - name: Check output environment variable + run: echo "Pull Request Number - ${{ env.PULL_REQUEST_NUMBER }}" +``` +`Install Test dependencies` step is only needed if you have test dependencies other than Test. Otherwise, you should comment it. In this case, if your examples or tests have dependencies, you should add a `Test.toml` to your test folder. + +```yaml +- name: Install Test dependencies + run: julia -e 'using SnoopCompile; SnoopCompile.addtestdep()' +``` + +For example for MatLang package: + +[Link](https://github.com/juliamatlab/MatLang/blob/master/.github/workflows/SnoopCompile.yml) + +---------------------------------- + + +- Precompile script + +Add a `snoopCompile.jl` file under `deps/SnoopCompile`. The content of the file should be a script that "exercises" the functionality you'd like to precompile. One option is to use your package's `"runtests.jl"` file, or you can write a custom script for this purpose. + + +For example, some examples that call the functions: + +```julia +using SnoopCompile + +@snoopiBot "MatLang" begin + using MatLang + examplePath = joinpath(dirname(dirname(pathof(MatLang))), "examples") + include(joinpath(examplePath,"Language_Fundamentals", "usage_Entering_Commands.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "usage_Matrices_and_Arrays.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "Data_Types", "usage_Numeric_Types.jl")) +end +``` +[Ref]( https://github.com/juliamatlab/MatLang/blob/master/deps/SnoopCompile/snoopCompile.jl) + +or if you do not have additional examples, you can use your runtests.jl file using this syntax: + +```julia +using SnoopCompile + +# using runtests: +@snoopiBot "MatLang" +``` + +[Also look at this](https://timholy.github.io/SnoopCompile.jl/stable/snoopi/#Precompile-scripts-1) + +---------------------------------- + +- Include precompile signatures + +Two lines of (commented) code that includes the precompile file in your main module. + +It is better to have these lines commented to continuously develop and change your package offline. snoopiBot will find these lines of code and will uncomment them in the created pull request. If they are not commented the bot will leave it as is in the pull request: + +```julia +# include("../deps/SnoopCompile/precompile/precompile_MatLang.jl") +# _precompile_() +``` + +[Ref](https://github.com/juliamatlab/MatLang/blob/072ff8ed9877cbb34f8583ae2cf928a5df18aa0c/src/MatLang.jl#L26) + + +---------------------------------- + + +## Benchmark + +To measure the effect of adding precompile files. Add a `snoopBenchmark.jl`. The content of this file can be the following: + +Benchmarking the load infer time +```julia +println("loading infer benchmark") + +@snoopiBench "MatLang" using MatLang +``` + +Benchmarking the example infer time +```julia +println("examples infer benchmark") + +@snoopiBench "MatLang" begin + using MatLang + examplePath = joinpath(dirname(dirname(pathof(MatLang))), "examples") + # include(joinpath(examplePath,"Language_Fundamentals", "usage_Entering_Commands.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "usage_Matrices_and_Arrays.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "Data_Types", "usage_Numeric_Types.jl")) +end +``` + +Benchmarking the tests: +```julia +@snoopiBench "MatLang" +``` +[Ref](https://github.com/juliamatlab/MatLang/blob/master/deps/SnoopCompile/snoopBenchmark.jl) + + +To run the benchmark online, add the following to your yaml file after `Generating precompile files` step: + +```yaml +- name: Running Benchmark + run: julia --project=@. -e 'include("deps/SnoopCompile/snoopBenchmark.jl")' +``` diff --git a/src/SnoopCompile.jl b/src/SnoopCompile.jl index 290edaa40..bfcfc58d1 100644 --- a/src/SnoopCompile.jl +++ b/src/SnoopCompile.jl @@ -18,48 +18,6 @@ include("snoopc.jl") include("parcel_snoopc.jl") include("write.jl") - -""" - timesum(snoop) - -Calculates and prints the total time measured by a snoop macro - -# Examples -Julia can cache inference results so to measure the effect of adding _precompile_() sentences generated by snoopi to your package, use the fllowing benchmark. This benchmark measures inference time taken during loading and running of a package. -- dev your package - -- comment the precompile part of your package (`include()` and `_precompile_()`) -- run the following benchmark -- restart Julia - -- uncomment the precompile part of your package (`include()` and `_precompile_()`) -- run the following benchmark -- restart Julia - -## Benchmark -```julia -using SnoopCompile - -println("Package load time:") -loadSnoop = @snoopi using MatLang - -timesum(loadSnoop) - -println("Running Examples/Tests:") -runSnoop = @snoopi begin - using MatLang - include(joinpath(dirname(dirname(pathof(MatLang))),"test","runtests.jl")) -end - -timesum(runSnoop) -``` -""" -function timesum(snoop::Vector{Tuple{Float64, Core.MethodInstance}}) - if isempty(snoop) - return 0.0 - else - return sum(first, snoop) - end -end +include("bot.jl") end # module diff --git a/src/bot.jl b/src/bot.jl new file mode 100644 index 000000000..61e16e497 --- /dev/null +++ b/src/bot.jl @@ -0,0 +1,30 @@ +export precompile_activator, precompile_deactivator, precompile_pather, @snoopiBot, @snoopiBench, BotConfig + +const UStrings = Union{AbstractString,Regex,AbstractChar} +################################################################ +""" + BotConfig + +Config object that holds the options and configuration for the SnoopCompile bot. This object is fed to the `@snoopiBot`. + +# Arguments: +- `packageName::String` +- `subst::Vector{Pair{UStrings, UStrings}}` : to replace a packages precompile setences with another's package like `["ImageTest" => "Images"]` +- `blacklist::Vector{UStrings}` : to remove some precompile sentences + +`const UStrings == Union{AbstractString,Regex,AbstractChar}` # every string like type that `replace()` has a method for. +""" +struct BotConfig + packageName::String + subst::Vector{Pair{T1, T2}} where {T1<:UStrings, T2 <: UStrings} + blacklist::Vector{T3} where {T3<:UStrings} +end + +function BotConfig(packageName::String; subst::Vector{Pair{T1, T2}} where {T1<:UStrings, T2 <: UStrings} = Vector{Pair{String, String}}(), blacklist::Vector{T3} where {T3<:UStrings}= String[]) + return BotConfig(packageName, subst, blacklist) +end + +include("bot/botutils.jl") +include("bot/precompileInclude.jl") +include("bot/snoopiBot.jl") +include("bot/snoopiBench.jl") diff --git a/src/bot/botutils.jl b/src/bot/botutils.jl new file mode 100644 index 000000000..c99f3eea0 --- /dev/null +++ b/src/bot/botutils.jl @@ -0,0 +1,18 @@ +################################################################ +import Pkg +""" +Should be removed once Pkg allows adding test dependencies to the current environment + +Used in Github Action workflow yaml file +""" +function addtestdep() + if isfile("test/Test.toml") + testToml = Pkg.Types.parse_toml("test/Test.toml") + else + error("please add a Test.toml to the /test directory for test dependencies") + end + + for (name, uuid) in testToml["deps"] + Pkg.add(Pkg.PackageSpec(name = name, uuid = uuid)) + end +end diff --git a/src/bot/precompileInclude.jl b/src/bot/precompileInclude.jl new file mode 100644 index 000000000..04d63059c --- /dev/null +++ b/src/bot/precompileInclude.jl @@ -0,0 +1,118 @@ + +""" + precompile_pather(packageName::String) + +To get the path of precompile_packageName.jl file + +Written exclusively for SnoopCompile Github actions. +# Examples +```julia +precompilePath, precompileFolder = precompile_pather("MatLang") +``` +""" +function precompile_pather(packageName::String) + return "\"../deps/SnoopCompile/precompile/precompile_$packageName.jl\"", + "$(pwd())/deps/SnoopCompile/precompile/" +end + +precompile_pather(packageName::Symbol) = precompile_pather(string(packageName)) +precompile_pather(packageName::Module) = precompile_pather(string(packageName)) + +################################################################ + +function precompile_regex(precompilePath) + # https://stackoverflow.com/questions/3469080/match-whitespace-but-not-newlines + # {1,} for any number of spaces + c1 = Regex("#[^\\S\\r\\n]{0,}include\\($(precompilePath)\\)") + c2 = r"#\s{0,}_precompile_\(\)" + a1 = "include($precompilePath)" + a2 = "_precompile_()" + return c1, c2, a1, a2 +end +################################################################ + +""" + precompile_activator(packagePath, precompilePath) + +Activates precompile of a package by adding or uncommenting include() of *.jl file generated by SnoopCompile and _precompile_(). + +packagePath is the same as `pathof`. However, `pathof(module)` isn't used to prevent loadnig the package. + +Written exclusively for SnoopCompile Github actions. +""" +function precompile_activator(packagePath::String, precompilePath::String) + + packageText = Base.read(packagePath, String) + + c1, c2, a1, a2 = precompile_regex(precompilePath) + + # Checking availability of _precompile_ code + commented = occursin(c1, packageText) && occursin(c2, packageText) + available = occursin(a1, packageText) && occursin(a2, packageText) + + if commented + packageEdited = foldl(replace, + ( + c1 => a1, + c2 => a2, + ), + init = packageText) + + Base.write(packagePath, packageEdited) + + println("precompile is activated") + elseif available + # do nothing + println("precompile is already activated") + else + # TODO: add code automatiaclly + error(""" add the following codes into your PackageName.jl file under src folder: + #include($precompilePath) + #_precompile_() + """) + end + +end + +""" + precompile_deactivator(packagePath, precompilePath) + +Deactivates precompile of a package by commenting include() of *.jl file generated by SnoopCompile and _precompile_(). + +packagePath is the same as `pathof`. However, `pathof(module)` isn't used to prevent loadnig the package. + +Written exclusively for SnoopCompile Github actions. +""" +function precompile_deactivator(packagePath::String, precompilePath::String) + + packageText = Base.read(packagePath, String) + + c1, c2, a1, a2 = precompile_regex(precompilePath) + + # Checking availability of _precompile_ code + commented = occursin(c1, packageText) && occursin(c2, packageText) + available = occursin(a1, packageText) && occursin(a2, packageText) + + if available && !commented + packageEdited = foldl(replace, + ( + a1 => "#"*a1, + a2 => "#"*a2, + ), + init = packageText) + + Base.write(packagePath, packageEdited) + + println("precompile is deactivated") + elseif commented + # do nothing + println("precompile is already deactivated") + else + # TODO: add code automatiaclly + error(""" add the following codes into your PackageName.jl file under src folder: + #include($precompilePath) + #_precompile_() + """) + end + +end diff --git a/src/bot/snoopiBench.jl b/src/bot/snoopiBench.jl new file mode 100644 index 000000000..1b07418ef --- /dev/null +++ b/src/bot/snoopiBench.jl @@ -0,0 +1,143 @@ +################################################################ +""" + timesum(snoop) + +Calculates and prints the total time measured by a snoop macro. + +It is used inside @snoopiBench. Julia can cache inference results so to measure the effect of adding _precompile_() sentences generated by snoopi to your package, use the [`@snoopiBench`](@ref). This benchmark measures inference time taken during loading and running of a package. + +# Examples +```julia +using SnoopCompile +data = @snoopi begin + include(joinpath(dirname(dirname(pathof(MatLang))),"test","runtests.jl")) +end; +println(timesum(data)); +``` + +## Manual Benchmark (withtout using [`@snoopiBench`](@ref)) +- dev your package + +- comment the precompile part of your package (`include()` and `_precompile_()`) +- run the following benchmark +- restart Julia + +- uncomment the precompile part of your package (`include()` and `_precompile_()`) +- run the following benchmark +- restart Julia + +### Benchmark +```julia +using SnoopCompile + +println("Package load time:") +loadSnoop = @snoopi using MatLang + +timesum(loadSnoop) + +println("Running Examples/Tests:") +runSnoop = @snoopi begin + using MatLang + include(joinpath(dirname(dirname(pathof(MatLang))),"test","runtests.jl")) +end + +timesum(runSnoop) +``` +""" +function timesum(snoop::Vector{Tuple{Float64, Core.MethodInstance}}) + if isempty(snoop) + return 0.0 + else + return sum(first, snoop) + end +end + +################################################################ +""" + @snoopiBench(packageName::String, snoopScript::Expr) + @snoopiBench(packageName::String) + +Performs an infertime benchmark by activating and deactivating the _precompile_() +# Examples +Benchmarking the load infer time +```julia +println("loading infer benchmark") + +@snoopiBench "MatLang" using MatLang +``` + +Benchmarking the example infer time +```julia +println("examples infer benchmark") + +@snoopiBench "MatLang" begin + using MatLang + examplePath = joinpath(dirname(dirname(pathof(MatLang))), "examples") + # include(joinpath(examplePath,"Language_Fundamentals", "usage_Entering_Commands.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "usage_Matrices_and_Arrays.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "Data_Types", "usage_Numeric_Types.jl")) +end +``` +""" +macro snoopiBench(packageName::String, snoopScript::Expr) + + ################################################################ + packagePath = joinpath(pwd(),"src","$packageName.jl") + precompilePath, precompileFolder = precompile_pather(packageName) + + juliaCode = """ + using SnoopCompile; data = @snoopi begin + $(string(snoopScript)); + end; + println(timesum(data)); + """ + juliaCmd = `julia --project=@. -e "$juliaCode"` + quote + packageSym = Symbol($packageName) + ################################################################ + using SnoopCompile + println("""******************* + Benchmark Started + ******************* + """) + ################################################################ + println("""Precompile Deactivated Benchmark + ------------------------ + """) + precompile_deactivator($packagePath, $precompilePath); + ### Log the compiles + run($juliaCmd) + ################################################################ + println("""Precompile Activated Benchmark + ------------------------ + """) + precompile_activator($packagePath, $precompilePath) + ### Log the compiles + run($juliaCmd) + println("""******************* + Benchmark Finished + ******************* + """) + end + +end + +""" + @snoopiBench packageName::String + +Benchmarking the infer time of the tests: +```julia +@snoopiBench "MatLang" +``` +""" +macro snoopiBench(packageName::String) + package = Symbol(packageName) + snoopScript = :( + using $(package); + runtestpath = joinpath(dirname(dirname(pathof($(package)))), "test", "runtests.jl"); + include(runtestpath); + ) + return quote + @snoopiBench $packageName $(snoopScript) + end +end diff --git a/src/bot/snoopiBot.jl b/src/bot/snoopiBot.jl new file mode 100644 index 000000000..4f24712a9 --- /dev/null +++ b/src/bot/snoopiBot.jl @@ -0,0 +1,109 @@ + +################################################################ +""" + @snoopiBot config::BotConfig snoopScript + +macro that generates precompile files and includes them in the package. Calls other bot functions. + +# Examples + +`@snoopiBot` the examples that call the package functions. + +```julia +using SnoopCompile + +@snoopiBot "MatLang" begin + using MatLang + examplePath = joinpath(dirname(dirname(pathof(MatLang))), "examples") + include(joinpath(examplePath,"Language_Fundamentals", "usage_Entering_Commands.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "usage_Matrices_and_Arrays.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "Data_Types", "usage_Numeric_Types.jl")) +end +``` +""" +macro snoopiBot(config::BotConfig, snoopScript) + + packageName = config.packageName + blacklist = config.blacklist + subst = config.subst + ################################################################ + packagePath = joinpath(pwd(),"src","$packageName.jl") + precompilePath, precompileFolder = precompile_pather(packageName) + + quote + packageSym = Symbol($packageName) + ################################################################ + using SnoopCompile + ################################################################ + precompile_deactivator($packagePath, $precompilePath); + ################################################################ + + ### Log the compiles + data = @snoopi begin + $(esc(snoopScript)) + end + + ################################################################ + ### Parse the compiles and generate precompilation scripts + pc = SnoopCompile.parcel(data, subst = $subst, blacklist = $blacklist) + onlypackage = Dict( packageSym => sort(pc[packageSym]) ) + SnoopCompile.write($precompileFolder,onlypackage) + ################################################################ + precompile_activator($packagePath, $precompilePath) + end + +end + +macro snoopiBot(packageName::String, snoopScript) + config = BotConfig(packageName) + return quote + @snoopiBot $config $(esc(snoopScript)) + end +end +macro snoopiBot(configExpr, snoopScript) + config = eval(configExpr) + return quote + @snoopiBot $config $(esc(snoopScript)) + end +end + +""" + @snoopiBot config::BotConfig + +If you do not have additional examples, you can use your runtests.jl file. To do that use: + +```julia +using SnoopCompile + +# using runtests: +@snoopiBot "MatLang" +``` +""" +macro snoopiBot(config::BotConfig) + + packageName = config.packageName + + package = Symbol(packageName) + snoopScript = esc(quote + using $(package) + runtestpath = joinpath(dirname(dirname(pathof( $package ))), "test", "runtests.jl") + include(runtestpath) + end) + return quote + @snoopiBot $config $(esc(snoopScript)) + end +end + +macro snoopiBot(packageName::String) + config = BotConfig(packageName) + return quote + @snoopiBot $config + end +end + +macro snoopiBot(configExpr) + config = eval(configExpr) + return quote + @snoopiBot $config + end +end diff --git a/test/bot/bot.jl b/test/bot/bot.jl new file mode 100644 index 000000000..98af069ab --- /dev/null +++ b/test/bot/bot.jl @@ -0,0 +1,67 @@ +using SnoopCompile, Test, Suppressor + +cd(@__DIR__) +@testset "bot" begin + @testset "precompileInclude" begin + @testset "precompile_pather" begin + precompilePath, precompileFolder = precompile_pather("TestPackage") + @test precompilePath == "\"../deps/SnoopCompile/precompile/precompile_TestPackage.jl\"" + @test precompileFolder == "$(pwd())/deps/SnoopCompile/precompile/" + end + + @testset "precompile_activator" begin + Base.write("activated.jl", """ + include("../deps/SnoopCompile/precompile/precompile_TestPackage.jl") + _precompile_() + """) + + precompilePath, precompileFolder = precompile_pather("TestPackage") + @test (@capture_out precompile_activator("activated.jl", precompilePath)) == "precompile is already activated\n" + end + + @testset "precompile_deactivator" begin + Base.write("deactivated.jl", """ + # include("../deps/SnoopCompile/precompile/precompile_TestPackage.jl") + # _precompile_() + """) + + precompilePath, precompileFolder = precompile_pather("TestPackage") + @test (@capture_out precompile_deactivator("deactivated.jl", precompilePath)) == "precompile is already deactivated\n" + end + end + + + # Fails. # snoopiBot pwd() differs from what it is cd to + # instead test the code in MatLang actions: https://github.com/juliamatlab/MatLang/actions?query=workflow%3ASnoopCompile + #= + @testset "snoopiBot" begin + using Pkg; Pkg.develop("MatLang") + + examplePath = Base.read(`cmd /c julia -e 'import MatLang; print(pathof(MatLang))'`, String) + cd(dirname(dirname(examplePath))) + + @snoopiBot "MatLang" begin + + using MatLang + examplePath = joinpath(dirname(dirname(pathof(MatLang))), "examples") + include(joinpath(examplePath,"Language_Fundamentals", "usage_Matrices_and_Arrays.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "Data_Types", "usage_Numeric_Types.jl")) + end + end + + @testset "snoopiBench" begin + using Pkg; Pkg.develop("MatLang") + + examplePath = Base.read(`cmd /c julia -e 'import MatLang; print(pathof(MatLang))'`, String) + cd(dirname(dirname(examplePath))) + + @snoopiBench "MatLang" begin + using MatLang + examplePath = joinpath(dirname(dirname(pathof(MatLang))), "examples") + include(joinpath(examplePath,"Language_Fundamentals", "usage_Matrices_and_Arrays.jl")) + include(joinpath(examplePath,"Language_Fundamentals", "Data_Types", "usage_Numeric_Types.jl")) + end + end + =# + +end diff --git a/test/runtests.jl b/test/runtests.jl index 814d5b29a..f74fdd87f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -86,3 +86,4 @@ end end include("colortypes.jl") +include("bot/bot.jl")