SourceWalk

Manifest.toml

@@ -3,12 +3,26 @@
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
+[[CSTParser]]
+deps = ["LibGit2", "Test", "Tokenize"]
+git-tree-sha1 = "d878de3315f9b6569851d919f7976fe527d00c24"
+repo-rev = "location"
+repo-url = "https://github.com/MikeInnes/CSTParser.jl"
+uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
+version = "0.5.2+"
+
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
 git-tree-sha1 = "49269e311ffe11ac5b334681d212329002a9832a"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
 version = "1.5.1"
 
+[[DataStructures]]
+deps = ["InteractiveUtils", "OrderedCollections", "Random", "Serialization", "Test"]
+git-tree-sha1 = "ca971f03e146cf144a9e2f2ce59674f5bf0e8038"
+uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+version = "0.15.0"
+
 [[Dates]]
 deps = ["Printf"]
 uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -45,6 +59,12 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[OrderedCollections]]
+deps = ["Random", "Serialization", "Test"]
+git-tree-sha1 = "85619a3f3e17bb4761fe1b1fd47f0e979f964d5b"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.0.2"
+
 [[Pkg]]
 deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -86,6 +106,12 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
+[[Tokenize]]
+deps = ["Printf", "Test"]
+git-tree-sha1 = "3e83f60b74911d3042d3550884ca2776386a02b8"
+uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624"
+version = "0.5.3"
+
 [[UUIDs]]
 deps = ["Random", "SHA"]
 uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

Project.toml

@@ -2,7 +2,10 @@ name = "MacroTools"
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 
 [deps]
+CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
+DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 julia = "≥ 1.0"

REQUIRE

@@ -1,2 +1,4 @@
 julia 1.0
 Compat 0.62
+DataStructures
+CSTParser

docs/Manifest.toml

@@ -3,12 +3,26 @@
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
+[[CSTParser]]
+deps = ["LibGit2", "Test", "Tokenize"]
+git-tree-sha1 = "d878de3315f9b6569851d919f7976fe527d00c24"
+repo-rev = "location"
+repo-url = "https://github.com/MikeInnes/CSTParser.jl"
+uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
+version = "0.5.2+"
+
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
 git-tree-sha1 = "84aa74986c5b9b898b0d1acaf3258741ee64754f"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
 version = "2.1.0"
 
+[[DataStructures]]
+deps = ["InteractiveUtils", "OrderedCollections", "Random", "Serialization", "Test"]
+git-tree-sha1 = "ca971f03e146cf144a9e2f2ce59674f5bf0e8038"
+uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+version = "0.15.0"
+
 [[Dates]]
 deps = ["Printf"]
 uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -57,7 +71,7 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
 [[MacroTools]]
-deps = ["Compat"]
+deps = ["CSTParser", "Compat", "DataStructures", "Test"]
 path = ".."
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 version = "0.4.5+"
@@ -69,6 +83,12 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[OrderedCollections]]
+deps = ["Random", "Serialization", "Test"]
+git-tree-sha1 = "85619a3f3e17bb4761fe1b1fd47f0e979f964d5b"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.0.2"
+
 [[Pkg]]
 deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -110,6 +130,12 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
+[[Tokenize]]
+deps = ["Printf", "Test"]
+git-tree-sha1 = "3e83f60b74911d3042d3550884ca2776386a02b8"
+uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624"
+version = "0.5.3"
+
 [[UUIDs]]
 deps = ["Random", "SHA"]
 uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

docs/Project.toml

@@ -1,3 +1,4 @@
 [deps]
+CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"

docs/make.jl

@@ -5,6 +5,7 @@ makedocs(
     pages = [
         "Home" => "index.md",
         "Pattern Matching" => "pattern-matching.md",
+        "SourceWalk" => "sourcewalk.md",
         "Utilities" => "utilities.md"],
     format = Documenter.HTML(prettyurls = haskey(ENV, "CI")))
 

docs/src/sourcewalk.md

@@ -0,0 +1,98 @@
+# SourceWalk
+
+Pattern matching is really useful if you're writing macros, but it also works
+on Julia source code files. You can try this with the `sourcewalk` function,
+which behaves like `postwalk` but with a source file as input.
+
+!!! note
+
+    To use this functionality you currently need to a fork of CSTParser.
+    ```julia
+    ] add https://github.com/MikeInnes/CSTParser.jl#location
+    ```
+
+For example, if we have a file like
+
+```julia
+# test.jl
+foo(a, b)
+```
+
+And run
+
+```julia
+sourcewalk("test.jl") do x
+  x == :foo ? :bar : x
+end
+```
+
+then our file will have changed to
+
+```julia
+# test.jl
+foo(a, b)
+```
+
+You can also pass `sourcewalk` a directory, and it will work on all `.jl` files
+in that directory. You can use `sourcewalk` anywhere you'd normally use a
+Regex-based find and replace, but it'll take care of subtle issues like matching
+brackets, operator precedence and code formatting.
+
+We can use this to do some very powerful code transformations with minimal
+effort. For example, [this whole
+patch](https://github.com/MikeInnes/julia/commit/45ccbc6a3c003accb0eedca889835071c371ae86)
+was generated simply with `sourcewalk(longdef, "base")`. Let's look at some
+examples.
+
+!!! warning
+
+    We recommend running SourceWalk on files that are checked into Git. This way
+    you can easily look through every change and check it's sensible. Being
+    careless with `sourcewalk` can delete all your code!
+
+## Find and Replace
+
+Here's a more realistic example. When working with strings, we use `nextind(s, i)`
+to get the next index after `i` – which will often just be `i+1`, but may
+be something else if we're working with unicode.
+
+How do you know your code handles unicode correctly? One way to check is to
+actually replace `nextind(s, i)` with `i + 1` and _make sure that the tests
+fail_ – if they don't, we're not testing with unicode input properly. This is a
+form of [mutation testing](https://en.wikipedia.org/wiki/Mutation_testing).
+
+Of course, the transformation is very simple with MacroTools:
+
+```julia
+julia> function nextinds(x)
+         @capture(x, nextind(str_, i_)) && return :($i + 1)
+         @capture(x, prevind(str_, i_)) && return :($i - 1)
+         return x
+       end
+nextinds (generic function with 1 method)
+
+julia> nextinds(:(nextind(a, b)))
+:(b + 1)
+```
+
+We can check this works on text, before running it on our file, using `textwalk`.
+
+```julia
+julia> using MacroTools: textwalk
+
+julia> textwalk(nextinds, "nextind(s, i)")
+"i + 1"
+```
+
+And verify that it correctly handles operator precedence – so we won't get invalid
+syntax or unexpected changes to the meaning of our code.
+
+```julia
+julia> textwalk(nextinds, "1:nextind(s, i)")
+"1:i + 1"
+
+julia> textwalk(nextinds, "1*nextind(s, i)")
+"1*(i + 1)"
+```
+
+Running this on Julia's base code yields [the following patch](https://github.com/MikeInnes/julia/commit/b3964317321150c4b9ae8d629f613ee1143b3629).

src/MacroTools.jl

@@ -1,8 +1,8 @@
 module MacroTools
 
-using Compat
+using DataStructures, Compat
 using Compat.Markdown
-export @match, @capture
+export @match, @capture, sourcewalk, sourcemap
 
 include("match/match.jl")
 include("match/types.jl")
@@ -16,6 +16,13 @@ include("examples/destruct.jl")
 include("examples/threading.jl")
 include("examples/forward.jl")
 
+using CSTParser
+
+if isdefined(CSTParser, :Location)
+  include("patch/diff.jl")
+  include("patch/cst.jl")
+end
+
 const animals = Symbol[]
 const animals_file = joinpath(@__DIR__, "..", "animals.txt")