update doc string syntax (JuliaLang#25938)

- disallow extra lines between a docstring and the documented object
- parse an extra expression after a newline in calls to `@doc`
- deprecate special parsing of `doc" "`
- deprecate `->` for doc strings

base/int.jl

@@ -452,7 +452,7 @@ end
 # doesn't work either at this point.
 if nameof(@__MODULE__) === :Base
     for fname in (:mod, :rem)
-        @eval @doc ("""
+        @eval @doc """
             rem(x::Integer, T::Type{<:Integer}) -> T
             mod(x::Integer, T::Type{<:Integer}) -> T
             %(x::Integer, T::Type{<:Integer}) -> T
@@ -466,7 +466,7 @@ if nameof(@__MODULE__) === :Base
         julia> 129 % Int8
         -127
         ```
-        """ -> $fname(x::Integer, T::Type{<:Integer}))
+        """ $fname(x::Integer, T::Type{<:Integer})
     end
 end
 

base/libc.jl

@@ -3,7 +3,7 @@
 module Libc
 @doc """
 Interface to libc, the C standard library.
-""" -> Libc
+""" Libc
 
 import Base: transcode
 

base/statistics.jl

@@ -546,8 +546,8 @@ cor(x::AbstractVecOrMat, y::AbstractVecOrMat, vardim::Int=1) =
 
 Compute the middle of a scalar value, which is equivalent to `x` itself, but of the type of `middle(x, x)` for consistency.
 """
-# Specialized functions for real types allow for improved performance
 middle(x::Union{Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128}) = Float64(x)
+# Specialized functions for real types allow for improved performance
 middle(x::AbstractFloat) = x
 middle(x::Real) = (x + zero(x)) / 1
 

base/sysinfo.jl

@@ -3,7 +3,7 @@
 module Sys
 @doc """
 Provide methods for retrieving information about hardware and the operating system.
-""" -> Sys
+""" Sys
 
 export BINDIR,
        CPU_CORES,

doc/src/manual/documentation.md

@@ -3,9 +3,10 @@
 Julia enables package developers and users to document functions, types and other objects easily
 via a built-in documentation system since Julia 0.4.
 
-The basic syntax is very simple: any string appearing at the top-level right before an object
+The basic syntax is simple: any string appearing at the top-level right before an object
 (function, macro, type or instance) will be interpreted as documenting it (these are called *docstrings*).
-Here is a very simple example:
+Note that no blank lines or comments may intervene between a docstring and the documented object.
+Here is a basic example:
 
 ```julia
 "Tell whether there are too foo items in the array."
@@ -248,6 +249,20 @@ The `@doc` macro associates its first argument with its second in a per-module d
 macro simply creates an object representing the Markdown content. In the future it is likely to
 do more advanced things such as allowing for relative image or link paths.
 
+To make it easier to write documentation, the parser treats the macro name `@doc` specially:
+if a call to `@doc` has one argument, but another expression appears after a single line
+break, then that additional expression is added as an argument to the macro.
+Therefore the following syntax is parsed as a 2-argument call to `@doc`:
+
+```julia
+@doc raw"""
+...
+"""
+f(x) = x
+```
+
+This makes it easy to use an arbitrary object (here a `raw` string) as a docstring.
+
 When used for retrieving documentation, the `@doc` macro (or equally, the `doc` function) will
 search all `META` dictionaries for metadata relevant to the given object and return it. The returned
 object (some Markdown content, for example) will by default display itself intelligently. This
@@ -308,24 +323,9 @@ y = MyType("y")
 
 A comprehensive overview of all documentable Julia syntax.
 
-In the following examples `"..."` is used to illustrate an arbitrary docstring which may be one
-of the follow four variants and contain arbitrary text:
-
-```julia
-"..."
-
-doc"..."
-
-"""
-...
-"""
-
-doc"""
-...
-"""
-```
+In the following examples `"..."` is used to illustrate an arbitrary docstring.
 
-`@doc_str` should only be used when the docstring contains `$` or `\` characters that should not
+`doc""` should only be used when the docstring contains `$` or `\` characters that should not
 be parsed by Julia such as LaTeX syntax or Julia source code examples containing interpolation.
 
 ### Functions and Methods

src/julia-parser.scm

@@ -2317,17 +2317,19 @@
                             (parse-unary-prefix s))))
               (peek-token s)
               (if (ts:space? s)
-                  `(macrocall ,(macroify-name head)
-                              ,startloc
-                              ,@(parse-space-separated-exprs s))
+                  (maybe-docstring
+                   s `(macrocall ,(macroify-name head)
+                                 ,startloc
+                                 ,@(parse-space-separated-exprs s)))
                   (let ((call (parse-call-chain s head #t)))
                     (if (and (pair? call) (eq? (car call) 'call))
                         `(macrocall ,(macroify-name (cadr call))
                                     ,startloc
                                     ,@(cddr call))
-                        `(macrocall ,(macroify-name call)
-                                    ,startloc
-                                    ,@(parse-space-separated-exprs s))))))))
+                        (maybe-docstring
+                         s `(macrocall ,(macroify-name call)
+                                       ,startloc
+                                       ,@(parse-space-separated-exprs s)))))))))
           ;; command syntax
           ((eqv? t #\`)
            (take-token s)
@@ -2352,30 +2354,60 @@
                (quote ,(apply macroify-name (cadr (caddr e)) suffixes))))
         (else (error (string "invalid macro usage \"@(" (deparse e) ")\"" )))))
 
-(define (simple-string-literal? e) (string? e))
+(define (called-macro-name e)
+  (if (and (length= e 3) (eq? (car e) '|.|)
+           (pair? (caddr e)) (eq? (car (caddr e)) 'quote))
+      (called-macro-name (cadr (caddr e)))
+      e))
 
-(define (string-macro-name? x)
-  (let ((x (string x)))
-    (and (> (string-length x) 4)
-         (eqv? (string.char x 0) #\@)
-         (eqv? (string.sub x (string.dec x (sizeof x) 4)) "_str"))))
+(define (maybe-docstring s e)
+  (if (and (length= e 4)
+           (eq? (called-macro-name (cadr e)) '@doc))
+      (let ((arg (cadddr e)))
+        (if (and (pair? arg) (eq? (car arg) '->))
+            (begin (parser-depwarn s "@doc call with ->" "a line break")
+                   e)
+            (let loop ((t  (peek-token s))
+                       (nl 0))
+              (cond ((closing-token? t) e)
+                    ((newline? t)
+                     (if (> nl 0)
+                         e
+                         (begin (take-token s)
+                                (loop (peek-token s) 1))))
+                    (else
+                     `(,@e ,(parse-eq s)))))))
+      e))
+
+(define (simple-string-literal? e) (string? e))
 
-(define (doc-string-literal? e)
+(define (doc-string-literal? s e)
   (or (simple-string-literal? e)
       (and (pair? e)
            (or (eq? (car e) 'string) ; string interpolation
                (and (eq? (car e) 'macrocall)
-                    (string-macro-name? (cadr e))
                     (or (and (length= e 3) (simple-string-literal? (caddr e)))
-                        (and (length= e 4) (simple-string-literal? (cadddr e)))))))))
+                        (and (length= e 4) (simple-string-literal? (cadddr e))))
+                    (eq? (cadr e) '@doc_str)
+                    (begin (parser-depwarn s "doc\" \"" "@doc doc\" \"")
+                           #t))))))
 
 (define (parse-docstring s production)
   (let ((startloc (line-number-node s)) ; be sure to use the line number from the head of the docstring
-        (ex (production s)))
-    (if (and (doc-string-literal? ex)
-             (let loop ((t (peek-token s)))
+        (ex       (production s)))
+    (if (and (doc-string-literal? s ex)
+             (let loop ((t  (peek-token s))
+                        (nl 0))
                (cond ((closing-token? t) #f)
-                     ((newline? t) (take-token s) (loop (peek-token s)))
+                     ((newline? t)
+                      (if (= nl 1)
+                          ;;#f  ;; extra line => not a doc string. enable when deprecation is removed.
+                          (begin (parser-depwarn s "multiple line breaks between doc string and object"
+                                                 "at most one line break")
+                                 (take-token s)
+                                 (loop (peek-token s) 2))
+                          (begin (take-token s)
+                                 (loop (peek-token s) 1))))
                      (else #t))))
         `(macrocall (core @doc) ,startloc ,ex ,(production s))
         ex)))

stdlib/LibGit2/src/consts.jl

@@ -362,7 +362,7 @@ if version() >= v"0.24.0"
     * `CONFIG_LEVEL_LOCAL` - Repository specific configuration file; `\$WORK_DIR/.git/config` on non-bare repos
     * `CONFIG_LEVEL_APP` - Application specific configuration file; freely defined by applications
     * `CONFIG_HIGHEST_LEVEL` - Represents the highest level available config file (i.e. the most specific config file available that actually is loaded)
-    """ ->
+    """
     @enum(GIT_CONFIG, CONFIG_LEVEL_DEFAULT     = 0,
                       CONFIG_LEVEL_PROGRAMDATA = 1,
                       CONFIG_LEVEL_SYSTEM      = 2,
@@ -384,7 +384,7 @@ else
     * `CONFIG_LEVEL_LOCAL` - Repository specific configuration file; `\$WORK_DIR/.git/config` on non-bare repos
     * `CONFIG_LEVEL_APP` - Application specific configuration file; freely defined by applications
     * `CONFIG_HIGHEST_LEVEL` - Represents the highest level available config file (i.e. the most specific config file available that actually is loaded)
-    """ ->
+    """
     @enum(GIT_CONFIG, CONFIG_LEVEL_DEFAULT     = 0,
                       CONFIG_LEVEL_SYSTEM      = 1,
                       CONFIG_LEVEL_XDG         = 2,

stdlib/Libdl/src/Libdl.jl

@@ -5,7 +5,7 @@ __precompile__(true)
 module Libdl
 @doc """
 Interface to libdl. Provides dynamic linking support.
-""" -> Libdl
+""" Libdl
 
 import Base.DL_LOAD_PATH
 
@@ -32,7 +32,7 @@ const RTLD_NOLOAD    = 0x00000020
 const RTLD_DEEPBIND  = 0x00000040
 const RTLD_FIRST     = 0x00000080
 
-@doc """
+"""
     RTLD_DEEPBIND
     RTLD_FIRST
     RTLD_GLOBAL
@@ -44,7 +44,7 @@ const RTLD_FIRST     = 0x00000080
 
 Enum constant for [`dlopen`](@ref). See your platform man page for details, if
 applicable.
-""" ->
+"""
 (RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW)
 
 """

stdlib/LinearAlgebra/src/blas.jl

@@ -3,7 +3,7 @@
 module BLAS
 @doc """
 Interface to BLAS subroutines.
-""" -> BLAS
+""" BLAS
 
 import ..axpy!, ..axpby!
 import Base: copyto!

stdlib/LinearAlgebra/src/lapack.jl

@@ -3,7 +3,7 @@
 module LAPACK
 @doc """
 Interfaces to LAPACK subroutines.
-""" -> LAPACK
+""" LAPACK
 
 const liblapack = Base.liblapack_name
 

test/choosetests.jl

@@ -5,7 +5,7 @@ using Random
 const STDLIB_DIR = joinpath(Sys.BINDIR, "..", "share", "julia", "site", "v$(VERSION.major).$(VERSION.minor)")
 const STDLIBS = readdir(STDLIB_DIR)
 
-@doc """
+"""
 
 `tests, net_on, exit_on_error, seed = choosetests(choices)` selects a set of tests to be
 run. `choices` should be a vector of test names; if empty or set to
@@ -30,7 +30,7 @@ in the `choices` argument:
    - "--seed=SEED", which sets the value of `seed` to `SEED`
      (parsed as an `UInt128`); `seed` is otherwise initialized randomly.
      This option can be used to reproduce failed tests.
-""" ->
+"""
 function choosetests(choices = [])
     testnames = [
         "subarray", "core", "compiler", "worlds",

test/docs.jl

@@ -30,19 +30,19 @@ function docstring_startswith(d1, d2)
 end
 docstring_startswith(d1::DocStr, d2) = docstring_startswith(parsedoc(d1), d2)
 
-@doc "Doc abstract type" ->
+@doc "Doc abstract type"
 abstract type C74685{T,N} <: AbstractArray{T,N} end
 @test stringmime("text/plain", Docs.doc(C74685))=="  Doc abstract type\n"
 @test string(Docs.doc(C74685))=="Doc abstract type\n"
 
 macro macro_doctest() end
-@doc "Helps test if macros can be documented with `@doc \"...\" -> @...`." ->
+@doc "Helps test if macros can be documented with `@doc \"...\" @...`."
 :@macro_doctest
 
 @test (@doc @macro_doctest) !== nothing
 
 # test that random stuff interpolated into docstrings doesn't break search or other methods here
-doc"""
+@doc doc"""
 break me:
 
     code
@@ -602,7 +602,7 @@ end
 
 f1_11993()
 
-@doc "Document inline function with old syntax" ->
+@doc "Document inline function with old syntax"
 @inline f2_11993() = nothing
 
 @test (@doc f2_11993) !== nothing
@@ -839,7 +839,7 @@ $(curmod_prefix)Undocumented.D <: $(curmod_prefix)Undocumented.B <: $(curmod_pre
 """)
 @test docstrings_equal(@doc(Undocumented.D), doc"$doc_str")
 
-let d = @doc Undocumented.f
+let d = @doc(Undocumented.f)
     io = IOBuffer()
     show(io, MIME"text/markdown"(), d)
     @test startswith(String(take!(io)),"""
@@ -849,7 +849,7 @@ let d = @doc Undocumented.f
     """)
 end
 
-let d = @doc Undocumented.undocumented
+let d = @doc(Undocumented.undocumented)
     io = IOBuffer()
     show(io, MIME"text/markdown"(), d)
     @test startswith(String(take!(io)), """

test/syntax.jl

@@ -1269,7 +1269,34 @@ end
 @test_throws ParseError Meta.parse("using Colors()")
 
 let ex = Meta.parse("md\"x\"
+                     f(x) = x", 1)[1]  # custom string literal is not a docstring
+    @test Meta.isexpr(ex, :macrocall)
+    @test ex.args[1] === Symbol("@md_str")
+    @test length(ex.args) == 3
+end
+
+let ex = Meta.parse("@doc raw\"
+                     \"
                      f(x) = x")
     @test Meta.isexpr(ex, :macrocall)
-    @test ex.args[1] == Core.GlobalRef(Core, Symbol("@doc"))
+    @test ex.args[1] === Symbol("@doc")
+    @test length(ex.args) == 4
+    @test Meta.isexpr(ex.args[4], :(=))
+end
+
+let ex = Meta.parse("@doc raw\"
+                     \"
+
+                     f(x) = x", 1)[1]
+    @test Meta.isexpr(ex, :macrocall)
+    @test ex.args[1] === Symbol("@doc")
+    @test length(ex.args) == 3
 end
+
+# TODO: enable when 0.7 deprecations are removed
+#@test Meta.parse("\"x\"
+#                  # extra line, not a doc string
+#                  f(x) = x", 1)[1] === "x"
+#@test Meta.parse("\"x\"
+#
+#                  f(x) = x", 1)[1] === "x"