Merge pull request #23876 from JuliaLang/jb/stdlib

approach to default packages

Author: JeffBezanson

Makefile

@@ -19,7 +19,7 @@ default: $(JULIA_BUILD_MODE) # contains either "debug" or "release"
 all: debug release
 
 # sort is used to remove potential duplicates
-DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_man1dir))
+DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/site $(build_man1dir))
 ifneq ($(BUILDROOT),$(JULIAHOME))
 BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src ui doc deps test test/perf examples examples/embedding)
 BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS))
@@ -50,6 +50,11 @@ endif
 
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
 $(foreach link,base test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia)))
+$(eval $(call symlink_target,stdlib,$(build_datarootdir)/julia/site))
+
+build_defaultpkgdir = $(build_datarootdir)/julia/site/$(shell echo $(VERSDIR))
+$(build_defaultpkgdir): $(build_datarootdir)/julia/site/stdlib
+	@mv $(build_datarootdir)/julia/site/stdlib $@
 
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
@@ -81,7 +86,7 @@ ifndef JULIA_VAGRANT_BUILD
 endif
 endif
 
-julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test
+julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test $(build_defaultpkgdir)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps
 
 julia-base: julia-deps $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl

README.md

@@ -372,11 +372,12 @@ It is highly recommended to start with a fresh clone of the Julia repository.
 
 The Julia source code is organized as follows:
 
-    base/          source code for Julia's standard library
+    base/          source code for the Base module (part of Julia's standard library)
+    stdlib/        source code for other standard library packages
     contrib/       editor support for Julia source, miscellaneous scripts
     deps/          external dependencies
-    doc/manual     source for the user manual
-    doc/stdlib     source for standard library function help text
+    doc/src/manual source for the user manual
+    doc/src/stdlib source for standard library function reference
     examples/      example Julia programs
     src/           source for Julia language core
     test/          test suites

base/deprecated.jl

@@ -122,22 +122,26 @@ macro deprecate_binding(old, new, export_old=true, dep_message=nothing)
          Expr(:call, :deprecate, __module__, Expr(:quote, old)))
 end
 
-macro deprecate_moved(old, new, export_old=true)
+macro deprecate_moved(old, new, export_old=true, default_package=false)
     eold = esc(old)
     return Expr(:toplevel,
-         :(function $eold(args...; kwargs...)
-               error($eold, " has been moved to the package ", $new, ".jl.\n",
-                     "Run `Pkg.add(\"", $new, "\")` to install it, restart Julia,\n",
-                     "and then run `using ", $new, "` to load it.")
-           end),
+         default_package ? :(function $eold(args...; kwargs...)
+                                 error($eold, " has been moved to the standard library package ", $new, ".\n",
+                                       "Restart Julia and then run `using ", $new, "` to load it.")
+                             end) :
+                           :(function $eold(args...; kwargs...)
+                                 error($eold, " has been moved to the package ", $new, ".jl.\n",
+                                       "Run `Pkg.add(\"", $new, "\")` to install it, restart Julia,\n",
+                                       "and then run `using ", $new, "` to load it.")
+                             end),
          export_old ? Expr(:export, eold) : nothing,
          Expr(:call, :deprecate, __module__, Expr(:quote, old), 2))
 end
 
 # BEGIN 0.6-alpha deprecations (delete when 0.6 is released)
 
 @deprecate isambiguous(m1::Method, m2::Method, b::Bool) isambiguous(m1, m2, ambiguous_bottom=b) false
-# TODO: delete allow_bottom keyword code in Base.Test.detect_ambiguities
+# TODO: delete allow_bottom keyword code in Test.detect_ambiguities
 
 # END 0.6-alpha deprecations
 
@@ -999,73 +1003,6 @@ iteratoreltype(::Type{Task}) = EltypeUnknown()
 
 isempty(::Task) = error("isempty not defined for Tasks")
 
-@eval Base.Test begin
-    approx_full(x::AbstractArray) = x
-    approx_full(x::Number) = x
-    approx_full(x) = full(x)
-
-    function test_approx_eq(va, vb, Eps, astr, bstr)
-        va = approx_full(va)
-        vb = approx_full(vb)
-        la, lb = length(linearindices(va)), length(linearindices(vb))
-        if la != lb
-            error("lengths of ", astr, " and ", bstr, " do not match: ",
-                "\n  ", astr, " (length $la) = ", va,
-                "\n  ", bstr, " (length $lb) = ", vb)
-        end
-        diff = real(zero(eltype(va)))
-        for (xa, xb) = zip(va, vb)
-            if isfinite(xa) && isfinite(xb)
-                diff = max(diff, abs(xa-xb))
-            elseif !isequal(xa,xb)
-                error("mismatch of non-finite elements: ",
-                    "\n  ", astr, " = ", va,
-                    "\n  ", bstr, " = ", vb)
-            end
-        end
-
-        if !isnan(Eps) && !(diff <= Eps)
-            sdiff = string("|", astr, " - ", bstr, "| <= ", Eps)
-            error("assertion failed: ", sdiff,
-                "\n  ", astr, " = ", va,
-                "\n  ", bstr, " = ", vb,
-                "\n  difference = ", diff, " > ", Eps)
-        end
-    end
-
-    array_eps(a::AbstractArray{Complex{T}}) where {T} = eps(float(maximum(x->(isfinite(x) ? abs(x) : T(NaN)), a)))
-    array_eps(a) = eps(float(maximum(x->(isfinite(x) ? abs(x) : oftype(x,NaN)), a)))
-
-    test_approx_eq(va, vb, astr, bstr) =
-        test_approx_eq(va, vb, 1E4*length(linearindices(va))*max(array_eps(va), array_eps(vb)), astr, bstr)
-
-    """
-        @test_approx_eq_eps(a, b, tol)
-
-    Test two floating point numbers `a` and `b` for equality taking into account
-    a margin of tolerance given by `tol`.
-    """
-    macro test_approx_eq_eps(a, b, c)
-        Base.depwarn(string("@test_approx_eq_eps is deprecated, use `@test ", a, " ≈ ", b, " atol=", c, "` instead"),
-                    Symbol("@test_approx_eq_eps"))
-        :(test_approx_eq($(esc(a)), $(esc(b)), $(esc(c)), $(string(a)), $(string(b))))
-    end
-    export @test_approx_eq_eps
-
-    """
-        @test_approx_eq(a, b)
-
-    Deprecated. Test two floating point numbers `a` and `b` for equality taking into
-    account small numerical errors.
-    """
-    macro test_approx_eq(a, b)
-        Base.depwarn(string("@test_approx_eq is deprecated, use `@test ", a, " ≈ ", b, "` instead"),
-                    Symbol("@test_approx_eq"))
-        :(test_approx_eq($(esc(a)), $(esc(b)), $(string(a)), $(string(b))))
-    end
-    export @test_approx_eq
-end
-
 # Deprecate Array(T, dims...) in favor of proper type constructors
 @deprecate Array(::Type{T}, d::NTuple{N,Int}) where {T,N}               Array{T}(d)
 @deprecate Array(::Type{T}, d::Int...) where {T}                        Array{T}(d...)
@@ -1432,6 +1369,18 @@ end
 using .DSP
 export conv, conv2, deconv, filt, filt!, xcorr
 
+module Test
+for f in [Symbol("@inferred"), Symbol("@test"), Symbol("@test_approx_eq"),
+          Symbol("@test_approx_eq_eps"), Symbol("@test_broken"), Symbol("@test_nowarn"),
+          Symbol("@test_skip"), Symbol("@test_throws"), Symbol("@test_warn"),
+          Symbol("@testset"), :GenericArray, :GenericDict, :GenericSet, :GenericString,
+          :detect_ambiguities, :detect_unbound_args]
+    @eval Base.@deprecate_moved $f "Test" true true
+end
+end
+export Test
+deprecate(@__MODULE__, :Test)
+
 # PR #21709
 @deprecate cov(x::AbstractVector, corrected::Bool) cov(x, corrected=corrected)
 @deprecate cov(x::AbstractMatrix, vardim::Int, corrected::Bool) cov(x, vardim, corrected=corrected)
@@ -1727,19 +1676,12 @@ import .LinAlg: diagm
 @deprecate_binding φ          MathConstants.φ
 @deprecate_binding golden     MathConstants.golden
 
-# deprecate writecsv
-@deprecate writecsv(io, a; opts...) writedlm(io, a, ','; opts...)
-
 # PR #23271
 function IOContext(io::IO; kws...)
     depwarn("IOContext(io, k=v, ...) is deprecated, use IOContext(io, :k => v, ...) instead.", :IOContext)
     IOContext(io, (k=>v for (k, v) in kws)...)
 end
 
-# deprecate readcsv
-@deprecate readcsv(io; opts...) readdlm(io, ','; opts...)
-@deprecate readcsv(io, T::Type; opts...) readdlm(io, ',', T; opts...)
-
 @deprecate IOContext(io::IO, key, value) IOContext(io, key=>value)
 
 # PR #23485

base/exports.jl

@@ -9,7 +9,6 @@ export
     Profile,
     Dates,
     Sys,
-    Test,
     Libc,
     Libdl,
     Mmap,
@@ -1048,7 +1047,6 @@ export
     readbytes!,
     readchomp,
     readdir,
-    readdlm,
     readline,
     readlines,
     readuntil,
@@ -1070,7 +1068,6 @@ export
     unmark,
     watch_file,
     write,
-    writedlm,
     TCPSocket,
     UDPSocket,
 

base/initdefs.jl

@@ -69,7 +69,7 @@ functionality is experimental and may break or change in Julia 1.0.
 const LOAD_PATH = Any[]
 const LOAD_CACHE_PATH = String[]
 
-function init_load_path()
+function init_load_path(JULIA_HOME = JULIA_HOME)
     vers = "v$(VERSION.major).$(VERSION.minor)"
     if haskey(ENV, "JULIA_LOAD_PATH")
         prepend!(LOAD_PATH, split(ENV["JULIA_LOAD_PATH"], @static Sys.iswindows() ? ';' : ':'))

base/io.jl

@@ -779,3 +779,26 @@ function skipchars(io::IO, pred; linecomment=nothing)
     end
     return io
 end
+
+"""
+    countlines(io::IO, eol::Char='\\n')
+
+Read `io` until the end of the stream/file and count the number of lines. To specify a file
+pass the filename as the first argument. EOL markers other than `'\\n'` are supported by
+passing them as the second argument.
+"""
+function countlines(io::IO, eol::Char='\n')
+    isascii(eol) || throw(ArgumentError("only ASCII line terminators are supported"))
+    aeol = UInt8(eol)
+    a = Vector{UInt8}(8192)
+    nl = 0
+    while !eof(io)
+        nb = readbytes!(io, a)
+        @simd for i=1:nb
+            @inbounds nl += a[i] == aeol
+        end
+    end
+    nl
+end
+
+countlines(f::AbstractString, eol::Char='\n') = open(io->countlines(io,eol), f)::Int

base/sysimg.jl

@@ -360,13 +360,10 @@ include("mmap.jl")
 import .Mmap
 
 # utilities - timing, help, edit
-include("datafmt.jl")
-using .DataFmt
 include("deepcopy.jl")
 include("interactiveutil.jl")
 include("summarysize.jl")
 include("replutil.jl")
-include("test.jl")
 include("i18n.jl")
 using .I18n
 
@@ -453,4 +450,13 @@ end # baremodule Base
 
 using Base
 
+# set up load path to be able to find stdlib packages
+Base.init_load_path(ccall(:jl_get_julia_home, Any, ()))
+
+# load some stdlib packages but don't put their names in Main
+Base.require(:DelimitedFiles)
+Base.require(:Test)
+
+empty!(LOAD_PATH)
+
 Base.isfile("userimg.jl") && Base.include(Main, "userimg.jl")

doc/make.jl

@@ -16,6 +16,15 @@ end
 
 # Documenter Setup.
 
+# make links for stdlib package docs
+if Sys.iswindows()
+    cp("../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
+    cp("../stdlib/Test/docs/src/index.md", "src/stdlib/test.md")
+else
+    symlink("../../../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
+    symlink("../../../stdlib/Test/docs/src/index.md", "src/stdlib/test.md")
+end
+
 const PAGES = [
     "Home" => "index.md",
     "Manual" => [
@@ -69,6 +78,7 @@ const PAGES = [
         "stdlib/linalg.md",
         "stdlib/constants.md",
         "stdlib/file.md",
+        "stdlib/delimitedfiles.md",
         "stdlib/io-network.md",
         "stdlib/punctuation.md",
         "stdlib/sort.md",
@@ -116,9 +126,11 @@ const PAGES = [
     ],
 ]
 
+using DelimitedFiles, Test
+
 makedocs(
     build     = joinpath(pwd(), "_build/html/en"),
-    modules   = [Base, Core, BuildSysImg],
+    modules   = [Base, Core, BuildSysImg, DelimitedFiles, Test],
     clean     = false,
     doctest   = "doctest" in ARGS,
     linkcheck = "linkcheck" in ARGS,

doc/src/stdlib/.gitignore

@@ -0,0 +1,2 @@
+delimitedfiles.md
+test.md

doc/src/stdlib/index.md

@@ -10,6 +10,7 @@
   * [Linear Algebra](@ref)
   * [Constants](@ref lib-constants)
   * [Filesystem](@ref)
+  * [Delimited Files](@ref)
   * [I/O and Network](@ref)
   * [Punctuation](@ref)
   * [Sorting and Related Functions](@ref)