Skip to content

Commit

Permalink
move Serializer to stdlib (#25628)
Browse files Browse the repository at this point in the history
  • Loading branch information
JeffBezanson authored Jan 19, 2018
1 parent 4f0b09e commit 625923f
Show file tree
Hide file tree
Showing 34 changed files with 220 additions and 194 deletions.
5 changes: 5 additions & 0 deletions base/deprecated.jl
Original file line number Diff line number Diff line change
Expand Up @@ -979,6 +979,11 @@ end
@deprecate_moved randjump "Random" true true
@deprecate_moved GLOBAL_RNG "Random" false true

@deprecate_moved serialize "Serialization" true true
@deprecate_moved deserialize "Serialization" true true
@deprecate_moved AbstractSerializer "Serialization" true true
@deprecate_moved SerializationState "Serialization" true true

# 24595
@deprecate falses(A::AbstractArray) falses(size(A))
@deprecate trues(A::AbstractArray) trues(size(A))
Expand Down
2 changes: 0 additions & 2 deletions base/dict.jl
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,6 @@ function show(io::IO, t::AbstractDict{K,V}) where V where K
end
end

abstract type AbstractSerializer end

# Dict

# These can be changed, to trade off better performance for space
Expand Down
5 changes: 0 additions & 5 deletions base/exports.jl
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ export
StackTraces,
Sys,
Libc,
Serializer,
Docs,
Markdown,
Threads,
Expand Down Expand Up @@ -83,8 +82,6 @@ export
RoundNearestTiesUp,
RoundToZero,
RoundUp,
AbstractSerializer,
SerializationState,
Set,
Some,
StepRange,
Expand Down Expand Up @@ -835,7 +832,6 @@ export
close,
connect,
countlines,
deserialize,
eachline,
eof,
fd,
Expand Down Expand Up @@ -886,7 +882,6 @@ export
seekend,
seekstart,
send,
serialize,
skip,
skipchars,
take!,
Expand Down
26 changes: 14 additions & 12 deletions base/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,7 @@ function create_expr_cache(input::String, output::String, concrete_deps::typeof(
rm(output, force=true) # Remove file if it exists
code_object = """
while !eof(STDIN)
eval(Main, deserialize(STDIN))
eval(Main, Meta.parse(chop(readuntil(STDIN, '\\0'))))
end
"""
io = open(pipeline(detach(`$(julia_cmd()) -O0
Expand All @@ -572,26 +572,28 @@ function create_expr_cache(input::String, output::String, concrete_deps::typeof(
"w", STDOUT)
in = io.in
try
serialize(in, quote
write(in, """
begin
empty!(Base.LOAD_PATH)
append!(Base.LOAD_PATH, $LOAD_PATH)
append!(Base.LOAD_PATH, $(repr(LOAD_PATH)))
empty!(Base.LOAD_CACHE_PATH)
append!(Base.LOAD_CACHE_PATH, $LOAD_CACHE_PATH)
append!(Base.LOAD_CACHE_PATH, $(repr(LOAD_CACHE_PATH)))
empty!(Base.DL_LOAD_PATH)
append!(Base.DL_LOAD_PATH, $DL_LOAD_PATH)
append!(Base.DL_LOAD_PATH, $(repr(DL_LOAD_PATH)))
empty!(Base._concrete_dependencies)
append!(Base._concrete_dependencies, $concrete_deps)
append!(Base._concrete_dependencies, $(repr(concrete_deps)))
Base._track_dependencies[] = true
end)
end\0
""")
source = source_path(nothing)
if source !== nothing
serialize(in, quote
task_local_storage()[:SOURCE_PATH] = $(source)
end)
write(in, """
task_local_storage()[:SOURCE_PATH] = $(repr(source))\0
""")
end
serialize(in, :(Base.include(Base.__toplevel__, $(abspath(input)))))
write(in, "Base.include(Base.__toplevel__, $(repr(abspath(input))))\0")
if source !== nothing
serialize(in, :(delete!(task_local_storage(), :SOURCE_PATH)))
write(in, "delete!(task_local_storage(), :SOURCE_PATH)\0")
end
close(in)
catch ex
Expand Down
11 changes: 5 additions & 6 deletions base/pkg/entry.jl
Original file line number Diff line number Diff line change
Expand Up @@ -581,8 +581,7 @@ end

function build(pkg::AbstractString, build_file::AbstractString, errfile::AbstractString)
# To isolate the build from the running Julia process, we execute each build.jl file in
# a separate process. Errors are serialized to errfile for later reporting.
# TODO: serialize the same way the load cache does, not with strings
# a separate process. Errors are written to errfile for later reporting.
LOAD_PATH = filter(x -> x isa AbstractString, Base.LOAD_PATH)
code = """
empty!(Base.LOAD_PATH)
Expand All @@ -603,8 +602,8 @@ function build(pkg::AbstractString, build_file::AbstractString, errfile::Abstrac
------------------------------------------------------------
# Build failed for \$pkg
\""" exception=err,catch_backtrace()
serialize(f, pkg)
serialize(f, err)
write(f, pkg); write(f, 0x00)
write(f, sprint(showerror, err)); write(f, 0x00)
end
end
"""
Expand Down Expand Up @@ -636,8 +635,8 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set())
mktemp() do errfile, f
build!(pkgs, seen, errfile)
while !eof(f)
pkg = deserialize(f)
err = deserialize(f)
pkg = chop(readuntil(f, '\0'))
err = chop(readuntil(f, '\0'))
errs[pkg] = err
end
end
Expand Down
40 changes: 0 additions & 40 deletions base/precompile.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1190,7 +1190,6 @@ precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Base.RefValue{UInt8}
precompile(Tuple{typeof(Core.Compiler.length), Tuple{Core.Compiler.Const, DataType, Core.Compiler.Const}})
precompile(Tuple{typeof(Core.Compiler.getindex), Tuple{Core.Compiler.Const, DataType, Core.Compiler.Const}, Int64})
precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Base.RefValue{Int32}, Int64})
precompile(Tuple{typeof(Base.Serializer.write_as_tag), Base.TCPSocket, Int32})
precompile(Tuple{typeof(Base.copyto!), Array{Any, 1}, Base.MethodList})
precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{UInt64, UInt64}, UInt64})
precompile(Tuple{typeof(Base.unique), Array{Symbol, 1}})
Expand All @@ -1203,7 +1202,6 @@ precompile(Tuple{typeof(Base.rehash!), Base.Dict{UInt64, Nothing}, Int64})
precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{UInt64, Nothing}, UInt64})
precompile(Tuple{typeof(Base._setindex!), Base.Dict{UInt64, Nothing}, Nothing, UInt64, Int64})
precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{WeakRef, Any}, TypeName})
precompile(Tuple{typeof(Base.Serializer.object_number), TypeName})
precompile(Tuple{typeof(Base.isassigned), Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}, Int64})
precompile(Tuple{typeof(Base.uvfinalize), Base.TCPSocket})
precompile(Tuple{typeof(Base.close), Base.TCPSocket})
Expand Down Expand Up @@ -1292,7 +1290,6 @@ precompile(Tuple{typeof(Base.put_unbuffered), Base.Channel{Int64}, Int64})
precompile(Tuple{typeof(Base.put!), Base.Channel{Int64}, Int64})
precompile(Tuple{typeof(Base._delete!), Base.Dict{Any, Any}, Int64})
precompile(Tuple{typeof(Base.write), Base.TCPSocket, Array{Int64, 1}})
precompile(Tuple{typeof(Base.Serializer.serialize_array_data), Base.TCPSocket, Array{Int64, 1}})
precompile(Tuple{typeof(Base.sort!), Array{Int64, 1}, Int64, Int64, Base.Sort.InsertionSortAlg, Base.Order.ForwardOrdering})
precompile(Tuple{typeof(Base.Sort.partition!), Array{Int64, 1}, Int64, Int64, Base.Order.ForwardOrdering})
precompile(Tuple{typeof(Base.sort!), Array{Int64, 1}, Int64, Int64, Base.Sort.QuickSortAlg, Base.Order.ForwardOrdering})
Expand Down Expand Up @@ -1455,43 +1452,6 @@ precompile(Tuple{typeof(Base.require), Symbol})
precompile(Tuple{typeof(Base.resize!), Array{Base.Condition, 1}, Int64})
precompile(Tuple{typeof(Base.resize!), Array{Tuple{String, Float64}, 1}, Int64})
precompile(Tuple{typeof(Base.resize!), Array{UInt8, 1}, Int32})
precompile(Tuple{typeof(Base.Serializer.deserialize_array), Base.Serializer.SerializationState{Base.PipeEndpoint}})
precompile(Tuple{typeof(Base.Serializer.deserialize), Base.PipeEndpoint})
precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Serializer.SerializationState{Base.PipeEndpoint}})
precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Serializer.SerializationState{Base.PipeEndpoint}, Type{Int64}})
precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Serializer.SerializationState{Base.PipeEndpoint}, Type{QuoteNode}})
precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Serializer.SerializationState{Base.PipeEndpoint}, Type{String}})
precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Serializer.SerializationState{Base.PipeEndpoint}, Type{UInt64}})
precompile(Tuple{typeof(Base.Serializer.deserialize_cycle), Base.Serializer.SerializationState{Base.PipeEndpoint}, Expr})
precompile(Tuple{typeof(Base.Serializer.deserialize_datatype), Base.Serializer.SerializationState{Base.PipeEndpoint}})
precompile(Tuple{typeof(Base.Serializer.deserialize_expr), Base.Serializer.SerializationState{Base.PipeEndpoint}, Int64})
precompile(Tuple{typeof(Base.Serializer.handle_deserialize), Base.Serializer.SerializationState{Base.PipeEndpoint}, Int32})
precompile(Tuple{typeof(Base.Serializer.serialize_any), Base.Serializer.SerializationState{Base.Pipe}, QuoteNode})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Array{Any, 1}})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Array{String, 1}})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Bool})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Expr})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Int64})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Module})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, QuoteNode})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, SimpleVector})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, String})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Symbol})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Tuple{Int64}})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Tuple{Symbol, UInt64}})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, Type{Any}})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, TypeName})
precompile(Tuple{typeof(Base.Serializer.serialize), Base.Serializer.SerializationState{Base.Pipe}, UInt64})
precompile(Tuple{typeof(Base.Serializer.serialize_cycle), Base.Serializer.SerializationState{Base.Pipe}, Expr})
precompile(Tuple{typeof(Base.Serializer.serialize_cycle), Base.Serializer.SerializationState{Base.Pipe}, TypeName})
precompile(Tuple{typeof(Base.Serializer.serialize_cycle), Base.Serializer.SerializationState{Base.Pipe}, UInt64})
precompile(Tuple{typeof(Base.Serializer.serialize_cycle_header), Base.Serializer.SerializationState{Base.Pipe}, QuoteNode})
precompile(Tuple{typeof(Base.Serializer.serialize_mod_names), Base.Serializer.SerializationState{Base.Pipe}, Module})
precompile(Tuple{typeof(Base.Serializer.serialize_type), Base.Serializer.SerializationState{Base.Pipe}, DataType})
precompile(Tuple{typeof(Base.Serializer.serialize_type_data), Base.Serializer.SerializationState{Base.Pipe}, Type{Any}, Bool})
precompile(Tuple{typeof(Base.Serializer.serialize_typename), Base.Serializer.SerializationState{Base.Pipe}, TypeName})
precompile(Tuple{typeof(Base.Serializer.should_send_whole_type), Base.Serializer.SerializationState{Base.Pipe}, Type{Any}})
precompile(Tuple{typeof(Base.Serializer.write_as_tag), Base.Pipe, Int32})
precompile(Tuple{typeof(Base.setindex!), Base.Dict{Symbol, Base.Condition}, Base.Condition, Symbol})
precompile(Tuple{typeof(Base._setindex!), Base.Dict{Symbol, Base.Condition}, Base.Condition, Symbol, Int64})
precompile(Tuple{typeof(Base.setindex!), Base.Dict{Symbol, UInt64}, UInt64, Symbol})
Expand Down
24 changes: 0 additions & 24 deletions base/stacktraces.jl
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ module StackTraces


import Base: hash, ==, show
import Base.Serializer: serialize, deserialize
using Base.Printf: @printf
using Base: coalesce

Expand Down Expand Up @@ -96,29 +95,6 @@ function hash(frame::StackFrame, h::UInt)
h = hash(frame.inlined, h)
end

# provide a custom serializer that skips attempting to serialize the `outer_linfo`
# which is likely to contain complex references, types, and module references
# that may not exist on the receiver end
function serialize(s::AbstractSerializer, frame::StackFrame)
Serializer.serialize_type(s, typeof(frame))
serialize(s, frame.func)
serialize(s, frame.file)
write(s.io, frame.line)
write(s.io, frame.from_c)
write(s.io, frame.inlined)
write(s.io, frame.pointer)
end

function deserialize(s::AbstractSerializer, ::Type{StackFrame})
func = deserialize(s)
file = deserialize(s)
line = read(s.io, Int)
from_c = read(s.io, Bool)
inlined = read(s.io, Bool)
pointer = read(s.io, UInt64)
return StackFrame(func, file, line, nothing, from_c, inlined, pointer)
end


"""
lookup(pointer::Union{Ptr{Cvoid}, UInt}) -> Vector{StackFrame}
Expand Down
5 changes: 2 additions & 3 deletions base/sysimg.jl
Original file line number Diff line number Diff line change
Expand Up @@ -410,9 +410,6 @@ include("Enums.jl")
using .Enums

# concurrency and parallelism
include("serialize.jl")
using .Serializer
import .Serializer: serialize, deserialize
include("channels.jl")

# utilities - timing, help, edit
Expand Down Expand Up @@ -509,6 +506,7 @@ Base.require(:Base64)
Base.require(:CRC32c)
Base.require(:Dates)
Base.require(:DelimitedFiles)
Base.require(:Serialization)
Base.require(:Distributed)
Base.require(:FileWatching)
Base.require(:Future)
Expand All @@ -533,6 +531,7 @@ Base.require(:Unicode)
@deprecate_binding Dates root_module(:Dates) true ", run `using Dates` instead"
@deprecate_binding Distributed root_module(:Distributed) true ", run `using Distributed` instead"
@deprecate_binding Random root_module(:Random) true ", run `using Random` instead"
@deprecate_binding Serializer root_module(:Serialization) true ", run `using Serialization` instead"

# PR #25249
@deprecate_binding SparseArrays root_module(:SparseArrays) true ", run `using SparseArrays` instead"
Expand Down
1 change: 0 additions & 1 deletion doc/src/base/base.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ Base.Libc
Base.Markdown
Base.Meta
Base.Pkg
Base.Serializer
Base.StackTraces
Base.Sys
Base.Threads
Expand Down
3 changes: 0 additions & 3 deletions doc/src/base/io-network.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,6 @@ Base.isreadonly
Base.iswritable
Base.isreadable
Base.isopen
Base.Serializer.serialize
Base.Serializer.deserialize
Base.Serializer.writeheader
Base.Grisu.print_shortest
Base.fd
Base.redirect_stdout
Expand Down
7 changes: 5 additions & 2 deletions stdlib/Distributed/src/Distributed.jl
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ module Distributed

# imports for extension
import Base: getindex, wait, put!, take!, fetch, isready, push!, length,
hash, ==, connect, kill, serialize, deserialize, close, showerror
hash, ==, connect, kill, close, showerror

# imports for use
using Base: Process, Semaphore, JLOptions, AnyDict, buffer_writes, wait_connected,
Expand All @@ -18,7 +18,10 @@ using Base: Process, Semaphore, JLOptions, AnyDict, buffer_writes, wait_connecte
AsyncGenerator, acquire, release, invokelatest,
shell_escape_posixly, uv_error, coalesce, notnothing

# NOTE: clusterserialize.jl imports additional symbols from Base.Serializer for use
using Serialization
import Serialization: serialize, deserialize

# NOTE: clusterserialize.jl imports additional symbols from Serialization for use

export
@spawn,
Expand Down
8 changes: 4 additions & 4 deletions stdlib/Distributed/src/clusterserialize.jl
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Base.Serializer: serialize_cycle, deserialize_cycle, writetag,
__deserialized_types__, serialize_typename, deserialize_typename,
TYPENAME_TAG, reset_state, serialize_type
using Serialization: serialize_cycle, deserialize_cycle, writetag,
__deserialized_types__, serialize_typename, deserialize_typename,
TYPENAME_TAG, reset_state, serialize_type

import Base.Serializer: object_number, lookup_object_number, remember_object
import Serialization: object_number, lookup_object_number, remember_object

mutable struct ClusterSerializer{I<:IO} <: AbstractSerializer
io::I
Expand Down
2 changes: 1 addition & 1 deletion stdlib/Distributed/src/messages.jl
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ struct JoinCompleteMsg <: AbstractMsg
end

# Avoiding serializing AbstractMsg containers results in a speedup
# of approximately 10%. Can be removed once module Serializer
# of approximately 10%. Can be removed once module Serialization
# has been suitably improved.

const msgtypes = Any[CallWaitMsg, IdentifySocketAckMsg, IdentifySocketMsg,
Expand Down
Loading

1 comment on commit 625923f

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Executing the daily benchmark build, I will reply here when finished:

@nanosoldier runbenchmarks(ALL, isdaily = true)

Please sign in to comment.