Fix backward compat for CodeInfo serialization

Here we introduce an explicit codeinfo_ver version number to track the
CodeInfo struct. Core.CodeInfo has seen multiple changes between Julia
versions and the addition of the hide_in_stacktrace flag is not possible
to detect via the types which exist in the serialized stream.

To solve this problem and to avoid backward compatibility code becoming
more convoluted and hard to maintain in the future, it seems simplest to
introduce an Int32 version flag which should be a single byte of
overhead.

An alternative which was considered was to rely on the value of the
header ser_version. However, this would need to be somehow stored with
any user-defined subtype of AbstractSerializer which would be a API
breaking change.

Author: c42f

stdlib/Serialization/src/Serialization.jl

@@ -77,7 +77,7 @@ const TAGS = Any[
 
 @assert length(TAGS) == 255
 
-const ser_version = 10 # do not make changes without bumping the version #!
+const ser_version = 11 # do not make changes without bumping the version #!
 
 const NTAGS = length(TAGS)
 
@@ -109,6 +109,7 @@ const ARRAY_TAG = findfirst(==(Array), TAGS)%Int32
 const EXPR_TAG = sertag(Expr)
 const MODULE_TAG = sertag(Module)
 const METHODINSTANCE_TAG = sertag(Core.MethodInstance)
+const CODEINFO_TAG = sertag(Core.CodeInfo)
 const METHOD_TAG = sertag(Method)
 const TASK_TAG = sertag(Task)
 const DATATYPE_TAG = sertag(DataType)
@@ -438,6 +439,41 @@ function serialize(s::AbstractSerializer, linfo::Core.MethodInstance)
     nothing
 end
 
+# For future compatibility detection, add a version specifically for CodeInfo.
+# Bump this each time CodeInfo changes, and add a deserializer for the old
+# version.
+#   1 - julia 1.0/1.1  - not stored; must be inferred
+#   2 - julia 1.2      - not stored; must be inferred
+#   3 - julia 1.3/1.4  - not stored; must be inferred
+#   4 - julia 1.5
+const _current_codeinfo_ver = Int32(4)
+
+function serialize(s::AbstractSerializer, ci::Core.CodeInfo)
+    serialize_cycle(s, ci) && return
+    writetag(s.io, CODEINFO_TAG)
+    serialize(s, _current_codeinfo_ver) # for backwards compat
+    serialize(s, ci.code)
+    serialize(s, ci.codelocs)
+    serialize(s, ci.ssavaluetypes)
+    serialize(s, ci.ssaflags)
+    serialize(s, ci.method_for_inference_limit_heuristics)
+    serialize(s, ci.linetable)
+    serialize(s, ci.slotnames)
+    serialize(s, ci.slotflags)
+    serialize(s, ci.slottypes)
+    serialize(s, ci.rettype)
+    serialize(s, ci.parent)
+    serialize(s, ci.edges)
+    serialize(s, ci.min_world)
+    serialize(s, ci.max_world)
+    serialize(s, ci.hide_in_stacktrace)
+    serialize(s, ci.inferred)
+    serialize(s, ci.inlineable)
+    serialize(s, ci.propagate_inbounds)
+    serialize(s, ci.pure)
+    nothing
+end
+
 function serialize(s::AbstractSerializer, t::Task)
     serialize_cycle(s, t) && return
     if istaskstarted(t) && !istaskdone(t)
@@ -1018,7 +1054,37 @@ end
 function deserialize(s::AbstractSerializer, ::Type{CodeInfo})
     ci = ccall(:jl_new_code_info_uninit, Ref{CodeInfo}, ())
     deserialize_cycle(s, ci)
-    ci.code = deserialize(s)::Vector{Any}
+    code_or_codeinfo_ver = deserialize(s)
+    if code_or_codeinfo_ver isa Vector{Any}
+        ci.code = code_or_codeinfo_ver
+        _deserialize_old_codeinfo!(s, ci)
+        return ci
+    end
+    codeinfo_ver = code_or_codeinfo_ver::Int32
+    @assert codeinfo_ver == _current_codeinfo_ver
+    ci.code          = deserialize(s)
+    ci.codelocs      = deserialize(s)
+    ci.ssavaluetypes = deserialize(s)
+    ci.ssaflags      = deserialize(s)
+    ci.method_for_inference_limit_heuristics = deserialize(s)
+    ci.linetable     = deserialize(s)
+    ci.slotnames     = deserialize(s)
+    ci.slotflags     = deserialize(s)
+    ci.slottypes     = deserialize(s)
+    ci.rettype       = deserialize(s)
+    ci.parent        = deserialize(s)
+    ci.edges         = deserialize(s)
+    ci.min_world     = deserialize(s)
+    ci.max_world     = deserialize(s)
+    ci.hide_in_stacktrace = deserialize(s)
+    ci.inferred      = deserialize(s)
+    ci.inlineable    = deserialize(s)
+    ci.propagate_inbounds = deserialize(s)
+    ci.pure          = deserialize(s)
+    return ci
+end
+
+function _deserialize_old_codeinfo!(s, ci)
     ci.codelocs = deserialize(s)::Vector{Int32}
     _x = deserialize(s)
     if _x isa Array || _x isa Int
@@ -1054,7 +1120,6 @@ function deserialize(s::AbstractSerializer, ::Type{CodeInfo})
             ci.max_world = reinterpret(UInt, deserialize(s))
         end
     end
-    ci.hide_in_stacktrace = deserialize(s)
     ci.inferred = deserialize(s)
     ci.inlineable = deserialize(s)
     ci.propagate_inbounds = deserialize(s)

stdlib/Serialization/test/runtests.jl

@@ -563,15 +563,55 @@ let f = tempname(), x = [rand(2,2), :x, "hello"]
     rm(f)
 end
 
-let f_data
-    # a serialized function from v1.0/v1.1
-    if Int === Int64
-        f_data = "N0pMBwQAAAA0MxMAAAAAAAAAAAEFIyM1IzYiAAAAABBYH04BBE1haW6bRCIAAAAAIgAAAABNTEy+AQIjNRUAI+AjAQAAAAAAAAAfTgEETWFpbkQBAiM1AQdSRVBMWzNdvxBTH04BBE1haW6bRAMAAAAzLAAARkYiAAAAAE7BTBsVRuIWA1YkH04BBE1haW5EAQEq4SXhFgNWJB9OAQRNYWluRJ0o4CXiFgFVKOEVAAbiAQAAAAEAAAABAAAATuIVRuA0EAEMTGluZUluZm9Ob2RlH04BBE1haW6bRB9OAQRNYWluRAECIzUBB1JFUExbM13g3xXfFeIAAAAVRuKifX5MTExMTuIp"
-    else
-        f_data = "N0pMBwAAAAA0MxMAAAAAAAAAAAEFIyM1IzYiAAAAABBYH04BBE1haW6bRCIAAAAAIgAAAABNTEy+AQIjNRUAI78jAQAAAAAAAAAfTgEETWFpbkQBAiM1AQdSRVBMWzJdvxBTH04BBE1haW6bRAMAAAAzLAAARkYiAAAAAE7BTBsVRsEWA1YkH04BBE1haW5EAQEqwCXAFgNWJB9OAQRNYWluRJ0ovyXBFgFVKMAVAAbBAQAAAAEAAAABAAAATsEVRr80EAEMTGluZUluZm9Ob2RlH04BBE1haW6bRB9OAQRNYWluRAECIzUBB1JFUExbMl2/vhW+FcEAAAAVRsGifX5MTExMTsEp"
+@testset "Closure serialization" begin
+    # Serialized versions of the closure
+    # (x,y)-> 2x + y
+    # from various versions of juila
+    let f_data
+        # From v1.0/v1.1
+        if Int === Int64
+            f_data = "N0pMBwQAAAA0MxMAAAAAAAAAAAEFIyM1IzYiAAAAABBYH04BBE1haW6bRCIAAAAAIgAAAABNTEy+AQIjNRUAI+AjAQAAAAAAAAAfTgEETWFpbkQBAiM1AQdSRVBMWzNdvxBTH04BBE1haW6bRAMAAAAzLAAARkYiAAAAAE7BTBsVRuIWA1YkH04BBE1haW5EAQEq4SXhFgNWJB9OAQRNYWluRJ0o4CXiFgFVKOEVAAbiAQAAAAEAAAABAAAATuIVRuA0EAEMTGluZUluZm9Ob2RlH04BBE1haW6bRB9OAQRNYWluRAECIzUBB1JFUExbM13g3xXfFeIAAAAVRuKifX5MTExMTuIp"
+        else
+            f_data = "N0pMBwAAAAA0MxMAAAAAAAAAAAEFIyM1IzYiAAAAABBYH04BBE1haW6bRCIAAAAAIgAAAABNTEy+AQIjNRUAI78jAQAAAAAAAAAfTgEETWFpbkQBAiM1AQdSRVBMWzJdvxBTH04BBE1haW6bRAMAAAAzLAAARkYiAAAAAE7BTBsVRsEWA1YkH04BBE1haW5EAQEqwCXAFgNWJB9OAQRNYWluRJ0ovyXBFgFVKMAVAAbBAQAAAAEAAAABAAAATsEVRr80EAEMTGluZUluZm9Ob2RlH04BBE1haW6bRB9OAQRNYWluRAECIzUBB1JFUExbMl2/vhW+FcEAAAAVRsGifX5MTExMTsEp"
+        end
+        f = deserialize(IOBuffer(base64decode(f_data)))
+        @test f(10,3) == 23
+    end
+
+    let f_data
+        # From v1.2.0
+        if Int === Int64
+            f_data = "N0pMCAQAAAA0MxMCAAAAAAAAAAEHIyMxMSMxMiIAAAAAEFgfTgEETWFpbptEIgAAAAAiAAAAAE1MTL4BAyMxMRUAI+AjAwAAAAAAAAAfTgEETWFpbkQBAyMxMQEIUkVQTFsxM12/EFMfTgEETWFpbptEAwAAADMsAABGRiELI3NlbGYjAHgAeQDBTBsVRuIWA1YkH04BBE1haW5EAQEq4SXhFgNWJB9OAQRNYWluRJ0o4CXiFgFVKOEVAAbiAQAAAAEAAAABAAAA4hXfThVG4DQQAQxMaW5lSW5mb05vZGUfTgEETWFpbptEAQMjMTEsBADg3xUAAeKifX4V4gAICE5GTuAx/////0xMTExO4ik="
+            # Found to be broken on 2020-05-06
+            @test_broken begin
+                f = deserialize(IOBuffer(base64decode(f_data)))
+                f(10,3)
+            end == 23
+        end
+    end
+
+    let f_data
+        # From v1.3 (v1.4 compatible)
+        # (x,y)-> 2x + y
+        if Int === Int64
+            f_data = "N0pMCAQAAAA0MxMAAAAAAAAAAAEFIzkjMTAiAAAAABBYH04BBE1haW6bRCIAAAAAIgAAAABNTEy+AQIjORUAI+AjAQAAAAAAAAAfTgEETWFpbkQBAiM5AQdSRVBMWzNdvxBTH04BBE1haW6bRAMAAAAzLAAARkYhCyNzZWxmIwB4AHkAwUwbFUbiFgNWJB9OAQRNYWluRAEBKuEl4RYDViQfTgEETWFpbkSdKOAl4hYBVSjhFQAG4gEAAAABAAAAAQAAAOIV304VRuA0EAEMTGluZUluZm9Ob2RlH04BBE1haW6bRAECIzkBB1JFUExbM13g3xUAAeKifX4V4gAICE5GTk4JAQAAAAAAAAAJ//////////9MTExMTuIp"
+        else
+            f_data = "N0pMCAAAAAA0MxMAAAAAAAAAAAEFIzkjMTAiAAAAABBYH04BBE1haW6bRCIAAAAAIgAAAABNTEy+AQIjORUAI78jAQAAAAAAAAAfTgEETWFpbkQBAiM5AQdSRVBMWzJdvxBTH04BBE1haW6bRAMAAAAzLAAARkYhCyNzZWxmIwB4AHkAwUwbFUbBFgNWJB9OAQRNYWluRAEBKsAlwBYDViQfTgEETWFpbkSdKL8lwRYBVSjAFQAGwQEAAAABAAAAAQAAAMEVvk4VRr80EAEMTGluZUluZm9Ob2RlH04BBE1haW6bRAECIzkBB1JFUExbMl2/vhUAAcGifX4VwQAICE5GTk4HAQAAAAf/////TExMTE7BKQ=="
+        end
+        @test begin
+            f = deserialize(IOBuffer(base64decode(f_data)))
+            f(10,3)
+        end == 23
+    end
+
+    let f_data
+        # From v1.5-DEV (codeinfo_ver = 4)
+        if Int === Int64
+            f_data = "N0pMCwQAAAA0MxMAAAAAAAAAAAEEIzcjOCIAAAAAEFgfTptEIgAAAAAiAAAAAE1MTL4BAiM3FQAj4CMBAAAAAAAAAB9OAQRNYWluRAECIzcBB1JFUExbMl2/EFMfTptEAwAAADMsAABGRjkhCyNzZWxmIwB4AHkAwUwbwhVG4hYDViQfTgEETWFpbkQBASrhJeEWA1YkH04BBE1haW5EnSjgJeIWAVUo4RUABuIBAAAAAQAAAAEAAADiFd9OFUbgNBABDExpbmVJbmZvTm9kZR9Om0QBAiM3AQdSRVBMWzJd4N8VAAHion1+FeIACAhORk5OCQEAAAAAAAAACf//////////TExMTExO4ik="
+            f = deserialize(IOBuffer(base64decode(f_data)))
+            @test f(10,3) == 23
+        end
     end
-    f = deserialize(IOBuffer(base64decode(f_data)))
-    @test f(10,3) == 23
 end
 
 # issue #33466, IdDict