Merge pull request #11280 from JuliaLang/jq/mmap

RFC: mmap makeover

Author: quinnj

base/datafmt.jl

@@ -48,7 +48,7 @@ end
 
 function as_mmap(fname::AbstractString, fsz::Int64)
     open(fname) do io
-        mmap_array(UInt8, (Int(fsz),), io)
+        Mmap.mmap(io, Vector{UInt8}, (Int(fsz),))
     end
 end
 

base/deprecated.jl

@@ -549,3 +549,89 @@ macro math_const(sym, val, def)
 end
 
 export MathConst, @math_const
+
+# 11280, mmap
+
+export msync
+msync{T}(A::Array{T}) = msync(pointer(A), length(A)*sizeof(T))
+msync(B::BitArray) = msync(pointer(B.chunks), length(B.chunks)*sizeof(UInt64))
+
+@unix_only begin
+
+function mmap(len::Integer, prot::Integer, flags::Integer, fd, offset::Integer)
+    depwarn("`mmap` is deprecated, use `mmap(io, Array{T,N}, dims, offset)` instead to return an mmapped-array", :mmap)
+    const pagesize::Int = ccall(:jl_getpagesize, Clong, ())
+    # Check that none of the computations will overflow
+    if len < 0
+        throw(ArgumentError("requested size must be ≥ 0, got $len"))
+    end
+    if len > typemax(Int)-pagesize
+        throw(ArgumentError("requested size must be ≤ $(typemax(Int)-pagesize), got $len"))
+    end
+    # Set the offset to a page boundary
+    offset_page::FileOffset = floor(Integer,offset/pagesize)*pagesize
+    len_page::Int = (offset-offset_page) + len
+    # Mmap the file
+    p = ccall(:jl_mmap, Ptr{Void}, (Ptr{Void}, Csize_t, Cint, Cint, Cint, FileOffset), C_NULL, len_page, prot, flags, fd, offset_page)
+    systemerror("memory mapping failed", reinterpret(Int,p) == -1)
+    # Also return a pointer that compensates for any adjustment in the offset
+    return p, Int(offset-offset_page)
+end
+
+function munmap(p::Ptr,len::Integer)
+    depwarn("`munmap` is deprecated, `mmap` Arrays are automatically munmapped when finalized", :munmap)
+    systemerror("munmap", ccall(:munmap,Cint,(Ptr{Void},Int),p,len) != 0)
+end
+
+const MS_ASYNC = 1
+const MS_INVALIDATE = 2
+const MS_SYNC = 4
+function msync(p::Ptr, len::Integer, flags::Integer=MS_SYNC)
+    depwarn("`msync` is deprecated, use `Mmap.sync!(array)` instead", :msync)
+    systemerror("msync", ccall(:msync, Cint, (Ptr{Void}, Csize_t, Cint), p, len, flags) != 0)
+end
+end
+
+
+@windows_only begin
+function munmap(viewhandle::Ptr, mmaphandle::Ptr)
+    depwarn("`munmap` is deprecated, `mmap` Arrays are automatically munmapped when finalized", :munmap)
+    status = ccall(:UnmapViewOfFile, stdcall, Cint, (Ptr{Void},), viewhandle)!=0
+    status |= ccall(:CloseHandle, stdcall, Cint, (Ptr{Void},), mmaphandle)!=0
+    if !status
+        error("could not unmap view: $(FormatMessage())")
+    end
+end
+
+function msync(p::Ptr, len::Integer)
+    depwarn("`msync` is deprecated, use `Mmap.sync!(array)` instead", :msync)
+    status = ccall(:FlushViewOfFile, stdcall, Cint, (Ptr{Void}, Csize_t), p, len)!=0
+    if !status
+        error("could not msync: $(FormatMessage())")
+    end
+end
+
+end
+
+@unix_only @deprecate mmap_array{T,N}(::Type{T}, dims::NTuple{N,Integer}, s::IO, offset=position(s)) mmap(s, Array{T,N}, dims, offset)
+
+@windows_only begin
+type SharedMemSpec
+    name :: AbstractString
+    readonly :: Bool
+    create :: Bool
+end
+export mmap_array
+function mmap_array{T,N}(::Type{T}, dims::NTuple{N,Integer}, s::Union(IO,SharedMemSpec), offset::FileOffset)
+    depwarn("`mmap_array` is deprecated, use `mmap(io, Array{T,N}, dims, offset)` instead to return an mmapped-array", :mmap_array)
+    if isa(s,SharedMemSpec)
+        a = Mmap.AnonymousMmap(s.name, s.readonly, s.create)
+    else
+        a = s
+    end
+    return mmap(a, Array{T,N}, dims, offset)
+end
+end
+
+@deprecate mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Integer}, s::IOStream, offset::FileOffset=position(s)) mmap(s, BitArray, dims, offset)
+@deprecate mmap_bitarray{N}(dims::NTuple{N,Integer}, s::IOStream, offset=position(s)) mmap(s, BitArray, dims, offset)

base/exports.jl

@@ -13,6 +13,7 @@ export
     Test,
     Libc,
     Libdl,
+    Mmap,
     LinAlg,
     BLAS,
     LAPACK,
@@ -1142,9 +1143,6 @@ export
     listenany,
     ltoh,
     mark,
-    mmap_array,
-    mmap_bitarray,
-    msync,
     nb_available,
     ntoh,
     open,

base/libc.jl

@@ -3,8 +3,7 @@
 module Libc
 
 export FILE, TmStruct, strftime, strptime, getpid, gethostname, free, malloc, calloc, realloc,
-    errno, strerror, flush_cstdio, systemsleep, time,
-    MS_ASYNC, MS_INVALIDATE, MS_SYNC, mmap, munmap, msync
+    errno, strerror, flush_cstdio, systemsleep, time
 
 include("errno.jl")
 
@@ -163,64 +162,4 @@ malloc(size::Integer) = ccall(:malloc, Ptr{Void}, (Csize_t,), size)
 realloc(p::Ptr, size::Integer) = ccall(:realloc, Ptr{Void}, (Ptr{Void}, Csize_t), p, size)
 calloc(num::Integer, size::Integer) = ccall(:calloc, Ptr{Void}, (Csize_t, Csize_t), num, size)
 
-## mmap ##
-
-msync{T}(A::Array{T}) = msync(pointer(A), length(A)*sizeof(T))
-
-msync(B::BitArray) = msync(pointer(B.chunks), length(B.chunks)*sizeof(UInt64))
-
-@unix_only begin
-# Low-level routines
-# These are needed for things like MAP_ANONYMOUS
-function mmap(len::Integer, prot::Integer, flags::Integer, fd, offset::Integer)
-    const pagesize::Int = ccall(:jl_getpagesize, Clong, ())
-    # Check that none of the computations will overflow
-    if len < 0
-        throw(ArgumentError("requested size must be ≥ 0, got $len"))
-    end
-    if len > typemax(Int)-pagesize
-        throw(ArgumentError("requested size must be ≤ $(typemax(Int)-pagesize), got $len"))
-    end
-    # Set the offset to a page boundary
-    offset_page::FileOffset = floor(Integer,offset/pagesize)*pagesize
-    len_page::Int = (offset-offset_page) + len
-    # Mmap the file
-    p = ccall(:jl_mmap, Ptr{Void}, (Ptr{Void}, Csize_t, Cint, Cint, Cint, FileOffset), C_NULL, len_page, prot, flags, fd, offset_page)
-    systemerror("memory mapping failed", reinterpret(Int,p) == -1)
-    # Also return a pointer that compensates for any adjustment in the offset
-    return p, Int(offset-offset_page)
-end
-
-function munmap(p::Ptr,len::Integer)
-    systemerror("munmap", ccall(:munmap,Cint,(Ptr{Void},Int),p,len) != 0)
-end
-
-const MS_ASYNC = 1
-const MS_INVALIDATE = 2
-const MS_SYNC = 4
-function msync(p::Ptr, len::Integer, flags::Integer)
-    systemerror("msync", ccall(:msync, Cint, (Ptr{Void}, Csize_t, Cint), p, len, flags) != 0)
-end
-msync(p::Ptr, len::Integer) = msync(p, len, MS_SYNC)
-end
-
-
-@windows_only begin
-function munmap(viewhandle::Ptr, mmaphandle::Ptr)
-    status = ccall(:UnmapViewOfFile, stdcall, Cint, (Ptr{Void},), viewhandle)!=0
-    status |= ccall(:CloseHandle, stdcall, Cint, (Ptr{Void},), mmaphandle)!=0
-    if !status
-        error("could not unmap view: $(FormatMessage())")
-    end
-end
-
-function msync(p::Ptr, len::Integer)
-    status = ccall(:FlushViewOfFile, stdcall, Cint, (Ptr{Void}, Csize_t), p, len)!=0
-    if !status
-        error("could not msync: $(FormatMessage())")
-    end
-end
-
-end
-
 end # module