Skip to content

Commit

Permalink
Improve number parsing in streams: use a PushVector to work around sl…
Browse files Browse the repository at this point in the history
…ow push! in Base (#264)

* restore special case parsing of numbers for in memory JSON

* use a PushVector to work around slow push! in Base
  • Loading branch information
KristofferC authored and TotalVerb committed Nov 10, 2018
1 parent 8d4346e commit 741cbd5
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 2 deletions.
6 changes: 4 additions & 2 deletions src/Parser.jl
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ module Parser # JSON
using Mmap
using ..Common

include("pushvector.jl")

"""
Like `isspace`, but work on bytes and includes only the four whitespace
characters defined by the JSON standard: space, tab, line feed, and carriage
Expand Down Expand Up @@ -31,9 +33,9 @@ mutable struct StreamingParserState{T <: IO} <: ParserState
io::T
cur::UInt8
used::Bool
utf8array::Vector{UInt8}
utf8array::PushVector{UInt8, Vector{UInt8}}
end
StreamingParserState(io::IO) = StreamingParserState(io, 0x00, true, UInt8[])
StreamingParserState(io::IO) = StreamingParserState(io, 0x00, true, PushVector{UInt8}())

struct ParserContext{DictType, IntType} end

Expand Down
33 changes: 33 additions & 0 deletions src/pushvector.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# This is a vector wrapper that we use as a workaround for `push!`
# being slow (it always calls into the runtime even if the underlying buffer,
# has enough space). Here we keep track of the length using an extra field
mutable struct PushVector{T, A<:AbstractVector{T}} <: AbstractVector{T}
v::A
l::Int
end

# Default length of 20 should be enough to never need to grow in most cases
PushVector{T}() where {T} = PushVector(Vector{T}(undef, 20), 0)

Base.unsafe_convert(::Type{Ptr{UInt8}}, v::PushVector) = pointer(v.v)
Base.length(v::PushVector) = v.l
Base.size(v::PushVector) = (v.l,)
@inline function Base.getindex(v::PushVector, i)
@boundscheck checkbounds(v, i)
@inbounds v.v[i]
end

function Base.push!(v::PushVector, i)
v.l += 1
if v.l > length(v.v)
resize!(v.v, v.l * 2)
end
v.v[v.l] = i
return v
end

function Base.resize!(v::PushVector, l::Integer)
# Only support shrinking for now, since that is all we need
@assert l <= v.l
v.l = l
end

0 comments on commit 741cbd5

Please sign in to comment.