From f9852f98fcc21ca0441472c3bee391969b30866b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 10 Jul 2015 17:20:56 -0600 Subject: [PATCH 1/5] Adding single quote option and two style of JSON string macros. --- src/JSON.jl | 12 ++++++++---- src/Parser.jl | 43 +++++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/JSON.jl b/src/JSON.jl index 48f1c6a..0ea86f6 100644 --- a/src/JSON.jl +++ b/src/JSON.jl @@ -2,7 +2,7 @@ module JSON using Compat -export json # returns a compact (or indented) JSON representation as a String +export json, @J_str, @JSON_str, @JSON_ORDERED_str # returns a compact (or indented) JSON representation as a String include("Parser.jl") @@ -17,8 +17,8 @@ type State{I} indentlen::Int prefix::AbstractString otype::Array{Bool, 1} - State(indentstep::Int) = new(indentstep, - 0, + State(indentstep::Int) = new(indentstep, + 0, "", Bool[]) end @@ -282,5 +282,9 @@ function parsefile(filename::AbstractString; ordered::Bool=false, use_mmap=true) end end -end # module +# Macros +macro JSON_str(arg::AbstractString) parse(arg) end +macro JSON_ORDERED_str(arg::AbstractString) parse(arg, ordered=true) end +macro J_str(arg::AbstractString) parse(arg, quote_char='\'') end # convenience intented for short single quoted json data +end # module diff --git a/src/Parser.jl b/src/Parser.jl index 480cdd7..9824c8b 100644 --- a/src/Parser.jl +++ b/src/Parser.jl @@ -21,8 +21,10 @@ type ParserState{T<:AbstractString} s::Int e::Int end + ParserState(str::AbstractString,s::Int,e::Int) = ParserState(str, s, e) + charat{T<:AbstractString}(ps::ParserState{T}) = ps.str[ps.s] incr(ps::ParserState) = (ps.s += 1) hasmore(ps::ParserState) = (ps.s < ps.e) @@ -77,13 +79,13 @@ end # PARSING -function parse_array{T<:AbstractString}(ps::ParserState{T}, ordered::Bool) +function parse_array{T<:AbstractString}(ps::ParserState{T}, ordered::Bool, quote_char::Char) incr(ps) # Skip over the '[' _array = TYPES[] chomp_space(ps) charat(ps)==']' && (incr(ps); return _array) # Check for empty array while true # Extract values from array - v = parse_value(ps, ordered) # Extract value + v = parse_value(ps, ordered, quote_char) # Extract value push!(_array, v) # Eat up trailing whitespace chomp_space(ps) @@ -101,23 +103,23 @@ function parse_array{T<:AbstractString}(ps::ParserState{T}, ordered::Bool) return _array end -function parse_object{T<:AbstractString}(ps::ParserState{T}, ordered::Bool) +function parse_object{T<:AbstractString}(ps::ParserState{T}, ordered::Bool, quote_char::Char) if ordered - parse_object(ps, ordered, OrderedDict{KEY_TYPES,TYPES}()) + parse_object(ps, ordered, quote_char, OrderedDict{KEY_TYPES,TYPES}()) else - parse_object(ps, ordered, Dict{KEY_TYPES,TYPES}()) + parse_object(ps, ordered, quote_char, Dict{KEY_TYPES,TYPES}()) end end -function parse_object{T<:AbstractString}(ps::ParserState{T}, ordered::Bool, obj) +function parse_object{T<:AbstractString}(ps::ParserState{T}, ordered::Bool, quote_char::Char, obj) incr(ps) # Skip over opening '{' chomp_space(ps) charat(ps)=='}' && (incr(ps); return obj) # Check for empty object while true chomp_space(ps) - _key = parse_string(ps) # Key + _key = parse_string(ps, quote_char) # Key skip_separator(ps) - _value = parse_value(ps, ordered) # Value + _value = parse_value(ps, ordered, quote_char) # Value obj[_key] = _value # Building object chomp_space(ps) c = charat(ps) # Find the next pair or end of object @@ -139,13 +141,13 @@ utf16_get_supplementary(lead::Uint16, trail::Uint16) = @compat(Char(@compat(UInt # TODO: Try to find ways to improve the performance of this (currently one # of the slowest parsing methods). -function parse_string{T<:AbstractString}(ps::ParserState{T}) +function parse_string{T<:AbstractString}(ps::ParserState{T}, quote_char::Char) str = ps.str s = ps.s e = ps.e - str[s]=='"' || _error("Missing opening string char", ps) - s = nextind(str, s) # Skip over opening '"' + str[s] == quote_char || _error("Missing opening string char", ps) + s = nextind(str, s) # Skip over opening quote_char '"' b = IOBuffer() found_end = false while s <= e @@ -180,9 +182,10 @@ function parse_string{T<:AbstractString}(ps::ParserState{T}) elseif c == 'n' write(b, '\n') elseif c == 'r' write(b, '\r') elseif c == 't' write(b, '\t') + elseif c == '\'' && quote_char == '\'' write(b, '\'') # not part of standard else _error("Unrecognized escaped character: " * string(c), ps) end - elseif c == '"' + elseif c == quote_char found_end = true s = nextind(str, s) break @@ -213,19 +216,19 @@ function parse_simple{T<:AbstractString}(ps::ParserState{T}) ret end -function parse_value{T<:AbstractString}(ps::ParserState{T}, ordered::Bool) +function parse_value{T<:AbstractString}(ps::ParserState{T}, ordered::Bool, quote_char::Char) chomp_space(ps) (ps.s > ps.e) && return nothing # Nothing left ch = charat(ps) - if ch == '"' - ret = parse_string(ps) + if ch == quote_char + ret = parse_string(ps, quote_char) elseif ch == '{' - ret = parse_object(ps, ordered) + ret = parse_object(ps, ordered, quote_char) elseif (ch >= '0' && ch <= '9') || ch=='-' || ch=='+' ret = parse_number(ps) elseif ch == '[' - ret = parse_array(ps, ordered) + ret = parse_array(ps, ordered, quote_char) elseif ch == 'f' || ch == 't' || ch == 'n' ret = parse_simple(ps) else @@ -315,12 +318,12 @@ function parse_number{T<:AbstractString}(ps::ParserState{T}) end end -function parse(str::AbstractString; ordered::Bool=false) +function parse(str::AbstractString; ordered::Bool=false, quote_char::Char='"') pos::Int = 1 len::Int = endof(str) len < 1 && return - ordered && !_HAVE_DATASTRUCTURES && error("DataStructures package required for ordered parsing: try `Pkg.add(\"DataStructures\")`") - parse_value(ParserState(str, pos, len), ordered) + ordered && !_HAVE_DATASTRUCTURES && error("DataStructures package required for ordered parsing: try `Pkg.add(\"DataStructures\")`") + parse_value(ParserState(str, pos, len), ordered, quote_char) end end #module Parser From ab45f074515e76f3e5e4bef707ca6cc2a2532e16 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 10 Jul 2015 18:06:12 -0600 Subject: [PATCH 2/5] Updating tests and documentation (tested the code in the readme). --- README.md | 18 ++++++++++++++++++ src/JSON.jl | 8 ++++---- src/Parser.jl | 3 ++- test/runtests.jl | 18 ++++++++++++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 04e666d..92330a2 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,24 @@ JSON.json(j) # "{\"an_array\":[\"string\",9],\"a_number\":5.0}" ``` +### Macro Strings + +`JSON` and `J` can be used to embed JSON data directly into source code: + +```julia +basic_dict = JSON""" +{ + "a_number" : 5.0, + "an_array" : ["string", 9] +} +""" +``` + + +```julia +basic_dict = J"{'a_number' : 5.0, 'an_array' : ['string', 9]}" +``` + ## Documentation ```julia diff --git a/src/JSON.jl b/src/JSON.jl index 0ea86f6..50b0cbd 100644 --- a/src/JSON.jl +++ b/src/JSON.jl @@ -2,7 +2,7 @@ module JSON using Compat -export json, @J_str, @JSON_str, @JSON_ORDERED_str # returns a compact (or indented) JSON representation as a String +export json, @J_str, @JSON_mstr, @JSON_ORDERED_mstr # returns a compact (or indented) JSON representation as a String include("Parser.jl") @@ -283,8 +283,8 @@ function parsefile(filename::AbstractString; ordered::Bool=false, use_mmap=true) end # Macros -macro JSON_str(arg::AbstractString) parse(arg) end -macro JSON_ORDERED_str(arg::AbstractString) parse(arg, ordered=true) end -macro J_str(arg::AbstractString) parse(arg, quote_char='\'') end # convenience intented for short single quoted json data +macro JSON_mstr(arg::AbstractString) parse(arg) end +macro JSON_ORDERED_mstr(arg::AbstractString) parse(arg, ordered=true) end +macro J_str(arg::AbstractString) parse(arg, single_quote=true) end # convenience intented for short single quoted json data end # module diff --git a/src/Parser.jl b/src/Parser.jl index 9824c8b..98a3868 100644 --- a/src/Parser.jl +++ b/src/Parser.jl @@ -318,9 +318,10 @@ function parse_number{T<:AbstractString}(ps::ParserState{T}) end end -function parse(str::AbstractString; ordered::Bool=false, quote_char::Char='"') +function parse(str::AbstractString; ordered::Bool=false, single_quote::Bool=false) pos::Int = 1 len::Int = endof(str) + quote_char::Char = single_quote ? '\'' : '\"' len < 1 && return ordered && !_HAVE_DATASTRUCTURES && error("DataStructures package required for ordered parsing: try `Pkg.add(\"DataStructures\")`") parse_value(ParserState(str, pos, len), ordered, quote_char) diff --git a/test/runtests.jl b/test/runtests.jl index b20f411..da16b9f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -235,3 +235,21 @@ let iob = IOBuffer() JSON.print(iob, t109(1)) @test get(JSON.parse(takebuf_string(iob)), "i", 0) == 1 end + +# Tests for Macro strings + +let mstr_test = @compat Dict("a" => 1) + @test mstr_test == JSON"""{"a":1}""" + @test mstr_test == J"{'a':1}" + @test JSON_ORDERED"""{"x": 3}""" == DataStructures.OrderedDict{String,Any}([("x",3)]) + + @test (@compat Dict("a_number" => 5, "an_array" => ["string"; 9]) ) == J"{'a_number' : 5, 'an_array' : ['string', 9] }" + + @test JSON""" + { + "a_number" : 5.0, + "an_array" : ["string", 9] + } + """ == (@compat Dict("a_number" => 5, "an_array" => ["string"; 9]) ) + +end From e6376d50f2858880a8b96a8f9c338d327a11b25b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 10 Jul 2015 18:28:39 -0600 Subject: [PATCH 3/5] Update README.md Documentation updates. --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 92330a2..4c38fee 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ basic_dict = JSON""" basic_dict = J"{'a_number' : 5.0, 'an_array' : ['string', 9]}" ``` +Note that the shorter `@J_str` relies on the `single_quote` parsing behavior. + ## Documentation ```julia @@ -68,9 +70,9 @@ json(a::Any) Returns a compact JSON representation as a String. ```julia -JSON.parse(s::String; ordered=false) -JSON.parse(io::IO; ordered=false) -JSON.parsefile(filename::String; ordered=false, use_mmap=true) +JSON.parse(s::String; ordered=false, single_quote=false) +JSON.parse(io::IO; ordered=false, single_quote=false) +JSON.parsefile(filename::String; ordered=false, single_quote=false, use_mmap=true) ``` Parses a JSON String or IO stream into a nested Array or Dict. @@ -79,4 +81,6 @@ If `ordered=true` is specified, JSON objects are parsed into `OrderedDicts`, which maintains the insertion order of the items in the object. (*) +Setting `single_quote` enables parsing on non-standard usage of single quote `'\''` for string values. + (*) Requires the `DataStructures.jl` package to be installed. From 05cb6baadebcf4046ea9b0f334d7a5196d22437b Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sat, 11 Jul 2015 15:27:14 -0600 Subject: [PATCH 4/5] Renaming macros. --- src/JSON.jl | 18 +++++++++--------- src/Parser.jl | 13 ++++++++++++- test/runtests.jl | 10 +++++----- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/JSON.jl b/src/JSON.jl index 50b0cbd..2b896d8 100644 --- a/src/JSON.jl +++ b/src/JSON.jl @@ -2,11 +2,12 @@ module JSON using Compat -export json, @J_str, @JSON_mstr, @JSON_ORDERED_mstr # returns a compact (or indented) JSON representation as a String +export json, @json_str, @json_mstr # returns a compact (or indented) JSON representation as a String include("Parser.jl") import .Parser.parse +import .Parser._parse const INDENT=true const NOINDENT=false @@ -246,7 +247,7 @@ function consumeString(io::IO, obj::IOBuffer) throw(EOFError()) end -function parse(io::IO; ordered::Bool=false) +function parse(io::IO; ordered::Bool=false, single_quote::Bool=false) open_bracket = close_bracket = nothing try open_bracket, close_bracket = determine_bracket_type(io) @@ -271,20 +272,19 @@ function parse(io::IO; ordered::Bool=false) consumeString(io, obj) end end - JSON.parse(takebuf_string(obj), ordered=ordered) + JSON.parse(takebuf_string(obj), ordered=ordered, single_quote=single_quote) end -function parsefile(filename::AbstractString; ordered::Bool=false, use_mmap=true) +function parsefile(filename::AbstractString; ordered::Bool=false, single_quote::Bool=false, use_mmap=true) sz = filesize(filename) open(filename) do io s = use_mmap ? UTF8String(mmap_array(Uint8, (sz,), io)) : readall(io) - JSON.parse(s, ordered=ordered) + JSON.parse(s, ordered=ordered, single_quote=single_quote) end end -# Macros -macro JSON_mstr(arg::AbstractString) parse(arg) end -macro JSON_ORDERED_mstr(arg::AbstractString) parse(arg, ordered=true) end -macro J_str(arg::AbstractString) parse(arg, single_quote=true) end # convenience intented for short single quoted json data +# Macros +macro json_mstr(arg::AbstractString, opts...) _parse(arg, opts) end +macro json_str(arg::AbstractString, opts...) _parse(arg, opts) end end # module diff --git a/src/Parser.jl b/src/Parser.jl index 98a3868..6486aa2 100644 --- a/src/Parser.jl +++ b/src/Parser.jl @@ -322,9 +322,20 @@ function parse(str::AbstractString; ordered::Bool=false, single_quote::Bool=fals pos::Int = 1 len::Int = endof(str) quote_char::Char = single_quote ? '\'' : '\"' - len < 1 && return + len < 1 && return ordered && !_HAVE_DATASTRUCTURES && error("DataStructures package required for ordered parsing: try `Pkg.add(\"DataStructures\")`") parse_value(ParserState(str, pos, len), ordered, quote_char) end +# Allow post-fix options in the forms: "so!" or "single_ordered" or "os!", etc. +function _parse(str::AbstractString, opt::Tuple) + if opt |> isempty + parse(str) + else + opts = opt[1][end] != '!' ? split(opt[1], "_") : split(opt[1], "") + opts = [ c[1] for c in opts ] + parse(str, ordered ='o' in opts, single_quote = 's' in opts) + end +end + end #module Parser diff --git a/test/runtests.jl b/test/runtests.jl index da16b9f..c6fb9ad 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -239,13 +239,13 @@ end # Tests for Macro strings let mstr_test = @compat Dict("a" => 1) - @test mstr_test == JSON"""{"a":1}""" - @test mstr_test == J"{'a':1}" - @test JSON_ORDERED"""{"x": 3}""" == DataStructures.OrderedDict{String,Any}([("x",3)]) + @test mstr_test == json"""{"a":1}""" + @test mstr_test == json"{'a':1}"single + @test json"""{"x": 3}"""ordered == DataStructures.OrderedDict{String,Any}([("x",3)]) - @test (@compat Dict("a_number" => 5, "an_array" => ["string"; 9]) ) == J"{'a_number' : 5, 'an_array' : ['string', 9] }" + @test (@compat Dict("a_number" => 5, "an_array" => ["string"; 9]) ) == json"{'a_number' : 5, 'an_array' : ['string', 9] }"s! - @test JSON""" + @test json""" { "a_number" : 5.0, "an_array" : ["string", 9] From 739a14fd83fc4171f8355301813c0fc1df97f862 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 20 Jul 2015 17:12:59 -0600 Subject: [PATCH 5/5] Added NaN option. --- src/Parser.jl | 16 ++++++++++++---- test/runtests.jl | 8 ++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Parser.jl b/src/Parser.jl index 98a3868..8bde6a5 100644 --- a/src/Parser.jl +++ b/src/Parser.jl @@ -219,14 +219,15 @@ end function parse_value{T<:AbstractString}(ps::ParserState{T}, ordered::Bool, quote_char::Char) chomp_space(ps) (ps.s > ps.e) && return nothing # Nothing left - + leniant = true + ch = charat(ps) if ch == quote_char ret = parse_string(ps, quote_char) elseif ch == '{' ret = parse_object(ps, ordered, quote_char) - elseif (ch >= '0' && ch <= '9') || ch=='-' || ch=='+' - ret = parse_number(ps) + elseif (ch >= '0' && ch <= '9') || ch=='-' || ch=='+' || (leniant && ch=='I' || ch=='N') + ret = parse_number(ps, true) elseif ch == '[' ret = parse_array(ps, ordered, quote_char) elseif ch == 'f' || ch == 't' || ch == 'n' @@ -248,7 +249,7 @@ if VERSION < v"0.4.0-dev+3874" end end -function parse_number{T<:AbstractString}(ps::ParserState{T}) +function parse_number{T<:AbstractString}(ps::ParserState{T}, leniant::Bool) str = ps.str p = ps.s e = ps.e @@ -259,6 +260,13 @@ function parse_number{T<:AbstractString}(ps::ParserState{T}) p += 1 (p <= e) ? (c = str[p]) : _error("Unrecognized number", ps) # Something must follow a sign end + + if leniant && (str[p]=='I' || str[p]=='N') + vs = SubString(ps.str, ps.s, p+2) + @show(vs) + ps.s = p+3 + return Base.parse(Float64, vs) + end if c == '0' # If number begins with 0, it must be int(0) or a floating point p += 1 diff --git a/test/runtests.jl b/test/runtests.jl index da16b9f..df4d479 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -253,3 +253,11 @@ let mstr_test = @compat Dict("a" => 1) """ == (@compat Dict("a_number" => 5, "an_array" => ["string"; 9]) ) end + +# Test Leniant +let tmp = JSON.parse("{\"a\": NaN}") + @test isnan(tmp["a"]) == true +end + + +