From 647775f2e0c0f3a3d821c92a8fe462f805b324a5 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Thu, 29 Aug 2024 10:27:19 -0700 Subject: [PATCH 1/9] Match JSON encoding of floats --- ext/rapidjson/encoder.hh | 5 ++++- test/data/roundtrip/roundtrip24.json | 2 +- test/data/roundtrip/roundtrip27.json | 2 +- test/test_encoder_compatibility.rb | 12 ++++++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ext/rapidjson/encoder.hh b/ext/rapidjson/encoder.hh index c090bd3..52a83e2 100644 --- a/ext/rapidjson/encoder.hh +++ b/ext/rapidjson/encoder.hh @@ -122,7 +122,10 @@ class RubyObjectEncoder { } } } else { - writer.Double(f); + VALUE str = rb_funcall(v, rb_intern("to_s"), 0); + Check_Type(str, T_STRING); + encode_raw_json_str(str); + //writer.Double(f); } } diff --git a/test/data/roundtrip/roundtrip24.json b/test/data/roundtrip/roundtrip24.json index f01efb6..0c3cb3c 100644 --- a/test/data/roundtrip/roundtrip24.json +++ b/test/data/roundtrip/roundtrip24.json @@ -1 +1 @@ -[5e-324] \ No newline at end of file +[5.0e-324] \ No newline at end of file diff --git a/test/data/roundtrip/roundtrip27.json b/test/data/roundtrip/roundtrip27.json index 17ce521..672438c 100644 --- a/test/data/roundtrip/roundtrip27.json +++ b/test/data/roundtrip/roundtrip27.json @@ -1 +1 @@ -[1.7976931348623157e308] \ No newline at end of file +[1.7976931348623157e+308] \ No newline at end of file diff --git a/test/test_encoder_compatibility.rb b/test/test_encoder_compatibility.rb index 1e2b872..8ffe6d3 100644 --- a/test/test_encoder_compatibility.rb +++ b/test/test_encoder_compatibility.rb @@ -47,6 +47,18 @@ def test_encode_float assert_compat 0.0 assert_compat(-0.0) assert_compat 155.0 + + 0.upto(1023) do |e| + assert_compat(2.0 ** e) + end + end + + def test_encode_randomized_floats + 1000.times do + f = [rand(2**64)].pack("Q").unpack1("D") + next if f.nan? || f.infinite? + assert_compat(f) + end end def test_encode_hash From 0fdd97a1352262c56287636dd6bfca5f08355d86 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Thu, 29 Aug 2024 14:02:50 -0700 Subject: [PATCH 2/9] Use writer.Double where we can --- ext/rapidjson/encoder.hh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ext/rapidjson/encoder.hh b/ext/rapidjson/encoder.hh index 52a83e2..710201f 100644 --- a/ext/rapidjson/encoder.hh +++ b/ext/rapidjson/encoder.hh @@ -122,10 +122,19 @@ class RubyObjectEncoder { } } } else { - VALUE str = rb_funcall(v, rb_intern("to_s"), 0); - Check_Type(str, T_STRING); - encode_raw_json_str(str); - //writer.Double(f); + // HACK: for standard notation we can use the RapidJSON + // implementation, but for values we need scientific notation we + // will fallback to Ruby's to_s so that we match JSON's output + // exactly. + // + // Ideally we would just implement both + if (f <= -1.0e15 || f >= 1.0e15 || (f >= -0.0001 && f <= 0.0001)) { + VALUE str = rb_funcall(v, rb_intern("to_s"), 0); + Check_Type(str, T_STRING); + encode_raw_json_str(str); + } else { + writer.Double(f); + } } } From d6d428db941c32aeeb4f607ae660ead009aab9da Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Thu, 29 Aug 2024 14:50:35 -0700 Subject: [PATCH 3/9] More tests of float compatibility --- test/test_encoder_compatibility.rb | 33 ++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/test/test_encoder_compatibility.rb b/test/test_encoder_compatibility.rb index 8ffe6d3..a15553a 100644 --- a/test/test_encoder_compatibility.rb +++ b/test/test_encoder_compatibility.rb @@ -50,6 +50,7 @@ def test_encode_float 0.upto(1023) do |e| assert_compat(2.0 ** e) + assert_compat(2.0 ** -e) end end @@ -61,6 +62,24 @@ def test_encode_randomized_floats end end + def test_float_scientific_threshold + assert_implementations_equal do |json| + (1.0..).bsearch{ |x| json.dump(x).include?("e") } + end + + assert_implementations_equal do |json| + (1.0..).bsearch{ |x| json.dump(-x).include?("e") } + end + + assert_implementations_equal do |json| + (1.0..).bsearch{ |x| json.dump(1.0 / x).include?("e") } + end + + assert_implementations_equal do |json| + (1.0..).bsearch{ |x| json.dump(-1.0 / x).include?("e") } + end + end + def test_encode_hash assert_compat({}) assert_compat({ "foo" => "bar" }) @@ -187,10 +206,20 @@ def assert_compat(object) end def assert_dump_equal(object, *args) - assert_equal ::JSON.dump(object, *args), RapidJSON::JSONGem.dump(object, *args) + assert_implementations_equal do |json| + json.dump(object, *args) + end end def assert_generate_equal(object, *args) - assert_equal ::JSON.generate(object, *args), RapidJSON::JSONGem.generate(object, *args) + assert_implementations_equal do |json| + json.generate(object, *args) + end + end + + def assert_implementations_equal(&block) + expected = yield(::JSON) + actual = yield(RapidJSON::JSONGem) + assert_equal expected, actual end end From 5ad5f80840f169bedbfabda6c22c96945a8c3f4d Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Sun, 1 Sep 2024 23:04:54 -0700 Subject: [PATCH 4/9] Add a test for json_ready --- test/test_jsonchecker.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_jsonchecker.rb b/test/test_jsonchecker.rb index a4f72d7..e492e74 100644 --- a/test/test_jsonchecker.rb +++ b/test/test_jsonchecker.rb @@ -20,7 +20,9 @@ class TestJsonchecker < Minitest::Test assert_match re, ex.message else assert RapidJSON.valid_json?(original_json) - assert RapidJSON.parse(original_json) + parsed = RapidJSON.parse(original_json) + assert parsed + assert RapidJSON.json_ready?(parsed) end end end From 2370eb84a1cc722c7ddba5f4c27821b5016f3fe5 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 4 Sep 2024 00:23:05 -0700 Subject: [PATCH 5/9] More explicit rounding tests --- test/test_encoder_compatibility.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_encoder_compatibility.rb b/test/test_encoder_compatibility.rb index a15553a..31e5578 100644 --- a/test/test_encoder_compatibility.rb +++ b/test/test_encoder_compatibility.rb @@ -48,6 +48,10 @@ def test_encode_float assert_compat(-0.0) assert_compat 155.0 + # Found via random test, this is the exact representation + assert_compat 103876218730131.625 + assert_compat -169986783765216.875 + 0.upto(1023) do |e| assert_compat(2.0 ** e) assert_compat(2.0 ** -e) From 0d2af51c451f809fb4f5504ab1d2d6be9c606634 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 4 Sep 2024 00:34:04 -0700 Subject: [PATCH 6/9] Test limits --- test/test_encoder_compatibility.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_encoder_compatibility.rb b/test/test_encoder_compatibility.rb index 31e5578..7329761 100644 --- a/test/test_encoder_compatibility.rb +++ b/test/test_encoder_compatibility.rb @@ -84,6 +84,12 @@ def test_float_scientific_threshold end end + def test_encode_limits + RbConfig::LIMITS.each_value do |v| + assert_compat(v) + end + end + def test_encode_hash assert_compat({}) assert_compat({ "foo" => "bar" }) From 8e1489db271732639b08ed1fceba203bf975ed53 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 4 Sep 2024 00:36:14 -0700 Subject: [PATCH 7/9] Use Ruby's to_s for floats This matches JSON's behaviour exactly. I'll look at ways to regain the speed later. --- ext/rapidjson/encoder.hh | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/ext/rapidjson/encoder.hh b/ext/rapidjson/encoder.hh index 710201f..f963381 100644 --- a/ext/rapidjson/encoder.hh +++ b/ext/rapidjson/encoder.hh @@ -122,19 +122,12 @@ class RubyObjectEncoder { } } } else { - // HACK: for standard notation we can use the RapidJSON - // implementation, but for values we need scientific notation we - // will fallback to Ruby's to_s so that we match JSON's output - // exactly. - // - // Ideally we would just implement both - if (f <= -1.0e15 || f >= 1.0e15 || (f >= -0.0001 && f <= 0.0001)) { - VALUE str = rb_funcall(v, rb_intern("to_s"), 0); - Check_Type(str, T_STRING); - encode_raw_json_str(str); - } else { - writer.Double(f); - } + // TODO: We should avoid relying on to_s and do this conversion + // ourselves. However it's difficult to get the exact same rounding + // and truncation that Ruby uses. + VALUE str = rb_funcall(v, rb_intern("to_s"), 0); + Check_Type(str, T_STRING); + encode_raw_json_str(str); } } From 9607cdc64e9da500883bdad55ff47cf444c2ec41 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 4 Sep 2024 17:14:51 -0700 Subject: [PATCH 8/9] Fix warnings --- test/test_encoder_compatibility.rb | 2 +- test/test_parser.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_encoder_compatibility.rb b/test/test_encoder_compatibility.rb index 7329761..3aa2d37 100644 --- a/test/test_encoder_compatibility.rb +++ b/test/test_encoder_compatibility.rb @@ -50,7 +50,7 @@ def test_encode_float # Found via random test, this is the exact representation assert_compat 103876218730131.625 - assert_compat -169986783765216.875 + assert_compat(-169986783765216.875) 0.upto(1023) do |e| assert_compat(2.0 ** e) diff --git a/test/test_parser.rb b/test/test_parser.rb index 52e4170..9018458 100644 --- a/test/test_parser.rb +++ b/test/test_parser.rb @@ -102,6 +102,6 @@ def test_parse_NaN_and_Infinity_allowed assert_predicate coder.load("NaN"), :nan? assert_equal Float::INFINITY, coder.load("Inf") - assert_equal -Float::INFINITY, coder.load("-Inf") + assert_equal(-Float::INFINITY, coder.load("-Inf")) end end From 1956c241ce53c9590b38ef608b79c408668ccfc2 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 4 Sep 2024 17:35:42 -0700 Subject: [PATCH 9/9] Test allocation counts --- test/test_allocations.rb | 54 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 test/test_allocations.rb diff --git a/test/test_allocations.rb b/test/test_allocations.rb new file mode 100644 index 0000000..e583a6a --- /dev/null +++ b/test/test_allocations.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "test_helper" + +class TestAllocations < Minitest::Test + def test_integer + assert_encode_allocations 1, [1, 2, 3, 4, 5] + end + + def test_boolean + assert_encode_allocations 1, [true, false, true, false, true, false] + end + + def test_float + # FIXME: ideally would not allocate + assert_encode_allocations 6, [1.0, 2.0, 3.0, 4.0, 5.0] + end + + def test_symbol + assert_encode_allocations 1, %i[foo bar baz quux] + end + + def test_string + assert_encode_allocations 1, %w[foo bar baz quux] + end + + def test_array + assert_encode_allocations 1, [[], [], [], [], []] + end + + def test_hash + assert_encode_allocations 1, {foo: 1, bar: 2, baz: 3, quux: 4} + end + + def assert_encode_allocations(expected, data) + allocations = measure_allocations { RapidJSON.encode(data) } + assert_equal expected, allocations + end + + def measure_allocations + i = 0 + while i < 2 + before = allocations + yield + after = allocations + i += 1 + end + after - before + end + + def allocations + GC.stat(:total_allocated_objects) + end +end