diff --git a/ChangeLog.rdoc b/ChangeLog.rdoc index 39604dc..a7d3f79 100644 --- a/ChangeLog.rdoc +++ b/ChangeLog.rdoc @@ -7,6 +7,7 @@ * Allow for nested tracing. * Skip :until_valid is now fast for :asserted_value. * Added Section - a way to transform the data stream. +* Added transforms for brotli, lz4, xor, zlib, zstd. == Version 2.4.15 (2023-02-07) diff --git a/lib/bindata/transform/brotli.rb b/lib/bindata/transform/brotli.rb new file mode 100755 index 0000000..9a9396f --- /dev/null +++ b/lib/bindata/transform/brotli.rb @@ -0,0 +1,35 @@ +require 'brotli' + +module BinData + module Transform + # Transforms a brotli compressed data stream. + # + # gem install brotli + class Brotli < BinData::IO::Transform + transform_changes_stream_length! + + def initialize(read_length) + super() + @length = read_length + end + + def read(n) + @read ||= ::Brotli::inflate(chain_read(@length)) + @read.slice!(0...n) + end + + def write(data) + @write ||= create_empty_binary_string + @write << data + end + + def after_read_transform + raise IOError, "didn't read all data" unless @read.empty? + end + + def after_write_transform + chain_write(::Brotli::deflate(@write)) + end + end + end +end diff --git a/lib/bindata/transform/lz4.rb b/lib/bindata/transform/lz4.rb new file mode 100755 index 0000000..227343e --- /dev/null +++ b/lib/bindata/transform/lz4.rb @@ -0,0 +1,35 @@ +require 'extlz4' + +module BinData + module Transform + # Transforms a LZ4 compressed data stream. + # + # gem install extlz4 + class LZ4 < BinData::IO::Transform + transform_changes_stream_length! + + def initialize(read_length) + super() + @length = read_length + end + + def read(n) + @read ||= ::LZ4::decode(chain_read(@length)) + @read.slice!(0...n) + end + + def write(data) + @write ||= create_empty_binary_string + @write << data + end + + def after_read_transform + raise IOError, "didn't read all data" unless @read.empty? + end + + def after_write_transform + chain_write(::LZ4::encode(@write)) + end + end + end +end diff --git a/lib/bindata/transform/lzma.rb b/lib/bindata/transform/lzma.rb new file mode 100755 index 0000000..332a4fe --- /dev/null +++ b/lib/bindata/transform/lzma.rb @@ -0,0 +1,35 @@ +require 'xz' + +module BinData + module Transform + # Transforms a lzma compressed data stream. + # + # gem install ruby-xz + class Lzma < BinData::IO::Transform + transform_changes_stream_length! + + def initialize(read_length) + super() + @length = read_length + end + + def read(n) + @read ||= ::XZ::decompress(chain_read(@length)) + @read.slice!(0...n) + end + + def write(data) + @write ||= create_empty_binary_string + @write << data + end + + def after_read_transform + raise IOError, "didn't read all data" unless @read.empty? + end + + def after_write_transform + chain_write(::XZ::compress(@write)) + end + end + end +end diff --git a/lib/bindata/transform/xor.rb b/lib/bindata/transform/xor.rb new file mode 100755 index 0000000..c67d046 --- /dev/null +++ b/lib/bindata/transform/xor.rb @@ -0,0 +1,19 @@ +module BinData + module Transform + # Transforms the data stream by xoring each byte. + class Xor < BinData::IO::Transform + def initialize(xor) + super() + @xor = xor + end + + def read(n) + chain_read(n).bytes.map { |byte| (byte ^ @xor).chr }.join + end + + def write(data) + chain_write(data.bytes.map { |byte| (byte ^ @xor).chr }.join) + end + end + end +end diff --git a/lib/bindata/transform/xz.rb b/lib/bindata/transform/xz.rb new file mode 100755 index 0000000..c828839 --- /dev/null +++ b/lib/bindata/transform/xz.rb @@ -0,0 +1,35 @@ +require 'xz' + +module BinData + module Transform + # Transforms a xz compressed data stream. + # + # gem install ruby-xz + class XZ < BinData::IO::Transform + transform_changes_stream_length! + + def initialize(read_length) + super() + @length = read_length + end + + def read(n) + @read ||= ::XZ::decompress(chain_read(@length)) + @read.slice!(0...n) + end + + def write(data) + @write ||= create_empty_binary_string + @write << data + end + + def after_read_transform + raise IOError, "didn't read all data" unless @read.empty? + end + + def after_write_transform + chain_write(::XZ::compress(@write)) + end + end + end +end diff --git a/lib/bindata/transform/zlib.rb b/lib/bindata/transform/zlib.rb new file mode 100755 index 0000000..4e724af --- /dev/null +++ b/lib/bindata/transform/zlib.rb @@ -0,0 +1,33 @@ +require 'zlib' + +module BinData + module Transform + # Transforms a zlib compressed data stream. + class Zlib < BinData::IO::Transform + transform_changes_stream_length! + + def initialize(read_length) + super() + @length = read_length + end + + def read(n) + @read ||= ::Zlib::Inflate.inflate(chain_read(@length)) + @read.slice!(0...n) + end + + def write(data) + @write ||= create_empty_binary_string + @write << data + end + + def after_read_transform + raise IOError, "didn't read all data" unless @read.empty? + end + + def after_write_transform + chain_write(::Zlib::Deflate.deflate(@write)) + end + end + end +end diff --git a/lib/bindata/transform/zstd.rb b/lib/bindata/transform/zstd.rb new file mode 100755 index 0000000..58e3b12 --- /dev/null +++ b/lib/bindata/transform/zstd.rb @@ -0,0 +1,35 @@ +require 'zstd-ruby' + +module BinData + module Transform + # Transforms a zstd compressed data stream. + # + # gem install zstd-ruby + class Zstd < BinData::IO::Transform + transform_changes_stream_length! + + def initialize(read_length) + super() + @length = read_length + end + + def read(n) + @read ||= ::Zstd::decompress(chain_read(@length)) + @read.slice!(0...n) + end + + def write(data) + @write ||= create_empty_binary_string + @write << data + end + + def after_read_transform + raise IOError, "didn't read all data" unless @read.empty? + end + + def after_write_transform + chain_write(::Zstd::compress(@write)) + end + end + end +end diff --git a/test/section_test.rb b/test/section_test.rb index 155e3b6..d0dbc4f 100755 --- a/test/section_test.rb +++ b/test/section_test.rb @@ -4,60 +4,70 @@ describe BinData::Section do it "transforms data byte at a time" do - class XorTransform < BinData::IO::Transform - def initialize(xor) - super() - @xor = xor - end - - def read(n) - chain_read(n).bytes.map { |byte| (byte ^ @xor).chr }.join - end - - def write(data) - chain_write(data.bytes.map { |byte| (byte ^ @xor).chr }.join) - end - end + require 'bindata/transform/xor' - obj = BinData::Section.new(transform: -> { XorTransform.new(0xff) }, + obj = BinData::Section.new(transform: -> { BinData::Transform::Xor.new(0xff) }, type: [:string, read_length: 5]) _(obj.read("\x97\x9A\x93\x93\x90")).must_equal "hello" end - it "expands data" do - class ZlibTransform < BinData::IO::Transform - require 'zlib' + begin + require 'brotli' + it "transform brotli" do + require 'bindata/transform/brotli' - transform_changes_stream_length! + class BrotliRecord < BinData::Record + int32le :len, value: -> { s.num_bytes } + section :s, transform: -> { BinData::Transform::Brotli.new(len) } do + int32le :str_len, value: -> { str.length } + string :str, read_length: :str_len - def initialize(read_length) - super() - @length = read_length + end end - def read(n) - @read ||= Zlib::Inflate.inflate(chain_read(@length)) - @read.slice!(0...n) - end + obj = BrotliRecord.new + data = "highly compressable" * 100 + obj.s.str = data + _(obj.len).must_be :<, (data.length / 10) - def write(data) - @write ||= create_empty_binary_string - @write << data - end + str = obj.to_binary_s + obj = BrotliRecord.read(str) + _(obj.s.str).must_equal data + end + rescue LoadError; end - def after_read_transform - raise IOError, "didn't read all data" unless @read.empty? - end + begin + require 'extlz4' + it "transform lz4" do + require 'bindata/transform/lz4' + + class LZ4Record < BinData::Record + int32le :len, value: -> { s.num_bytes } + section :s, transform: -> { BinData::Transform::LZ4.new(len) } do + int32le :str_len, value: -> { str.length } + string :str, read_length: :str_len - def after_write_transform - chain_write(Zlib::Deflate.deflate(@write)) + end end + + obj = LZ4Record.new + data = "highly compressable" * 100 + obj.s.str = data + _(obj.len).must_be :<, (data.length / 10) + + str = obj.to_binary_s + obj = LZ4Record.read(str) + _(obj.s.str).must_equal data end + rescue LoadError; end + + it "transform zlib" do + require 'bindata/transform/zlib' class ZlibRecord < BinData::Record int32le :len, value: -> { s.num_bytes } - section :s, transform: -> { ZlibTransform.new(len) } do + section :s, transform: -> { BinData::Transform::Zlib.new(len) } do int32le :str_len, value: -> { str.length } string :str, read_length: :str_len @@ -73,4 +83,29 @@ class ZlibRecord < BinData::Record obj = ZlibRecord.read(str) _(obj.s.str).must_equal data end + + begin + require 'zstd-ruby' + it "transform zstd" do + require 'bindata/transform/zstd' + + class ZstdRecord < BinData::Record + int32le :len, value: -> { s.num_bytes } + section :s, transform: -> { BinData::Transform::Zstd.new(len) } do + int32le :str_len, value: -> { str.length } + string :str, read_length: :str_len + + end + end + + obj = ZstdRecord.new + data = "highly compressable" * 100 + obj.s.str = data + _(obj.len).must_be :<, (data.length / 10) + + str = obj.to_binary_s + obj = ZstdRecord.read(str) + _(obj.s.str).must_equal data + end + rescue LoadError; end end