Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Speed up compression by 2100% #18

Merged
merged 5 commits into from
Jun 3, 2024
Merged

Speed up compression by 2100% #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a38afee
Speed up compression by 2100%
cadmic May 31, 2024
341fee2
Fix typo
cadmic May 31, 2024
1b59532
Restore comment
cadmic May 31, 2024
454cf0e
Avoid computing hash value more than once
cadmic Jun 1, 2024
dff13f4
Optimize longest_common_prefix
cadmic Jun 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions lib/src/mio0.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ fn size_for_compressed_buffer(input_size: usize) -> Result<usize, Crunch64Error>
pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
let input_size = bytes.len();

let mut window = utils::Window::new(bytes);

let mut pp: usize = 0;
let mut index_cur_layout_byte: usize = 0;

Expand All @@ -100,7 +102,7 @@ pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
let mut cur_layout_bit: u32 = 0x80000000;

while input_pos < input_size {
let (mut group_pos, mut group_size) = utils::search(input_pos, bytes, 18);
let (mut group_pos, mut group_size) = window.search(input_pos, 18);

// If the group isn't larger than 2 bytes, copying the input without compression is smaller
if group_size <= 2 {
Expand All @@ -110,7 +112,7 @@ pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
input_pos += 1;
} else {
// Search for a new group after one position after the current one
let (new_position, new_size) = utils::search(input_pos + 1, bytes, 18);
let (new_position, new_size) = window.search(input_pos + 1, 18);

// If the new group is better than the current group by at least 2 bytes, use it instead
if new_size >= group_size + 2 {
Expand Down
200 changes: 133 additions & 67 deletions lib/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,92 +78,158 @@ pub(crate) fn set_pointer_array_from_u8_array(
Ok(())
}

pub(crate) fn search(input_pos: usize, data_in: &[u8], max_match_length: usize) -> (u32, u32) {
let mut cur_size = 3;
let mut found_pos = 0;
let mut search_pos = cmp::max(input_pos as isize - 0x1000, 0) as usize;
let search_size = cmp::min(data_in.len() - input_pos, max_match_length);

if search_size < 3 {
return (0, 0);
}
pub(crate) fn longest_common_prefix(a: &[u8], b: &[u8]) -> usize {
a.iter().zip(b.iter()).take_while(|&(a, b)| a == b).count()
}

const HASH_SIZE: usize = 1 << 15;
const HASH_MASK: usize = HASH_SIZE - 1;

const WINDOW_SIZE: usize = 0x1000;
const WINDOW_MASK: usize = WINDOW_SIZE - 1;

while search_pos < input_pos {
let found_offset = mischarsearch(
&data_in[input_pos..],
cur_size,
&data_in[search_pos..],
cur_size + input_pos - search_pos,
);
const MIN_MATCH: usize = 3;
const NULL: u16 = 0xFFFF;

// Updates a running hash value with a new byte. The shift ensure that only the
// last 3 bytes of the input can affect the hash value.
fn update_hash(hash: usize, byte: u8) -> usize {
((hash << 5) ^ (byte as usize)) & HASH_MASK
}

// Finds the longest match in a 0x1000-byte sliding window, searching
// front-to-back with a minimum match size of 3 bytes. The algorithm is similar
// to the one described in section 4 of RFC 1951
// (https://www.rfc-editor.org/rfc/rfc1951.html#section-4), using a chained hash
// table of 3-byte sequences to find matches. Each character in the window is
// identified by its position & 0xFFF (like in a circular buffer).
pub(crate) struct Window<'a> {
// Compression input
input: &'a [u8],
// Current position in the input
input_pos: usize,
// Hash value at the window start
hash_start: usize,
// Hash value at the current input position
hash_end: usize,
// Head of hash chain for each hash value, or NULL
head: [u16; HASH_SIZE],
// Tail of hash chain for each hash value, or NULL
tail: [u16; HASH_SIZE],
// Next index in the hash chain, or NULL
next: [u16; WINDOW_SIZE],
}

if found_offset >= input_pos - search_pos {
break;
impl Window<'_> {
pub(crate) fn new(input: &[u8]) -> Window {
let mut hash = 0;
for &b in input.iter().take(MIN_MATCH) {
hash = update_hash(hash, b);
}

while cur_size < search_size {
if data_in[cur_size + search_pos + found_offset] != data_in[cur_size + input_pos] {
break;
}
cur_size += 1;
Window {
input,
input_pos: 0,
hash_start: hash,
hash_end: hash,
head: [NULL; HASH_SIZE],
tail: [NULL; HASH_SIZE],
next: [NULL; WINDOW_SIZE],
}
}

if search_size == cur_size {
return ((found_offset + search_pos) as u32, cur_size as u32);
// Advances the window by one byte, updating the hash chains.
pub(crate) fn advance(&mut self) {
if self.input_pos >= self.input.len() {
return;
}

found_pos = (search_pos + found_offset) as isize;
search_pos = (found_pos + 1) as usize;
cur_size += 1;
}
// Remove the oldest byte from the hash chain
if self.input_pos >= WINDOW_SIZE {
let head = self.head[self.hash_start];
let next = self.next[head as usize];

(found_pos as u32, cmp::max(cur_size as isize - 1, 0) as u32)
}
self.head[self.hash_start] = next;
if next == NULL {
self.tail[self.hash_start] = NULL;
}

self.hash_start = update_hash(
self.hash_start,
self.input[self.input_pos - WINDOW_SIZE + MIN_MATCH],
);
}

// Add the current byte to the hash chain
if self.input_pos + MIN_MATCH < self.input.len() {
let tail = self.tail[self.hash_end];
let pos = (self.input_pos & WINDOW_MASK) as u16;

fn mischarsearch(pattern: &[u8], pattern_len: usize, data: &[u8], data_len: usize) -> usize {
let mut skip_table = [0u16; 256];
let mut i: isize;
self.next[pos as usize] = NULL;
self.tail[self.hash_end] = pos;
if tail == NULL {
self.head[self.hash_end] = pos;
} else {
self.next[tail as usize] = pos;
}

let mut v6: isize;
let mut j: isize;
self.hash_end = update_hash(self.hash_end, self.input[self.input_pos + MIN_MATCH]);
}

if pattern_len <= data_len {
initskip(pattern, pattern_len, &mut skip_table);
self.input_pos += 1;
}

i = pattern_len as isize - 1;
loop {
if pattern[pattern_len - 1] == data[i as usize] {
i -= 1;
j = pattern_len as isize - 2;
if j < 0 {
return (i + 1) as usize;
}
// Move the window forward the input position, and seach the window back-to-front for a match
// at most `max_match_length` bytes long, returning the offset and length of the longest match found.
// Successive searches can only be performed at increasing input positions.
pub(crate) fn search(&mut self, input_pos: usize, max_match_length: usize) -> (u32, u32) {
if input_pos < self.input_pos {
panic!("window moved backwards");
} else if input_pos >= self.input.len() {
return (0, 0);
}

while pattern[j as usize] == data[i as usize] {
i -= 1;
j -= 1;
if j < 0 {
return (i + 1) as usize;
}
}
let max_match_length = cmp::min(max_match_length, self.input.len() - input_pos);
if max_match_length < MIN_MATCH {
return (0, 0);
}

v6 = pattern_len as isize - j;
while self.input_pos < input_pos {
self.advance();
}

if skip_table[data[i as usize] as usize] as isize > v6 {
v6 = skip_table[data[i as usize] as usize] as isize;
let mut pos = self.head[self.hash_end];
let mut best_len = MIN_MATCH - 1;
let mut best_offset = 0;

while pos != NULL {
// Figure out the current match offset from `pos` (which is equal to `match_offset & WINDOW_MASK`)
// using the fact that `1 <= input_pos - match_offset <= WINDOW_SIZE`
let match_offset =
input_pos - 1 - (input_pos.wrapping_sub(pos as usize + 1) & WINDOW_MASK);

if self.input[input_pos] == self.input[match_offset]
&& self.input[input_pos + 1] == self.input[match_offset + 1]
&& self.input[match_offset + best_len] == self.input[input_pos + best_len]
{
// The hash function guarantees that if the hashes are equal and
// the first two bytes match, the third byte will too
let candidate_len = 3 + longest_common_prefix(
&self.input[input_pos + 3..input_pos + max_match_length],
&self.input[match_offset + 3..match_offset + max_match_length],
);
if candidate_len > best_len {
best_len = candidate_len;
best_offset = match_offset;
if best_len == max_match_length {
break;
}
}
} else {
v6 = skip_table[data[i as usize] as usize] as isize;
}
i += v6;
}
}
data_len
}

fn initskip(pattern: &[u8], len: usize, skip: &mut [u16; 256]) {
skip.fill(len as u16);
pos = self.next[pos as usize];
}

for i in 0..len {
skip[pattern[i] as usize] = (len - i - 1) as u16;
(best_offset as u32, best_len as u32)
}
}
6 changes: 4 additions & 2 deletions lib/src/yay0.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ fn size_for_compressed_buffer(input_size: usize) -> Result<usize, Crunch64Error>
pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
let input_size = bytes.len();

let mut window = utils::Window::new(bytes);

let mut pp: usize = 0;
let mut index_cur_layout_byte: usize = 0;

Expand All @@ -108,7 +110,7 @@ pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
let mut cur_layout_bit: u32 = 0x80000000;

while input_pos < input_size {
let (mut group_pos, mut group_size) = utils::search(input_pos, bytes, 0x111);
let (mut group_pos, mut group_size) = window.search(input_pos, 0x111);

// If the group isn't larger than 2 bytes, copying the input without compression is smaller
if group_size <= 2 {
Expand All @@ -118,7 +120,7 @@ pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
input_pos += 1;
} else {
// Search for a new group after one position after the current one
let (new_position, new_size) = utils::search(input_pos + 1, bytes, 0x111);
let (new_position, new_size) = window.search(input_pos + 1, 0x111);

// If the new group is better than the current group by at least 2 bytes, use it instead
if new_size >= group_size + 2 {
Expand Down
5 changes: 3 additions & 2 deletions lib/src/yaz0.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
let input_size = bytes.len();

let mut output: Vec<u8> = Vec::with_capacity(size_for_compressed_buffer(input_size)?);
let mut window = utils::Window::new(bytes);

write_header(&mut output, input_size)?;

Expand All @@ -116,7 +117,7 @@ pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
index_out_ptr += 1;
}

let (mut group_pos, mut group_size) = utils::search(input_pos, bytes, 0x111);
let (mut group_pos, mut group_size) = window.search(input_pos, 0x111);

// If the group isn't larger than 2 bytes, copying the input without compression is smaller
if group_size <= 2 {
Expand All @@ -127,7 +128,7 @@ pub fn compress(bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
index_out_ptr += 1;
} else {
// Search for a new group after one position after the current one
let (new_position, new_size) = utils::search(input_pos + 1, bytes, 0x111);
let (new_position, new_size) = window.search(input_pos + 1, 0x111);

// If the new group is better than the current group by at least 2 bytes, use it instead
if new_size >= group_size + 2 {
Expand Down
Loading