Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pedantic suggestions #141

Merged
merged 5 commits into from
Sep 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions bitshuffle/ext.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ IF ZSTD_SUPPORT:
@cython.wraparound(False)
def compress_zstd(np.ndarray arr not None, int block_size=0, int comp_lvl=1):
"""Bitshuffle then compress an array using ZSTD.

Parameters
----------
arr : numpy array
Expand All @@ -504,14 +504,14 @@ IF ZSTD_SUPPORT:
automatically.
comp_lvl : positive integer
Compression level applied by ZSTD

Returns
-------
out : array with np.uint8 data type
Buffer holding compressed data.

"""

cdef int ii, size, itemsize, count=0
shape = (arr.shape[i] for i in range(arr.ndim))
if not arr.flags['C_CONTIGUOUS']:
Expand All @@ -520,12 +520,12 @@ IF ZSTD_SUPPORT:
size = arr.size
dtype = arr.dtype
itemsize = dtype.itemsize

max_out_size = bshuf_compress_zstd_bound(size, itemsize, block_size)

cdef np.ndarray out
out = np.empty(max_out_size, dtype=np.uint8)

cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat
arr_flat = arr.view(np.uint8).ravel()
cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat
Expand All @@ -540,12 +540,12 @@ IF ZSTD_SUPPORT:
excp = RuntimeError(msg % count, count)
raise excp
return out[:count]

@cython.boundscheck(False)
@cython.wraparound(False)
def decompress_zstd(np.ndarray arr not None, shape, dtype, int block_size=0):
"""Decompress a buffer using ZSTD then bitunshuffle it yielding an array.

Parameters
----------
arr : numpy array
Expand All @@ -559,24 +559,24 @@ IF ZSTD_SUPPORT:
block_size : positive integer
Block size in number of elements. Must match value used for
compression.

Returns
-------
out : numpy array with shape *shape* and data type *dtype*
Decompressed data.

"""

cdef int ii, size, itemsize, count=0
if not arr.flags['C_CONTIGUOUS']:
msg = "Input array must be C-contiguous."
raise ValueError(msg)
size = np.prod(shape)
itemsize = dtype.itemsize

cdef np.ndarray out
out = np.empty(tuple(shape), dtype=dtype)

cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat
arr_flat = arr.view(np.uint8).ravel()
cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat
Expand Down
2 changes: 0 additions & 2 deletions bitshuffle/h5.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -231,5 +231,3 @@ def create_bitshuffle_compressed_dataset(parent, name, shape, dtype,
filter_opts=filter_opts, maxshape=maxshape,
fillvalue=fillvalue, track_times=track_times)
return dset_id


13 changes: 9 additions & 4 deletions src/bitshuffle.c
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,17 @@


// Macros.
#define CHECK_ERR_FREE_LZ(count, buf) if (count < 0) { \
free(buf); return count - 1000; }
#define CHECK_ERR_FREE_LZ(count, buf) \
do { \
if ((count) < 0) { \
free(buf); \
return (count) - 1000; \
} \
} while (0)


/* Bitshuffle and compress a single block. */
int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr, \
int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size, const int option) {

int64_t nbytes, count;
Expand Down Expand Up @@ -115,7 +120,7 @@ int64_t bshuf_decompress_lz4_block(ioc_chain *C_ptr,

#ifdef ZSTD_SUPPORT
/* Bitshuffle and compress a single block. */
int64_t bshuf_compress_zstd_block(ioc_chain *C_ptr, \
int64_t bshuf_compress_zstd_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size, const int comp_lvl) {

int64_t nbytes, count;
Expand Down
20 changes: 10 additions & 10 deletions src/bitshuffle_core.c
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ typedef size_t omp_size_t;
#endif

// Macros.
#define CHECK_MULT_EIGHT(n) if (n % 8) return -80;
#define CHECK_MULT_EIGHT(n) do { if ((n) % 8) return -80; } while (0)
#define MAX(X,Y) ((X) > (Y) ? (X) : (Y))


Expand Down Expand Up @@ -309,7 +309,7 @@ int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t siz
for (jj = 0; jj < elem_size; jj++) {
for (ii = 0; ii < nbyte_row; ii++) {
for (kk = 0; kk < 8; kk++) {
out_b[ii * 8 * elem_size + jj * 8 + kk] = \
out_b[ii * 8 * elem_size + jj * 8 + kk] =
in_b[(jj * 8 + kk) * nbyte_row + ii];
}
}
Expand All @@ -319,7 +319,7 @@ int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t siz


/* Shuffle bits within the bytes of eight element blocks. */
int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, \
int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out,
const size_t size, const size_t elem_size) {

const char *in_b;
Expand Down Expand Up @@ -890,7 +890,7 @@ int64_t bshuf_shuffle_bit_eightelem_NEON(const void* in, void* out, const size_t
/* ---- Worker code that uses SSE2 ----
*
* The following code makes use of the SSE2 instruction set and specialized
* 16 byte registers. The SSE2 instructions are present on modern x86
* 16 byte registers. The SSE2 instructions are present on modern x86
* processors. The first Intel processor microarchitecture supporting SSE2 was
* Pentium 4 (2000).
*
Expand Down Expand Up @@ -1795,7 +1795,7 @@ int64_t bshuf_untrans_bit_elem_AVX512(const void* in, void* out, const size_t si

/* ---- Drivers selecting best instruction set at compile time. ---- */

int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
const size_t elem_size) {

int64_t count;
Expand All @@ -1814,7 +1814,7 @@ int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
}


int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
const size_t elem_size) {

int64_t count;
Expand All @@ -1837,7 +1837,7 @@ int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,

/* Wrap a function for processing a single block to process an entire buffer in
* parallel. */
int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out, \
int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out,
const size_t size, const size_t elem_size, size_t block_size, const int option) {

omp_size_t ii = 0;
Expand Down Expand Up @@ -1895,7 +1895,7 @@ int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out,


/* Bitshuffle a single block. */
int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \
int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size, const int option) {

size_t this_iter;
Expand All @@ -1904,7 +1904,7 @@ int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \
int64_t count;



in = ioc_get_in(C_ptr, &this_iter);
ioc_set_next_in(C_ptr, &this_iter,
(void*) ((char*) in + size * elem_size));
Expand All @@ -1918,7 +1918,7 @@ int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \


/* Bitunshuffle a single block. */
int64_t bshuf_bitunshuffle_block(ioc_chain* C_ptr, \
int64_t bshuf_bitunshuffle_block(ioc_chain* C_ptr,
const size_t size, const size_t elem_size, const int option) {


Expand Down
8 changes: 7 additions & 1 deletion src/bitshuffle_internals.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,13 @@


// Macros.
#define CHECK_ERR_FREE(count, buf) if (count < 0) { free(buf); return count; }
#define CHECK_ERR_FREE(count, buf) \
do { \
if ((count) < 0) { \
free(buf); \
return (count); \
} \
} while (0)


#ifdef __cplusplus
Expand Down
26 changes: 13 additions & 13 deletions src/bshuf_h5filter.c
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ herr_t bshuf_h5_set_local(hid_t dcpl, hid_t type, hid_t space){

elem_size = H5Tget_size(type);
if(elem_size <= 0) {
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
"Invalid element size.");
return -1;
}
Expand All @@ -83,7 +83,7 @@ herr_t bshuf_h5_set_local(hid_t dcpl, hid_t type, hid_t space){
break;
#endif
default:
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
"Invalid bitshuffle compression.");
}
}
Expand All @@ -108,23 +108,23 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
void *out_buf;

if (cd_nelmts < 3) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Not enough parameters.");
return 0;
}
elem_size = cd_values[2];
#ifdef ZSTD_SUPPORT
const int comp_lvl = cd_values[5];
const int comp_lvl = cd_values[5];
#endif

// User specified block size.
if (cd_nelmts > 3) block_size = cd_values[3];

if (block_size == 0) block_size = bshuf_default_block_size(elem_size);

#ifndef ZSTD_SUPPORT
if (cd_nelmts > 4 && (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD)) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"ZSTD compression filter chosen but ZSTD support not installed.");
return 0;
}
Expand All @@ -145,12 +145,12 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
nbytes_uncomp = nbytes;
// Pick which compressions library to use
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
buf_size_out = bshuf_compress_lz4_bound(nbytes_uncomp / elem_size,
buf_size_out = bshuf_compress_lz4_bound(nbytes_uncomp / elem_size,
elem_size, block_size) + 12;
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
buf_size_out = bshuf_compress_zstd_bound(nbytes_uncomp / elem_size,
buf_size_out = bshuf_compress_zstd_bound(nbytes_uncomp / elem_size,
elem_size, block_size) + 12;
}
#endif
Expand All @@ -162,15 +162,15 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,

// TODO, remove this restriction by memcopying the extra.
if (nbytes_uncomp % elem_size) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Non integer number of elements.");
return 0;
}
size = nbytes_uncomp / elem_size;

out_buf = malloc(buf_size_out);
if (out_buf == NULL) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Could not allocate output buffer.");
return 0;
}
Expand Down Expand Up @@ -199,16 +199,16 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
bshuf_write_uint32_BE((char*) out_buf + 8, block_size * elem_size);
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
err = bshuf_compress_lz4(in_buf, (char*) out_buf + 12, size,
elem_size, block_size);
elem_size, block_size);
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
err = bshuf_compress_zstd(in_buf, (char*) out_buf + 12, size,
elem_size, block_size, comp_lvl);
elem_size, block_size, comp_lvl);
}
#endif
nbytes_out = err + 12;
}
}
} else {
if (flags & H5Z_FLAG_REVERSE) {
// Bit unshuffle.
Expand Down
1 change: 0 additions & 1 deletion src/bshuf_h5plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,3 @@

H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;}
const void* H5PLget_plugin_info(void) {return bshuf_H5Filter;}

Loading
Loading