Skip to content

Commit

Permalink
Discard trailing spaces and empty lines
Browse files Browse the repository at this point in the history
  • Loading branch information
DimitriPapadopoulos committed May 23, 2023
1 parent e3392e9 commit 2f331b6
Show file tree
Hide file tree
Showing 9 changed files with 42 additions and 48 deletions.
28 changes: 14 additions & 14 deletions bitshuffle/ext.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ IF ZSTD_SUPPORT:
@cython.wraparound(False)
def compress_zstd(np.ndarray arr not None, int block_size=0, int comp_lvl=1):
"""Bitshuffle then compress an array using ZSTD.
Parameters
----------
arr : numpy array
Expand All @@ -504,14 +504,14 @@ IF ZSTD_SUPPORT:
automatically.
comp_lvl : positive integer
Compression level applied by ZSTD
Returns
-------
out : array with np.uint8 data type
Buffer holding compressed data.
"""

cdef int ii, size, itemsize, count=0
shape = (arr.shape[i] for i in range(arr.ndim))
if not arr.flags['C_CONTIGUOUS']:
Expand All @@ -520,12 +520,12 @@ IF ZSTD_SUPPORT:
size = arr.size
dtype = arr.dtype
itemsize = dtype.itemsize

max_out_size = bshuf_compress_zstd_bound(size, itemsize, block_size)

cdef np.ndarray out
out = np.empty(max_out_size, dtype=np.uint8)

cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat
arr_flat = arr.view(np.uint8).ravel()
cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat
Expand All @@ -540,12 +540,12 @@ IF ZSTD_SUPPORT:
excp = RuntimeError(msg % count, count)
raise excp
return out[:count]

@cython.boundscheck(False)
@cython.wraparound(False)
def decompress_zstd(np.ndarray arr not None, shape, dtype, int block_size=0):
"""Decompress a buffer using ZSTD then bitunshuffle it yielding an array.
Parameters
----------
arr : numpy array
Expand All @@ -559,24 +559,24 @@ IF ZSTD_SUPPORT:
block_size : positive integer
Block size in number of elements. Must match value used for
compression.
Returns
-------
out : numpy array with shape *shape* and data type *dtype*
Decompressed data.
"""

cdef int ii, size, itemsize, count=0
if not arr.flags['C_CONTIGUOUS']:
msg = "Input array must be C-contiguous."
raise ValueError(msg)
size = np.prod(shape)
itemsize = dtype.itemsize

cdef np.ndarray out
out = np.empty(tuple(shape), dtype=dtype)

cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat
arr_flat = arr.view(np.uint8).ravel()
cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat
Expand Down
2 changes: 0 additions & 2 deletions bitshuffle/h5.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -231,5 +231,3 @@ def create_bitshuffle_compressed_dataset(parent, name, shape, dtype,
filter_opts=filter_opts, maxshape=maxshape,
fillvalue=fillvalue, track_times=track_times)
return dset_id


8 changes: 4 additions & 4 deletions src/bitshuffle_core.c
Original file line number Diff line number Diff line change
Expand Up @@ -890,7 +890,7 @@ int64_t bshuf_shuffle_bit_eightelem_NEON(const void* in, void* out, const size_t
/* ---- Worker code that uses SSE2 ----
*
* The following code makes use of the SSE2 instruction set and specialized
* 16 byte registers. The SSE2 instructions are present on modern x86
* 16 byte registers. The SSE2 instructions are present on modern x86
* processors. The first Intel processor microarchitecture supporting SSE2 was
* Pentium 4 (2000).
*
Expand Down Expand Up @@ -1795,7 +1795,7 @@ int64_t bshuf_untrans_bit_elem_AVX512(const void* in, void* out, const size_t si

/* ---- Drivers selecting best instruction set at compile time. ---- */

int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
const size_t elem_size) {

int64_t count;
Expand All @@ -1814,7 +1814,7 @@ int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
}


int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
const size_t elem_size) {

int64_t count;
Expand Down Expand Up @@ -1904,7 +1904,7 @@ int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \
int64_t count;



in = ioc_get_in(C_ptr, &this_iter);
ioc_set_next_in(C_ptr, &this_iter,
(void*) ((char*) in + size * elem_size));
Expand Down
26 changes: 13 additions & 13 deletions src/bshuf_h5filter.c
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ herr_t bshuf_h5_set_local(hid_t dcpl, hid_t type, hid_t space){

elem_size = H5Tget_size(type);
if(elem_size <= 0) {
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
"Invalid element size.");
return -1;
}
Expand All @@ -83,7 +83,7 @@ herr_t bshuf_h5_set_local(hid_t dcpl, hid_t type, hid_t space){
break;
#endif
default:
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
"Invalid bitshuffle compression.");
}
}
Expand All @@ -108,23 +108,23 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
void *out_buf;

if (cd_nelmts < 3) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Not enough parameters.");
return 0;
}
elem_size = cd_values[2];
#ifdef ZSTD_SUPPORT
const int comp_lvl = cd_values[5];
const int comp_lvl = cd_values[5];
#endif

// User specified block size.
if (cd_nelmts > 3) block_size = cd_values[3];

if (block_size == 0) block_size = bshuf_default_block_size(elem_size);

#ifndef ZSTD_SUPPORT
if (cd_nelmts > 4 && (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD)) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"ZSTD compression filter chosen but ZSTD support not installed.");
return 0;
}
Expand All @@ -145,12 +145,12 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
nbytes_uncomp = nbytes;
// Pick which compressions library to use
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
buf_size_out = bshuf_compress_lz4_bound(nbytes_uncomp / elem_size,
buf_size_out = bshuf_compress_lz4_bound(nbytes_uncomp / elem_size,
elem_size, block_size) + 12;
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
buf_size_out = bshuf_compress_zstd_bound(nbytes_uncomp / elem_size,
buf_size_out = bshuf_compress_zstd_bound(nbytes_uncomp / elem_size,
elem_size, block_size) + 12;
}
#endif
Expand All @@ -162,15 +162,15 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,

// TODO, remove this restriction by memcopying the extra.
if (nbytes_uncomp % elem_size) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Non integer number of elements.");
return 0;
}
size = nbytes_uncomp / elem_size;

out_buf = malloc(buf_size_out);
if (out_buf == NULL) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Could not allocate output buffer.");
return 0;
}
Expand Down Expand Up @@ -199,16 +199,16 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
bshuf_write_uint32_BE((char*) out_buf + 8, block_size * elem_size);
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
err = bshuf_compress_lz4(in_buf, (char*) out_buf + 12, size,
elem_size, block_size);
elem_size, block_size);
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
err = bshuf_compress_zstd(in_buf, (char*) out_buf + 12, size,
elem_size, block_size, comp_lvl);
elem_size, block_size, comp_lvl);
}
#endif
nbytes_out = err + 12;
}
}
} else {
if (flags & H5Z_FLAG_REVERSE) {
// Bit unshuffle.
Expand Down
1 change: 0 additions & 1 deletion src/bshuf_h5plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,3 @@

H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;}
const void* H5PLget_plugin_info(void) {return bshuf_H5Filter;}

20 changes: 10 additions & 10 deletions src/hdf5_dl.c
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,11 @@ static bool is_init = false;

/*
* Try to find a symbol within a library
*
*
* handle: Handle to the library
* symbol: Symbol to look for
* Returns: a pointer to the symbol or NULL
* if the symbol can't be found
* Returns: a pointer to the symbol or NULL
* if the symbol can't be found
*/
void *find_sym(void *handle, const char *symbol) {

Expand All @@ -127,7 +127,7 @@ void *find_sym(void *handle, const char *symbol) {

/*
* Check that all symbols have been loaded
*
*
* Returns: -1 if an error occured, 0 for success
*/
int check_symbols() {
Expand Down Expand Up @@ -158,10 +158,10 @@ int check_symbols() {

if(DL_H5Functions.H5Tget_class == NULL)
return -1;

if(DL_H5Functions.H5Tget_super == NULL)
return -1;

if(DL_H5Functions.H5Tclose == NULL)
return -1;

Expand Down Expand Up @@ -213,14 +213,14 @@ int init_filter(const char *libname)

/*H5P*/
if(DL_H5Functions.H5Pget_filter_by_id2 == NULL)
DL_H5Functions.H5Pget_filter_by_id2 =
DL_H5Functions.H5Pget_filter_by_id2 =
(DL_func_H5Pget_filter_by_id2)find_sym(handle, "H5Pget_filter_by_id2");

if(DL_H5Functions.H5Pget_chunk == NULL)
DL_H5Functions.H5Pget_chunk = (DL_func_H5Pget_chunk)find_sym(handle, "H5Pget_chunk");

if(DL_H5Functions.H5Pmodify_filter == NULL)
DL_H5Functions.H5Pmodify_filter =
DL_H5Functions.H5Pmodify_filter =
(DL_func_H5Pmodify_filter)find_sym(handle, "H5Pmodify_filter");

/*H5T*/
Expand All @@ -229,10 +229,10 @@ int init_filter(const char *libname)

if(DL_H5Functions.H5Tget_class == NULL)
DL_H5Functions.H5Tget_class = (DL_func_H5Tget_class)find_sym(handle, "H5Tget_class");

if(DL_H5Functions.H5Tget_super == NULL)
DL_H5Functions.H5Tget_super = (DL_func_H5Tget_super)find_sym(handle, "H5Tget_super");

if(DL_H5Functions.H5Tclose == NULL)
DL_H5Functions.H5Tclose = (DL_func_H5Tclose)find_sym(handle, "H5Tclose");

Expand Down
1 change: 0 additions & 1 deletion src/iochain.c
Original file line number Diff line number Diff line change
Expand Up @@ -87,4 +87,3 @@ void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr) {
omp_unset_lock(&(C->in_pl[(*this_iter) % IOC_SIZE].lock));
#endif
}

3 changes: 1 addition & 2 deletions src/iochain.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* Usage
* -----
* - Call `ioc_init` in serial block.
* - Each thread should create a local variable *size_t this_iter* and
* - Each thread should create a local variable *size_t this_iter* and
* pass its address to all function calls. Its value will be set
* inside the functions and is used to identify the thread.
* - Each thread must call each of the `ioc_get*` and `ioc_set*` methods
Expand Down Expand Up @@ -91,4 +91,3 @@ void * ioc_get_out(ioc_chain *C, size_t *this_iter);
void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr);

#endif // IOCHAIN_H

1 change: 0 additions & 1 deletion src/lzf_h5plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,3 @@ H5Z_class_t lzf_H5Filter[1] = {{

H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;}
const void* H5PLget_plugin_info(void) {return lzf_H5Filter;}

0 comments on commit 2f331b6

Please sign in to comment.