From d222cc4c75a1ee428f5c7b4fd04d6c4b454e4da3 Mon Sep 17 00:00:00 2001 From: Linh Tran Tuan Date: Tue, 21 Dec 2021 10:08:53 +0900 Subject: [PATCH] Adapt rocksdb 6.26.1 (#59) --- .github/workflows/go.yml | 2 +- build.sh | 2 +- c.h | 5 +++++ compaction_filter_test.go | 2 +- db_test.go | 14 +++++++------- docker/Dockerfile | 39 --------------------------------------- options.go | 21 +++++++++++++++++++++ options_blob.go | 4 ++++ options_test.go | 4 ++-- transactiondb_test.go | 2 +- 10 files changed, 43 insertions(+), 52 deletions(-) delete mode 100644 docker/Dockerfile diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index fe8a8e2..45a46d7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -33,7 +33,7 @@ jobs: uses: actions/cache@v2 with: path: dist - key: v6.25.3 + key: v6.26.1 - name: Build if: steps.cache-built-rocksdb.outputs.cache-hit != 'true' diff --git a/build.sh b/build.sh index 603f4af..88d9118 100644 --- a/build.sh +++ b/build.sh @@ -33,7 +33,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version} -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DZSTD_ZLIB_SUPPORT=ON -DZSTD_LZMA_SUPPORT=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j$(nproc) install && \ cd $BUILD_PATH && rm -rf * && ldconfig -rocksdb_version="6.25.3" +rocksdb_version="6.26.1" cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \ mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \ -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON \ diff --git a/c.h b/c.h index c8ee0c9..730bef9 100644 --- a/c.h +++ b/c.h @@ -1117,6 +1117,11 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_age_cutoff( extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_age_cutoff( rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_force_threshold( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_force_threshold( + rocksdb_options_t* opt); + /* returns a pointer to a malloc()-ed, null terminated string */ extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string( rocksdb_options_t* opt); diff --git a/compaction_filter_test.go b/compaction_filter_test.go index 9a7d4b0..07d509b 100644 --- a/compaction_filter_test.go +++ b/compaction_filter_test.go @@ -16,7 +16,7 @@ func TestCompactionFilter(t *testing.T) { ) db := newTestDB(t, "TestCompactionFilter", func(opts *Options) { opts.SetCompactionFilter(&mockCompactionFilter{ - filter: func(level int, key, val []byte) (remove bool, newVal []byte) { + filter: func(_ int, key, val []byte) (remove bool, newVal []byte) { if bytes.Equal(key, changeKey) { return false, changeValNew } diff --git a/db_test.go b/db_test.go index 6929014..c86e301 100755 --- a/db_test.go +++ b/db_test.go @@ -52,14 +52,14 @@ func TestDBCRUD(t *testing.T) { // retrieve pinned for i := 0; i < 1000; i++ { - v3, err := db.GetPinned(ro, givenKey) - require.Nil(t, err) + v3, e := db.GetPinned(ro, givenKey) + require.Nil(t, e) require.EqualValues(t, v3.Data(), givenVal2) v3.Destroy() v3.Destroy() - v3NE, err := db.GetPinned(ro, []byte("justFake")) - require.Nil(t, err) + v3NE, e := db.GetPinned(ro, []byte("justFake")) + require.Nil(t, e) require.False(t, v3NE.Exists()) v3NE.Destroy() v3NE.Destroy() @@ -203,9 +203,9 @@ func newTestDBPathNames(t *testing.T, name string, names []string, targetSizes [ paths := make([]string, len(names)) for i, name := range names { - dir, err := ioutil.TempDir("", "gorocksdb-"+name) - require.Nil(t, err) - paths[i] = dir + directory, e := ioutil.TempDir("", "gorocksdb-"+name) + require.Nil(t, e) + paths[i] = directory } dbpaths := NewDBPathsFromData(paths, targetSizes) diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index db17708..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM centos:7 - -# install toolchain -RUN yum -y update && yum -y --setopt=tsflags=nodocs install gcc gcc-c++ git pkg-config make which unzip automake wget libtool && \ - yum clean all && rm -rf /var/cache/yum - -# openssl dev -ARG ssl_version="OpenSSL_1_1_1k" -RUN cd /tmp && wget https://github.com/openssl/openssl/archive/${ssl_version}.tar.gz && tar xzf ${ssl_version}.tar.gz && cd openssl-${ssl_version} && \ - ./Configure linux-x86_64 -Wa,--noexecstack no-shared no-dso -DDSO_NONE --prefix=/usr/local --openssldir=/usr/local && make -j$(nproc) install_dev && \ - cd /tmp && rm -rf * && ldconfig - -# install latest cmake -ARG cmake_version="3.20.2" -RUN cd /tmp && wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}.tar.gz && tar xzf cmake-${cmake_version}.tar.gz && \ - cd cmake-${cmake_version} && ./bootstrap --parallel=$(nproc) && make -j$(nproc) && make install && \ - cd /tmp && rm -rf * - -# setup cxx standard -ENV CFLAGS='-fPIC -O3 -pipe' -ENV CXXFLAGS='-fPIC -O3 -pipe' - -# install go -ENV GOLANG_PACKAGE go1.17.2.linux-amd64.tar.gz - -RUN curl https://dl.google.com/go/${GOLANG_PACKAGE} -o ${GOLANG_PACKAGE} && \ - tar -C /usr/local -xzf ${GOLANG_PACKAGE} && rm ${GOLANG_PACKAGE} - -ENV GOROOT /usr/local/go -ENV GOPATH /go -ENV PATH $PATH:$GOROOT/bin:$GOPATH/bin - -RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" - -# cleanup -RUN yum remove -y automake libtool && yum clean all && rm -rf /var/cache/yum - -# setup pkg-config path -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig diff --git a/options.go b/options.go index b2c8c03..c6d2813 100644 --- a/options.go +++ b/options.go @@ -2034,6 +2034,27 @@ func (opts *Options) GetBlobGCAgeCutoff() float64 { return float64(C.rocksdb_options_get_blob_gc_age_cutoff(opts.c)) } +// SetBlobGCForceThreshold if the ratio of garbage in the oldest blob files exceeds this threshold, +// targeted compactions are scheduled in order to force garbage collecting +// the blob files in question, assuming they are all eligible based on the +// value of blob_garbage_collection_age_cutoff above. This option is +// currently only supported with leveled compactions. +// Note that enable_blob_garbage_collection has to be set in order for this +// option to have any effect. +// +// Default: 1.0 +func (opts *Options) SetBlobGCForceThreshold(val float64) { + C.rocksdb_options_set_blob_gc_force_threshold(opts.c, C.double(val)) +} + +// GetBlobGCForceThreshold get the threshold for ratio of garbage in the oldest blob files. +// See also: `SetBlobGCForceThreshold` +// +// Default: 1.0 +func (opts *Options) GetBlobGCForceThreshold() float64 { + return float64(C.rocksdb_options_get_blob_gc_force_threshold(opts.c)) +} + // SetMaxWriteBufferNumberToMaintain sets total maximum number of write buffers // to maintain in memory including copies of buffers that have already been flushed. // Unlike max_write_buffer_number, this parameter does not affect flushing. diff --git a/options_blob.go b/options_blob.go index 7abfce0..adaf476 100644 --- a/options_blob.go +++ b/options_blob.go @@ -31,4 +31,8 @@ func TestOptionBlobFile(t *testing.T) { require.EqualValues(t, 0.25, opt.GetBlobGCAgeCutoff()) opt.SetBlobGCAgeCutoff(0.3) require.EqualValues(t, 0.3, opt.GetBlobGCAgeCutoff()) + + require.EqualValues(t, 1.0, opt.GetBlobGCForceThreshold()) + opt.SetBlobGCForceThreshold(1.3) + require.EqualValues(t, 1.3, opt.GetBlobGCForceThreshold()) } diff --git a/options_test.go b/options_test.go index b888c1b..a6161b7 100644 --- a/options_test.go +++ b/options_test.go @@ -376,14 +376,14 @@ func TestOptions(t *testing.T) { } func TestOptions2(t *testing.T) { - t.Run("SetUniversalCompactionOpts", func(t *testing.T) { + t.Run("SetUniversalCompactionOpts", func(*testing.T) { opts := NewDefaultOptions() defer opts.Destroy() opts.SetUniversalCompactionOptions(NewDefaultUniversalCompactionOptions()) }) - t.Run("SetFifoCompactionOpts", func(t *testing.T) { + t.Run("SetFifoCompactionOpts", func(*testing.T) { opts := NewDefaultOptions() defer opts.Destroy() diff --git a/transactiondb_test.go b/transactiondb_test.go index 1df7e28..a4c511c 100644 --- a/transactiondb_test.go +++ b/transactiondb_test.go @@ -95,7 +95,7 @@ func TestTransactionDBCRUD(t *testing.T) { func TestTransactionDBGetForUpdate(t *testing.T) { lockTimeoutMilliSec := int64(50) - applyOpts := func(opts *Options, transactionDBOpts *TransactionDBOptions) { + applyOpts := func(_ *Options, transactionDBOpts *TransactionDBOptions) { transactionDBOpts.SetTransactionLockTimeout(lockTimeoutMilliSec) } db := newTestTransactionDB(t, "TestOpenTransactionDb", applyOpts)