Dynamic modification of configuration files at the Codis level

.github/workflows/pika.yml

@@ -9,6 +9,7 @@ on:
 env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
   BUILD_TYPE: RelWithDebInfo
+  ARTIFACT_PIKA_NAME: artifact-pika
 
 jobs:
   build_on_ubuntu:
@@ -50,15 +51,19 @@ jobs:
         id: cache-ubuntu
         with:
           key: ${{ runner.os }}-build-ubuntu-${{ hashFiles('**/CMakeLists.txt') }}
-          path: ${{ github.workspace }}
-          restore-keys: |
-            ${{ runner.os }}-build-ubuntu-
+          path: |
+            ${{ github.workspace }}/buildtrees
+            ${{ github.workspace }}/deps
 
       - name: Build
-        if: ${{ steps.cache-ubuntu.outputs.cache-hit != 'true' }}
         # Build your program with the given configuration
         run: cmake --build build --config ${{ env.BUILD_TYPE }}
 
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_PIKA_NAME }}
+          path: ${{ github.workspace }}/build/pika
+
       - name: Test
         working-directory: ${{ github.workspace }}/build
         # Execute tests defined by the CMake configuration.
@@ -115,17 +120,7 @@ jobs:
           source /opt/rh/devtoolset-10/enable
           cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
 
-      - name: Cache Build
-        uses: actions/cache@v3
-        id: cache-centos
-        with:
-          key: ${{ runner.os }}-build-centos-${{ hashFiles('**/CMakeLists.txt') }}
-          path: ${{ github.workspace }}
-          restore-keys: |
-            ${{ runner.os }}-build-centos-
-
       - name: Build
-        if: ${{ steps.cache-centos.outputs.cache-hit != 'true' }}
         run: |
           source /opt/rh/devtoolset-10/enable
           cmake --build build --config ${{ env.BUILD_TYPE }}
@@ -150,7 +145,6 @@ jobs:
           cd ../tests/integration/
           chmod +x integrate_test.sh 
           sh integrate_test.sh
-  
 
   build_on_macos:
     runs-on: macos-latest
@@ -174,17 +168,7 @@ jobs:
           export CC=/usr/local/opt/gcc@10/bin/gcc-10 
           cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
 
-      - name: Cache Build
-        uses: actions/cache@v3
-        id: cache-macos
-        with:
-          key: ${{ runner.os }}-build-macos-${{ hashFiles('**/CMakeLists.txt') }}
-          path: ${{ github.workspace }}
-          restore-keys: |
-            ${{ runner.os }}-build-macos-
-
       - name: Build
-        if: ${{ steps.cache-macos.outputs.cache-hit != 'true' }}
         run: |
           cmake --build build --config ${{ env.BUILD_TYPE }}
 
@@ -213,6 +197,7 @@ jobs:
   build_pika_image:
     name: Build Pika Docker image
     runs-on: ubuntu-latest
+    needs: build_on_ubuntu
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
@@ -229,11 +214,16 @@ jobs:
         with:
           images: pikadb/pika
 
+      - uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_PIKA_NAME }}
+          path: artifact/
+
       - name: Build Docker image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./Dockerfile
+          file: ./ci/Dockerfile
           push: false
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

ci/Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    rsync && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists /var/cache/apt/archives
+
+ENV PIKA=/pika \
+    PATH=${PIKA}:${PIKA}/bin:${PATH}
+
+WORKDIR ${PIKA}
+
+COPY artifact/pika ${PIKA}/bin/pika
+COPY entrypoint.sh /entrypoint.sh
+COPY conf/pika.conf ${PIKA}/conf/pika.conf
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+EXPOSE 9221
+
+CMD ["/pika/bin/pika", "-c", "/pika/conf/pika.conf"]

codis/config/proxy.toml

@@ -102,7 +102,6 @@ session_break_on_failure = false
 # Slowlog-log-slower-than(us), from receive command to send response, 0 is allways print slow log
 slowlog_log_slower_than = 100000
 
-
 # Set metrics server (such as http://localhost:28000), proxy will report json formatted metrics to specified server in a predefined period.
 metrics_report_server = ""
 metrics_report_period = "1s"

codis/pkg/proxy/config.go

@@ -118,7 +118,6 @@ session_break_on_failure = false
 # Slowlog-log-slower-than(us), from receive command to send response, 0 is allways print slow log
 slowlog_log_slower_than = 100000
 
-
 # Set metrics server (such as http://localhost:28000), proxy will report json formatted metrics to specified server in a predefined period.
 metrics_report_server = ""
 metrics_report_period = "1s"

integrate_test.sh

@@ -1,18 +1,7 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# Copyright (c) 2023-present, Qihoo, Inc.  All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
 
 #!/bin/bash
 

src/net/src/build_version.cc.in

@@ -1,3 +1,8 @@
+// Copyright (c) 2015-present, Qihoo, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
 #include "net/include/build_version.h"
 const char* net_build_git_sha = "net_build_git_sha:@@GIT_SHA@@";
 const char* net_build_git_date = "net_build_git_date:@@GIT_DATE_TIME@@";

src/net/src/http_conn.cc

@@ -2,6 +2,7 @@
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
+
 #include "net/include/http_conn.h"
 #include <climits>
 #include <cstdio>

src/net/src/net_thread_name.h

@@ -2,6 +2,7 @@
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
+
 #ifndef NET_THREAD_NAME_H
 #define NET_THREAD_NAME_H
 

src/net/src/net_util.cc

@@ -2,6 +2,7 @@
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
+
 #include "net/src/net_util.h"
 #include <fcntl.h>
 #include <netinet/in.h>

src/pstd/src/build_version.cc

@@ -1,3 +1,8 @@
+// Copyright (c) 2015-present, Qihoo, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
 #include "pstd/include/version.h"
 const char* pstd_build_git_sha = "pstd_build_git_sha:2f67b928b3ccd2f23109802aa9932a7af45abcd9";
 const char* pstd_build_git_date = "pstd_build_git_date:2023-03-27";

src/storage/src/redis_strings.cc

@@ -348,8 +348,9 @@ Status RedisStrings::Decrby(const Slice& key, int64_t value, int64_t* ret) {
       int32_t timestamp = parsed_strings_value.timestamp();
       std::string old_user_value = parsed_strings_value.value().ToString();
       char* end = nullptr;
+      errno = 0;
       int64_t ival = strtoll(old_user_value.c_str(), &end, 10);
-      if (*end != 0) {
+      if (errno == ERANGE || *end != 0) {
         return Status::Corruption("Value is not a integer");
       }
       if ((value >= 0 && LLONG_MIN + value > ival) || (value < 0 && LLONG_MAX + value < ival)) {
@@ -657,10 +658,12 @@ Status RedisStrings::SetBit(const Slice& key, int64_t offset, int32_t on, int32_
   Status s = db_->Get(default_read_options_, key, &meta_value);
   if (s.ok() || s.IsNotFound()) {
     std::string data_value;
+    int32_t timestamp = 0;
     if (s.ok()) {
       ParsedStringsValue parsed_strings_value(&meta_value);
       if (!parsed_strings_value.IsStale()) {
         data_value = parsed_strings_value.value().ToString();
+        timestamp = parsed_strings_value.timestamp();
       }
     }
     size_t byte = offset >> 3;
@@ -686,6 +689,7 @@ Status RedisStrings::SetBit(const Slice& key, int64_t offset, int32_t on, int32_
       data_value.append(1, byte_val);
     }
     StringsValue strings_value(data_value);
+    strings_value.set_timestamp(timestamp);
     return db_->Put(rocksdb::WriteOptions(), key, strings_value.Encode());
   } else {
     return s;
@@ -802,13 +806,15 @@ Status RedisStrings::Setrange(const Slice& key, int64_t start_offset, const Slic
   ScopeRecordLock l(lock_mgr_, key);
   Status s = db_->Get(default_read_options_, key, &old_value);
   if (s.ok()) {
+    int32_t timestamp = 0;
     ParsedStringsValue parsed_strings_value(&old_value);
     parsed_strings_value.StripSuffix();
     if (parsed_strings_value.IsStale()) {
       std::string tmp(start_offset, '\0');
       new_value = tmp.append(value.data());
       *ret = static_cast<int32_t>(new_value.length());
     } else {
+      timestamp = parsed_strings_value.timestamp();
       if (static_cast<size_t>(start_offset) > old_value.length()) {
         old_value.resize(start_offset);
         new_value = old_value.append(value.data());
@@ -823,6 +829,7 @@ Status RedisStrings::Setrange(const Slice& key, int64_t start_offset, const Slic
     }
     *ret = static_cast<int32_t>(new_value.length());
     StringsValue strings_value(new_value);
+    strings_value.set_timestamp(timestamp);
     return db_->Put(default_write_options_, key, strings_value.Encode());
   } else if (s.IsNotFound()) {
     std::string tmp(start_offset, '\0');

src/storage/tests/strings_test.cc

@@ -938,6 +938,9 @@ TEST_F(StringsTest, BitPosTest) {
 
 // PKSetexAt
 TEST_F(StringsTest, PKSetexAtTest) {
+#ifdef OS_MACOSX
+  return ;
+#endif
   int64_t unix_time;
   rocksdb::Env::Default()->GetCurrentTime(&unix_time);
   std::map<storage::DataType, int64_t> ttl_ret;

tests/integration/integrate_test.sh

@@ -1,18 +1,7 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# Copyright (c) 2023-present, Qihoo, Inc.  All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
 
 go mod tidy
 go test