Merge pull request opencurve#984 from wu-hanqing/merge-branch-fs

Merge branch 'fs'

.gitignore

@@ -81,4 +81,19 @@ runlog/
 .DS_Store
 
 test/client/configs/*
-!test/client/configs/.gitkeep
+!test/client/configs/.gitkeep
+
+# curvefs
+curvefs/devops/projects/
+curvefs/devops/ssh/
+curvefs/devops/inventory
+curvefs/BUILD_MODE
+*.pyc
+.facts/
+*retry
+
+curvefs/docker/curvefs
+curvefs/docker/base/*
+!curvefs/docker/base/Dockerfile
+!curvefs/docker/base/Makefile
+curvefs/BUILD_MODE

BUILD

@@ -0,0 +1,39 @@
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+#
+# Copyright 2017 The Abseil Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//visibility:public"])
+
+config_setting(
+    name = "clang_compiler",
+    flag_values = {
+        "@bazel_tools//tools/cpp:compiler": "clang",
+    },
+)

LICENSE

@@ -150,3 +150,7 @@ Files: thirdparties/aws/*
        common.bzl
 Copyright: Copyright 2019 The TensorFlow Authors
 License: Apache 2.0 (see LICENSE_APACHE)
+
+Files: tools/ci/filterbr.py
+       tools/ci/gen-coverage.py
+License: GPL 3.0 (see LICENSE_GPLv3)

WORKSPACE

@@ -189,3 +189,19 @@ new_http_archive(
     strip_prefix = "aws-checksums-0.1.5",
     build_file = "//:thirdparties/aws/aws-checksums.BUILD",
 )
+
+# C++ rules for Bazel.
+http_archive(
+    name = "rules_cc",
+    urls = ["https://github.com/bazelbuild/rules_cc/archive/9e10b8a6db775b1ecd358d8ddd3dab379a2c29a5.zip"],
+    strip_prefix = "rules_cc-9e10b8a6db775b1ecd358d8ddd3dab379a2c29a5",
+    sha256 = "954b7a3efc8752da957ae193a13b9133da227bdacf5ceb111f2e11264f7e8c95",
+)
+
+# abseil-cpp
+http_archive(
+  name = "com_google_absl",
+  urls = ["https://github.com/abseil/abseil-cpp/archive/refs/tags/20210324.2.tar.gz"],
+  strip_prefix = "abseil-cpp-20210324.2",
+  sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
+)

buildfs.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+cpplint --recursive curvefs
+if [ $? -ne 0 ]
+then
+    echo "cpplint failed"
+    exit
+fi
+
+if [ "$1" = "debug" ]
+then
+DEBUG_FLAG="--compilation_mode=dbg"
+fi
+
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
+if [ $? -ne 0 ]
+then
+    echo "build curvefs failed"
+    exit
+fi
+echo "build curvefs success"
+
+if [ "$1" = "test" ]
+then
+./bazel-bin/curvefs/test/metaserver/curvefs_metaserver_test
+if [ $? -ne 0 ]
+then
+    echo "metaserver_test failed"
+    exit
+fi
+./bazel-bin/curvefs/test/mds/curvefs_mds_test
+if [ $? -ne 0 ]
+then
+    echo "mds_test failed"
+    exit
+fi
+fi

conf/s3.conf

@@ -1,8 +1,12 @@
 #
 # S3 options
 #
+# reserved for backward compatible
 s3.nos_address=
+s3.endpoint=
+# reserved for backward compatible
 s3.snapshot_bucket_name=
+s3.bucket_name=
 s3.ak=
 s3.sk=
 # http = 0, https = 1
@@ -14,6 +18,7 @@ s3.connect_timeout=60000
 s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
 s3.loglevel=4
+s3.logPrefix=/data/log/curve/aws_
 s3.async_thread_num=64
 # throttle
 s3.throttle.iopsTotalLimit=5000

copts.bzl

@@ -14,26 +14,146 @@
 #  limitations under the License.
 #
 
-GCC_FLAGS = [
-        "-pthread",
-        "-std=c++11",
-#        "-Werror",
-#        "-Wsign-compare",
-#        "-Wno-unused-parameter",
-#        "-Wno-unused-variable",
-#        "-Woverloaded-virtual",
-#        "-Wnon-virtual-dtor",
-#        "-Wno-missing-field-initializers",
-]
-
-GCC_TEST_FLAGS = [
-        "-pthread",
-        "-std=c++11",
-#        "-Werror",
-#        "-Wsign-compare",
-#        "-Wno-unused-parameter",
-#        "-Wno-unused-variable",
-#        "-Woverloaded-virtual",
-#        "-Wnon-virtual-dtor",
-#        "-Wno-missing-field-initializers",
+#
+# Copyright 2017 The Abseil Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BASE_FLAGS = [
+    "-DOS_LINUX",
+    "-DSNAPPY",
+    "-DHAVE_SSE42",
+    "-fno-omit-frame-pointer",
+    "-momit-leaf-frame-pointer",
+    "-msse4.2",
+    "-pthread",
+]
+
+CXX_FLAGS = [
+    "-std=c++11",
+]
+
+CURVE_GCC_FLAGS = [
+    "-Wall",
+    "-Wextra",
+    "-Wconversion-null",
+    "-Wformat-security",
+    "-Woverlength-strings",
+    "-Wpointer-arith",
+    "-Wundef",
+    "-Wunused-local-typedefs",
+    "-Wunused-result",
+    "-Wvarargs",
+    "-Wvla",
+    "-Wwrite-strings",
+    "-Werror",
+    "-DNOMINMAX",
+]
+
+CURVE_GCC_TEST_FLAGS = [
+    "-Wno-conversion-null",
+    "-Wno-deprecated-declarations",
+    "-Wno-missing-declarations",
+    "-Wno-sign-compare",
+    "-Wno-unused-function",
+    "-Wno-unused-parameter",
+    "-Wno-unused-result",
 ]
+
+# FIXME: temporary disabled because triggered in many places
+CURVE_GCC_DISABLED_FLGAS = [
+    "-Wno-error=sign-compare",
+    "-Wno-error=reorder",
+    "-Wno-error=unused-parameter",
+    "-Wno-error=unused-variable",
+    "-Wno-error=deprecated-declarations",
+    "-Wno-error=narrowing",
+]
+
+# FIXME: temporary disabled because triggered in many places
+CURVE_GCC_TEST_DISABLED_FLAGS = [
+    "-Wno-error=unused-but-set-variable",
+    "-Wno-error=vla",
+    "-Wno-error=uninitialized",
+    "-Wno-error=maybe-uninitialized",
+    "-Wno-error=format",
+    "-Wno-error=write-strings",
+    "-Wno-error=missing-field-initializers",
+]
+
+# FIXME: Verify these flags
+CURVE_LLVM_FLAGS = [
+    "-Wall",
+    "-Wextra",
+    "-Wcast-qual",
+    "-Wconversion",
+    "-Wfloat-overflow-conversion",
+    "-Wfloat-zero-conversion",
+    "-Wfor-loop-analysis",
+    "-Wformat-security",
+    "-Wgnu-redeclared-enum",
+    "-Winfinite-recursion",
+    "-Wliteral-conversion",
+    "-Wmissing-declarations",
+    "-Woverlength-strings",
+    "-Wpointer-arith",
+    "-Wself-assign",
+    "-Wshadow",
+    "-Wstring-conversion",
+    "-Wtautological-overlap-compare",
+    "-Wundef",
+    "-Wuninitialized",
+    "-Wunreachable-code",
+    "-Wunused-comparison",
+    "-Wunused-local-typedefs",
+    "-Wunused-result",
+    "-Wvla",
+    "-Wwrite-strings",
+    "-Wno-float-conversion",
+    "-Wno-implicit-float-conversion",
+    "-Wno-implicit-int-float-conversion",
+    "-Wno-implicit-int-conversion",
+    "-Wno-shorten-64-to-32",
+    "-Wno-sign-conversion",
+    "-DNOMINMAX",
+]
+
+CURVE_LLVM_TEST_FLAGS = [
+    "-Wno-c99-extensions",
+    "-Wno-deprecated-declarations",
+    "-Wno-missing-noreturn",
+    "-Wno-missing-prototypes",
+    "-Wno-missing-variable-declarations",
+    "-Wno-null-conversion",
+    "-Wno-shadow",
+    "-Wno-shift-sign-overflow",
+    "-Wno-sign-compare",
+    "-Wno-unused-function",
+    "-Wno-unused-member-function",
+    "-Wno-unused-parameter",
+    "-Wno-unused-private-field",
+    "-Wno-unused-template",
+    "-Wno-used-but-marked-unused",
+    "-Wno-zero-as-null-pointer-constant",
+    "-Wno-gnu-zero-variadic-macro-arguments",
+]
+
+CURVE_DEFAULT_COPTS = select({
+    "//:clang_compiler": CURVE_LLVM_FLAGS + CXX_FLAGS + BASE_FLAGS,
+    "//conditions:default": CURVE_GCC_FLAGS + CXX_FLAGS + BASE_FLAGS + CURVE_GCC_DISABLED_FLGAS,
+})
+
+CURVE_TEST_COPTS = CURVE_DEFAULT_COPTS + select({
+    "//:clang_compiler": CURVE_LLVM_TEST_FLAGS,
+    "//conditions:default": CURVE_GCC_TEST_FLAGS + CURVE_GCC_TEST_DISABLED_FLAGS,
+})

curve-ansible/roles/generate_config/defaults/main.yml

@@ -267,6 +267,7 @@ s3_max_connections: 32
 s3_connect_timeout: 60000
 s3_request_timeout: 10000
 s3_loglevel: 4
+s3_logPrefix: /data/log/curve/aws_
 s3_async_thread_num: 64
 s3_throttle_iopsTotalLimit: 5000
 s3_throttle_iopsReadLimit: 5000

curve-ansible/roles/generate_config/templates/s3.conf.j2

@@ -1,8 +1,10 @@
 #
 # S3 options
 #
-s3.nos_address={{ s3_nos_address }}
-s3.snapshot_bucket_name={{ s3_snapshot_bucket_name }}
+s3.nos_address={{ s3_endpoint }}
+s3.endpoint={{ s3_endpoint }}
+s3.snapshot_bucket_name={{ s3_bucket_name }}
+s3.bucket_name={{ s3_bucket_name }}
 s3.ak={{ s3_ak }}
 s3.sk={{ s3_sk }}
 # http = 0, https = 1
@@ -14,6 +16,7 @@ s3.connect_timeout={{ s3_connect_timeout }}
 s3.request_timeout={{ s3_request_timeout }}
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
 s3.loglevel={{ s3_loglevel }}
+s3.logPrefix={{ s3_logPrefix }}
 s3.async_thread_num={{ s3_async_thread_num }}
 # throttle
 s3.throttle.iopsTotalLimit={{ s3_throttle_iopsTotalLimit }}

curve-ansible/server.ini

@@ -130,8 +130,8 @@ wait_service_timeout=20
 deploy_dir="${HOME}"
 s3_ak=""
 s3_sk=""
-s3_nos_address=""
-s3_snapshot_bucket_name=""
+s3_endpoint=""
+s3_bucket_name=""
 ansible_ssh_port=22
 curve_root_username=root
 curve_root_password=root_password