diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml index 4512a8e2..d8f263e0 100644 --- a/.github/workflows/cpp_full_node_workflow.yml +++ b/.github/workflows/cpp_full_node_workflow.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-12, ubuntu-20.04] + os: [macos-13, ubuntu-20.04] steps: - uses: actions/checkout@v2 with: @@ -74,6 +74,7 @@ jobs: if: runner.os == 'macOS' run: | bash -x cpp/tools/install_depends.sh -o macos + rm -rf /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ make -j3 - name: Publish Error diff --git a/.github/workflows/cpp_sdk_workflow.yml b/.github/workflows/cpp_sdk_workflow.yml index 4c361914..3ba1ec06 100644 --- a/.github/workflows/cpp_sdk_workflow.yml +++ b/.github/workflows/cpp_sdk_workflow.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, windows-2019, macos-12] + os: [ubuntu-22.04, windows-2019, macos-13] steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml index 04635c77..025f276c 100644 --- a/.github/workflows/cpp_toolkit_workflow.yml +++ b/.github/workflows/cpp_toolkit_workflow.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, windows-2019, macos-12] + os: [ubuntu-22.04, windows-2019, macos-13] steps: - uses: actions/checkout@v2 with: @@ -72,7 +72,7 @@ jobs: - name: Build for windows if: runner.os == 'Windows' run: | - mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64 + mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=OFF -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64 - name: Build for linux if: runner.os == 'Linux' run: | diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh index 5852e25f..1e2a1e6c 100644 --- a/cpp/tools/install_depends.sh +++ b/cpp/tools/install_depends.sh @@ -75,12 +75,12 @@ install_centos_depends() { LOG_INFO "install depends for centos ..." # install the basic package - sudo yum install -y lcov bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel krb5-devel autoconf + sudo yum install -y lcov bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel autoconf LOG_INFO "install basic-package-depends for centos success..." - LOG_INFO "install gsasl ... " - install_gsasl_depend "centos" - LOG_INFO "install gsasl success!" + #LOG_INFO "install gsasl ... " + #install_gsasl_depend "centos" + #LOG_INFO "install gsasl success!" LOG_INFO "install nasm ... " install_nasm_depend @@ -93,12 +93,12 @@ install_ubuntu_depends() { LOG_INFO "install depends for ubuntu ..." # install the basic package - sudo apt install -y lcov bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget libkrb5-dev autoconf + sudo apt install -y lcov bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget autoconf LOG_INFO "install basic-package-depends for ubuntu success..." - LOG_INFO "install gsasl ... " - install_gsasl_depend "ubuntu" - LOG_INFO "install gsasl success!" + #LOG_INFO "install gsasl ... " + #install_gsasl_depend "ubuntu" + #LOG_INFO "install gsasl success!" LOG_INFO "install nasm ... " install_nasm_depend @@ -125,9 +125,9 @@ install_iconv_depend() install_macos_depends() { LOG_INFO "install depends for macos ..." - brew install autoconf nasm lcov iconv - install_gsasl_depend "macos" - install_iconv_depend + brew install autoconf nasm lcov + #install_gsasl_depend "macos" + #install_iconv_depend LOG_INFO "install depends for macos success ..." } diff --git a/cpp/wedpr-helper/ppc-utilities/Utilities.h b/cpp/wedpr-helper/ppc-utilities/Utilities.h index c5e52009..9832b65d 100644 --- a/cpp/wedpr-helper/ppc-utilities/Utilities.h +++ b/cpp/wedpr-helper/ppc-utilities/Utilities.h @@ -64,6 +64,11 @@ inline std::string generateUUID() static thread_local auto uuid_gen = boost::uuids::basic_random_generator(); return boost::uuids::to_string(uuid_gen()); } + +inline std::string getServiceName(std::string const& agencyID, std::string const& serviceID) +{ + return agencyID + "_" + serviceID; +} template inline std::string printVector(T const& list) { diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp index 384fd44a..a0ecaea0 100644 --- a/cpp/wedpr-initializer/Initializer.cpp +++ b/cpp/wedpr-initializer/Initializer.cpp @@ -40,6 +40,7 @@ #include "ppc-psi/src/labeled-psi/LabeledPSIFactory.h" #include "ppc-psi/src/ra2018-psi/RA2018PSIFactory.h" #include "ppc-tools/src/config/PPCConfig.h" +#include "wedpr-helper/ppc-utilities/Utilities.h" #include #include @@ -86,8 +87,9 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway) TransportBuilder transportBuilder; // register the serviceInfo auto serviceConfig = m_serviceConfigBuilder.buildServiceConfig(); - auto entryPoint = - m_serviceConfigBuilder.buildEntryPoint(PSI_SERVICE_TYPE, m_config->accessEntrypoint()); + auto entryPoint = m_serviceConfigBuilder.buildEntryPoint( + getServiceName(m_config->agencyID(), PSI_SERVICE_TYPE), m_config->accessEntrypoint()); + serviceConfig.addEntryPoint(entryPoint); auto serviceMeta = serviceConfig.encode(); m_config->frontConfig()->setMeta(serviceMeta); diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp index afe8dd20..36187f03 100644 --- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp +++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp @@ -22,6 +22,7 @@ #include "ppc-framework/protocol/Constant.h" #include "ppc-mpc/src/MPCService.h" #include "ppc-tools/src/config/PPCConfig.h" +#include "wedpr-helper/ppc-utilities/Utilities.h" #include "wedpr-protocol/protocol/src/ServiceConfig.h" #include "wedpr-transport/sdk/src/TransportBuilder.h" @@ -61,7 +62,7 @@ void MPCInitializer::init(std::string const& _configPath) auto threadPool = std::make_shared("mpc-pool", threadPoolSize); INIT_LOG(INFO) << LOG_DESC("init the mpc threadpool") - << LOG_KV("threadPoolSize", threadPoolSize); + << LOG_KV("threadPoolSize", threadPoolSize); auto mpcService = std::make_shared(); mpcService->setMPCConfig(mpcConfig); @@ -71,11 +72,11 @@ void MPCInitializer::init(std::string const& _configPath) m_rpc->registerHandler("run", std::bind(&MPCService::runMpcRpc, mpcService, std::placeholders::_1, std::placeholders::_2)); m_rpc->registerHandler("asyncRun", std::bind(&MPCService::asyncRunMpcRpc, mpcService, - std::placeholders::_1, std::placeholders::_2)); + std::placeholders::_1, std::placeholders::_2)); m_rpc->registerHandler("kill", std::bind(&MPCService::killMpcRpc, mpcService, std::placeholders::_1, std::placeholders::_2)); m_rpc->registerHandler("query", std::bind(&MPCService::queryMpcRpc, mpcService, - std::placeholders::_1, std::placeholders::_2)); + std::placeholders::_1, std::placeholders::_2)); INIT_LOG(INFO) << LOG_DESC("init the mpc rpc success"); // init the transport initTransport(pt); @@ -89,8 +90,8 @@ void MPCInitializer::initTransport(boost::property_tree::ptree const& property) // add the service meta ServiceConfigBuilder serviceConfigBuilder; - auto entryPoint = - serviceConfigBuilder.buildEntryPoint(MPC_SERVICE_TYPE, m_config->accessEntrypoint()); + auto entryPoint = serviceConfigBuilder.buildEntryPoint( + getServiceName(m_config->agencyID(), MPC_SERVICE_TYPE), m_config->accessEntrypoint()); auto serviceConfig = serviceConfigBuilder.buildServiceConfig(); serviceConfig.addEntryPoint(entryPoint); auto serviceMeta = serviceConfig.encode(); diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile new file mode 100644 index 00000000..6be13fc1 --- /dev/null +++ b/docker-files/cpp/Dockerfile @@ -0,0 +1,44 @@ +FROM ubuntu:18.04 as builder + +LABEL maintainer service@webank.com + +WORKDIR / + +ARG SOURCE_BRANCH +ENV DEBIAN_FRONTEND=noninteractive \ + SOURCE=${SOURCE_BRANCH:-master} + +RUN apt-get -q update && apt-get install -qy --no-install-recommends \ + vim curl git make build-essential cmake \ + libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config curl zip tar ccache uuid-runtime automake autoconf \ + m4 tcpdump net-tools libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev gcc g++ \ + && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ + && apt-get install -qy --no-install-recommends tzdata \ + && dpkg-reconfigure --frontend noninteractive tzdata \ + && rm -rf /var/lib/apt/lists/* + +# install rust +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y + +ENV PATH="/root/.cargo/bin:${PATH}" + +ENV VCPKG_FORCE_SYSTEM_BINARIES=1 + +RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursive --depth=1 -b ${SOURCE} \ + && mkdir -p WeDPR-Component/cpp/build && cd WeDPR-Component/cpp/build \ + && cmake .. -DBUILD_STATIC=ON && make -j8 && cat /WeDPR-Component/cpp/build/*.log + + +FROM ubuntu:18.04 +LABEL maintainer service@webank.com + +RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev \ + && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ + && apt-get install -qy --no-install-recommends tzdata \ + && dpkg-reconfigure --frontend noninteractive tzdata \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-gateway-service /usr/local/bin/ppc-gateway-service +COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /usr/local/bin/ppc-pro-node +COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-air-node /usr/local/bin/ppc-air-node +COPY --from=builder /WeDPR-Component/cpp/build/bin/wedpr-mpc /usr/local/bin/wedpr-mpc \ No newline at end of file diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py index d678bcde..70f99a6e 100644 --- a/python/ppc_model/model_result/task_result_handler.py +++ b/python/ppc_model/model_result/task_result_handler.py @@ -11,10 +11,13 @@ class TaskResultRequest: - def __init__(self, job_id, task_type, only_fetch_log, user): + def __init__(self, job_id, task_type, + fetch_log: bool, + fetch_result: bool, user): self.job_id = job_id self.task_type = task_type - self.only_fetch_log = only_fetch_log + self.fetch_log = fetch_log + self.fetch_result = fetch_result self.user = user @@ -346,7 +349,7 @@ def __init__(self, task_result_request: TaskResultRequest, components: Initializ def get_response(self): response = dict() - if not self.task_result_request.only_fetch_log: + if self.task_result_request.fetch_result is True: merged_result = dict() for result in self.result_list: merged_result.update(result.to_dict()) @@ -356,13 +359,14 @@ def get_response(self): response = {"jobPlanetResult": merged_result, "modelData": self.model_data} # record the log - log_size, log_path, log_content = self.components.log_retriever.retrieve_log( - self.task_result_request.job_id, self.task_result_request.user) - log_result = {} - log_result.update({"logSize": log_size}) - log_result.update({"logPath": log_path}) - log_result.update({"logContent": log_content}) - response.update({"logDetail": log_result}) + if self.task_result_request.fetch_log is True: + log_size, log_path, log_content = self.components.log_retriever.retrieve_log( + self.task_result_request.job_id, self.task_result_request.user) + log_result = {} + log_result.update({"logSize": log_size}) + log_result.update({"logPath": log_path}) + log_result.update({"logContent": log_content}) + response.update({"logDetail": log_result}) return utils.make_response(PpcErrorCode.SUCCESS.get_code(), PpcErrorCode.SUCCESS.get_msg(), response) def _get_evaluation_result(self): diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py index 39e3301f..d99efc2a 100644 --- a/python/ppc_model/network/http/model_controller.py +++ b/python/ppc_model/network/http/model_controller.py @@ -105,12 +105,12 @@ def post(self, task_id): f"get task result, task_id: {task_id}, args: {args}") user_name = args['user'] task_type = args['jobType'] - only_fetch_log = {'True': True, 'False': False}.get( - args['onlyFetchLog']) + fetch_log = args['fetchLog'] + fetch_job_result = args['fetchJobResult'] components.logger().info( - f"get_job_direct_result_response, job: {task_id}") + f"get_job_direct_result_response, job: {task_id}, fetch_log: {fetch_log}, fetch_job_result: {fetch_job_result}") task_result_request = TaskResultRequest( - task_id, task_type, only_fetch_log, user_name) + task_id, task_type, fetch_log, fetch_job_result, user_name) job_result_handler = TaskResultHandler( task_result_request=task_result_request, components=components) response = job_result_handler.get_response() diff --git a/python/wedpr_ml_toolkit/setup.py b/python/wedpr_ml_toolkit/setup.py index e78e60b7..e1bba009 100644 --- a/python/wedpr_ml_toolkit/setup.py +++ b/python/wedpr_ml_toolkit/setup.py @@ -20,7 +20,7 @@ def run(self): setup_args = dict( name='wedpr_ml_toolkit', packages=find_packages(), - version="1.0.0.dev-20241129", + version="1.0.0.dev-20241201", description="wedpr-ml-toolkit: The ML toolkit for WeDPR", long_description_content_type="text/markdown", author="WeDPR Development Team", diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py index c30d6ec9..2b5f6e9c 100644 --- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py +++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py @@ -54,7 +54,7 @@ class StorageConfig(BaseObject): def __init__(self, user_config: UserConfig, storage_endpoint: str = None, - enable_krb5_auth: bool = False, + enable_krb5_auth: str = "False", hdfs_auth_principal: str = None, hdfs_auth_password: str = None, hdfs_hostname_override: str = None):