diff --git a/Jenkinsfile b/Jenkinsfile index 62280860b97..c93c8415730 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes -//@Library(value="pipeline-lib@your_branch") _ +@Library(value="pipeline-lib@java_package") _ // For master, this is just some wildly high number next_version = "1000" diff --git a/ci/functional/required_packages.sh b/ci/functional/required_packages.sh index c123adb68f4..b1e2c844dde 100755 --- a/ci/functional/required_packages.sh +++ b/ci/functional/required_packages.sh @@ -16,7 +16,9 @@ pkgs="openmpi3 hwloc ndctl fio \ hdf5-vol-daos-openmpi3-tests \ MACSio-mpich \ MACSio-openmpi3 \ - mpifileutils-mpich-daos-${client_ver}" + mpifileutils-mpich-daos-${client_ver} \ + protobuf-c \ + protobuf-c-devel" if [[ $distro = el7* ]] || [[ $distro = centos7* ]]; then pkgs="--exclude openmpi $pkgs" diff --git a/ci/rpm/scan_daos_node.sh b/ci/rpm/scan_daos_node.sh index 05f5389d429..e96e7064987 100755 --- a/ci/rpm/scan_daos_node.sh +++ b/ci/rpm/scan_daos_node.sh @@ -3,7 +3,7 @@ set -uex sudo yum -y install \ - daos{,-{client,server,tests,debuginfo,devel}}-"${DAOS_PKG_VERSION}" + daos{,-{client,server,tests,debuginfo,devel,tests-java}}-"${DAOS_PKG_VERSION}" lmd_src="maldet-current" lmd_tarball="maldetect-current.tar.gz" diff --git a/src/client/java/daos-java/find_build_paths.sh b/src/client/java/daos-java/find_build_paths.sh index ad114670fbd..bd17ff3f554 100755 --- a/src/client/java/daos-java/find_build_paths.sh +++ b/src/client/java/daos-java/find_build_paths.sh @@ -2,14 +2,15 @@ daos_install_path=$1 var=$2 -build_vars=../../../../.build_vars.sh +in_tree=$3 +build_vars=../../../../.build_vars_.sh if [ ! -f "${build_vars}" ] then if [ "$var" = "SL_PREFIX" ] then - echo "${daos_install_path}" + echo "${daos_install_path}" else # SL_PROTOBUFC_PREFIX, default to release - echo "${daos_install_path}/prereq/release/protobufc/" + echo "${daos_install_path}/prereq/release/protobufc/" fi else source "${build_vars}" diff --git a/src/client/java/daos-java/find_java_home.sh b/src/client/java/daos-java/find_java_home.sh old mode 100644 new mode 100755 diff --git a/src/client/java/daos-java/pom.xml b/src/client/java/daos-java/pom.xml index 64f06e00075..94a1e5cdf52 100644 --- a/src/client/java/daos-java/pom.xml +++ b/src/client/java/daos-java/pom.xml @@ -2,6 +2,7 @@ + 4.0.0 @@ -18,6 +19,7 @@ ${project.basedir}/build ${project.basedir}/install false + false @@ -98,12 +100,18 @@ maven-antrun-plugin - 1.8 + 3.0.0 compile-proto generate-sources + + + + + build-so compile - + - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -161,6 +192,7 @@ + @@ -173,9 +205,11 @@ executable="gcc"> + + @@ -188,10 +222,13 @@ executable="gcc"> + + + @@ -204,8 +241,11 @@ executable="gcc"> + + + @@ -220,8 +260,11 @@ executable="gcc"> + + + @@ -242,6 +285,9 @@ + + + @@ -257,13 +303,31 @@ - + run + + + ant-contrib + ant-contrib + 1.0b3 + + + ant + ant + + + + + org.apache.ant + ant-nodeps + 1.8.1 + + maven-shade-plugin diff --git a/src/client/java/distribution/pom.xml b/src/client/java/distribution/pom.xml index a7e439b6eb3..098e435a232 100644 --- a/src/client/java/distribution/pom.xml +++ b/src/client/java/distribution/pom.xml @@ -30,6 +30,7 @@ src/assembly/make-assembly.xml + posix @@ -55,6 +56,7 @@ src/assembly/make-assembly-with-dependencies.xml + posix diff --git a/src/tests/ftest/java/ci_integration.py b/src/tests/ftest/java/ci_integration.py new file mode 100644 index 00000000000..23bfabc1b77 --- /dev/null +++ b/src/tests/ftest/java/ci_integration.py @@ -0,0 +1,86 @@ +#!/usr/bin/python3 +""" + (C) Copyright 2020 Intel Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + GOVERNMENT LICENSE RIGHTS-OPEN SOURCE SOFTWARE + The Government's rights to use, modify, reproduce, release, perform, display, + or disclose this software are subject to the terms of the Apache License as + provided in Contract No. B609815. + Any reproduction of computer software, computer software documentation, or + portions thereof marked with this legend must also reproduce the markings. +""" +import os +from general_utils import error_count +from java_utils import JavaUtils + +class JavaCIIntegration(JavaUtils): + """Test class Description: + + Create a DAOS pool and container and use their + uuid's to run the java-hadoop integration tests + using maven. + + :avocado: recursive + """ + + def test_java_hadoop_it(self): + """Jira ID: DAOS-4093 + + Test Description: + Create a DAOS pool and container and use their + uuid's to run the java-hadoop integration tests + using maven. + + :avocado: tags=all,pr,hw,small,javaciintegration + """ + # Copy built java folder to /var/tmp. + # Need to be copied to perform maven operations. + test_jdir = str("{}/../java".format(os.getcwd())) + cp_cmd = "cp -r {} {}/..".format(test_jdir, self.log_dir) + self.run_command(self.hostlist_clients, [cp_cmd]) + + # get java dir + self.jdir = str("{}/../java".format(self.log_dir)) + + # create pool and container + pool = self._create_pool() + container = self._create_cont(pool) + + # obtain java version + version = self.java_version() + + # output file + output_file = "{}/maven_integration_test_output.log".\ + format(self.log_dir) + + # run intergration-test + cmd = "cd {};".format(self.jdir) + cmd += " export LD_PRELOAD={}/jre/lib/amd64/libjsig.so;".format(version) + cmd += " ls -l daos-java/target/antrun/build-compile-proto.xml;" + cmd += " mvn -X integration-test -Ddaos.install.path={}".\ + format(self.prefix) + cmd += " -Dpool_id={} -Dcont_id={} ".format(pool.uuid, container.uuid) + cmd += " >> {}".format(output_file) + print("***{}".format(cmd)) + + self.run_command(self.hostlist_clients, [cmd], True, 120) + # look for errors + found_errors = error_count("Errors: [1-9]+", self.hostlist_clients, + output_file, 'Errors: [0-9]') + self.log.info("Matched error lines: %s", found_errors[0]) + + if found_errors[0] > 0: + self.fail("Java Integration test failed. Check" + " maven_integration_test_output.log for more details.") diff --git a/src/tests/ftest/java/ci_integration.yaml b/src/tests/ftest/java/ci_integration.yaml new file mode 100644 index 00000000000..abd84777aa1 --- /dev/null +++ b/src/tests/ftest/java/ci_integration.yaml @@ -0,0 +1,21 @@ +hosts: + test_servers: + - server-A + test_clients: + - client-B +timeout: 600 +server_config: + name: daos_server + servers: + log_mask: INFO + scm_class: dcpm + scm_list: ["/dev/pmem0"] +pool: + mode: 146 # 146 is RW + name: daos_server + scm_size: 10000000000 + svcn: 1 + control_method: dmg +container: + type: POSIX + control_method: daos diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 467014e3a54..165e183b67f 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -835,7 +835,7 @@ def get_remote_file_size(host, file_name): return int(result.stdout_text) -def error_count(error, hostlist, log_file): +def error_count(error, hostlist, log_file, pattern='\" ERR \"'): """Count the number of specific ERRORs found in the log file. This function also returns a count of the other ERRORs from same log file. @@ -844,6 +844,7 @@ def error_count(error, hostlist, log_file): error (str): DAOS error to look for in .log file. for example -1007 hostlist (list): System list to looks for an error. log_file (str): Log file name (server/client log). + pattern (str): Pattern to look for in each line. Defaults to ERR Returns: tuple: a tuple of the count of errors matching the specified error type @@ -853,11 +854,14 @@ def error_count(error, hostlist, log_file): # Get the Client side Error from client_log file. requested_error_count = 0 other_error_count = 0 - command = 'cat {} | grep \" ERR \"'.format(get_log_file(log_file)) + command = "cat {} | grep '{}'".format(get_log_file(log_file), pattern) results = run_pcmd(hostlist, command, False, None, None) for result in results: for line in result["stdout"]: - if 'ERR' in line: + if pattern is not "ERR": + if re.findall(error, line): + requested_error_count += 1 + elif pattern in line: if error in line: requested_error_count += 1 else: diff --git a/src/tests/ftest/util/java_utils.py b/src/tests/ftest/util/java_utils.py new file mode 100644 index 00000000000..4312d243ec6 --- /dev/null +++ b/src/tests/ftest/util/java_utils.py @@ -0,0 +1,85 @@ +#!/usr/bin/python +""" + (C) Copyright 2020-2021 Intel Corporation. + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" + +from apricot import TestWithServers +from daos_utils import DaosCommand +from test_utils_pool import TestPool +from test_utils_container import TestContainer +from general_utils import run_pcmd + +class JavaUtils(TestWithServers): + """ + Base class for Java test code + :avocado: recursive + """ + + def __init__(self, *args, **kwargs): + """Initialize a JavaUtils object.""" + super().__init__(*args, **kwargs) + self.pool = None + self.container = None + self.jdir = None + + def _create_pool(self): + """Create a TestPool object. + """ + # Get the pool params + pool = TestPool( + self.context, dmg_command=self.get_dmg_command()) + pool.get_params(self) + # Create a pool + pool.create() + return pool + + def _create_cont(self, pool, path=None): + """Create a TestContainer object to be used to create container. + Args: + pool (TestPool): pool object + path (str): Unified namespace path for container + """ + # Get container params + container = TestContainer(pool, daos_command=DaosCommand(self.bin)) + container.get_params(self) + if path is not None: + container.path.update(path) + # create container + container.create() + return container + + def java_version(self): + """Check if java is installed. + + Returns: + bool: whether java is installed or not. + + """ + # look for java home + command = [u"source {}/daos-java/find_java_home.sh".format(self.jdir)] + return self.run_command(self.hostlist_clients, command) + + def run_command(self, hosts, commands, verbose=True, timeout=60): + """Method to execute run_pcmd and return a + failure if the cmd returns non zero value. + Args: + hosts: list of hosts cmd needs to be run on + commands (list): Commands to run + verbose (bool): Display output. Default is True + timeout (int): Timeout for a particular command. + Default is 60 secs + Returns: + Returns command output + """ + + for cmd in commands: + task = run_pcmd(hosts, cmd, verbose, timeout) + for result in task: + if result["exit_status"] != 0: + self.log.info(result["stdout"]) + self.fail("Failed to run cmd {} on {}".\ + format(cmd, result["hosts"])) + if result["stdout"]: + return str(result["stdout"][0]) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 10ea4157855..fa544190df8 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -8,7 +8,7 @@ Name: daos Version: 1.3.0 -Release: 13%{?relval}%{?dist} +Release: 14%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -64,6 +64,7 @@ BuildRequires: openssl-devel BuildRequires: libevent-devel BuildRequires: libyaml-devel BuildRequires: libcmocka-devel +BuildRequires: maven BuildRequires: valgrind-devel BuildRequires: systemd %if (0%{?rhel} >= 7) @@ -205,6 +206,14 @@ Summary: The DAOS development libraries and headers %description devel This is the package needed to build software with the DAOS library. +%package tests-java +Summary: The DAOS Java test suite +Requires: %{name}-devel = %{version}-%{release} +Requires: maven + +%description tests-java +This is the package needed to run the DAOS java test suite. + %package firmware Summary: The DAOS firmware management helper Requires: %{name}-server%{?_isa} = %{version}-%{release} @@ -266,6 +275,11 @@ install -m 644 utils/systemd/%{agent_svc_name} %{buildroot}/%{_unitdir} %endif mkdir -p %{buildroot}/%{conf_dir}/certs/clients mv %{buildroot}/%{_sysconfdir}/daos/bash_completion.d %{buildroot}/%{_sysconfdir} +mvn clean install -DskipITs -Ddaos.install.path=%{buildroot}/%{_prefix} --file src/client/java/daos-java/pom.xml -Dgpg.skip -X +cp -r src/client/java %{buildroot}/%{_prefix}/lib/daos/TESTING/ +BUILDROOT="%{buildroot}" +sed -i -e "s/${BUILDROOT//\//\\/}//g" %{buildroot}/usr/lib/daos/TESTING/java/daos-java/target/{{antrun/build-main,surefire-reports/TEST-io.daos.{dfs.Daos{FilePath,Uns}Test,obj.{DaosObject{,Id},IO{Data,Key}Desc}Test,Daos{Utils,IOException}Test}}.xml,surefire/surefire_*tmp} +chmod 755 %{buildroot}/%{_prefix}/lib/daos/TESTING/java/daos-java/target/antrun/build-compile-proto.xml %pre server getent group daos_metrics >/dev/null || groupadd -r daos_metrics @@ -412,6 +426,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %{_prefix}/lib/daos/.build_vars.sh %{_libdir}/libdts.so +%files tests-java +%{_prefix}/lib/daos/TESTING/java + %files devel %{_includedir}/* %{_libdir}/libdaos.so @@ -422,6 +439,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %attr(4750,root,daos_server) %{_bindir}/daos_firmware %changelog +* Fri Apr 23 2021 Saurabh Tandan 1.3.0-14 +- Add daos-tests-java package + * Wed Apr 21 2021 Tom Nabarro - 1.3.0-13 - Relax ipmctl version requirement on leap15 as we have runtime checks @@ -454,7 +474,6 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent - Update libfabric to v1.12.0 - Disable grdcopy/gdrapi linkage in libfabric - * Thu Mar 18 2021 Maureen Jean 1.3.0-3 - Update to python3 diff --git a/utils/rpms/packaging/rpm_chrootbuild b/utils/rpms/packaging/rpm_chrootbuild index e2b201de803..17ce21d1a5b 100755 --- a/utils/rpms/packaging/rpm_chrootbuild +++ b/utils/rpms/packaging/rpm_chrootbuild @@ -44,4 +44,4 @@ else echo "Unable to update /etc/mock/$CHROOT_NAME.cfg." echo "You need to make sure it has the needed repos in it yourself." fi -eval mock -r "$CHROOT_NAME" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +eval mock --enable-network -r "$CHROOT_NAME" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET"