DAOS-14369 test: Rework pydaos and and avocado install/setup process. (…

…#13565)

Update pydaos install process. Include pydaos sources in daos-devel rpm to allow
installation on non-default python versions and virtual environments.
Use updated python version for NLT to improve reporting and verify pydaos install process.
Update dependencies for daos-client-tests rpm, add daos-devel to get pydaos sources
and remove some no longer required python libs.
Re-write entire python requirements.txt files whilst keeping install process the same.
requirements.txt is now split into build, utest and ftest files, the origional file pulls in all
three as well as cq for linting.
Update the build to only install build requirements, have utest install utest requirements and
ftest install ftest requirements.

Include avocado 82 in ftest requirements so this is used universally throughout CI.
Update code as required to no longer handle or patch older avocado versions and
add check that at least 82 is in use.

Signed-off-by: Ashley Pittman [email protected]

.dockerignore

@@ -24,7 +24,9 @@
 !ftest.sh
 !site_scons
 !.clang-format
-!requirements.txt
+!requirements-build.txt
+!requirements-utest.txt
+!requirements-ftest.txt
 
 # Now disallow extra files which may be present but are not required.
 # Include the rdb repo as well.

.github/workflows/landing-builds.yml

@@ -14,7 +14,8 @@ on:
       - 'utils/scripts/helpers/*'
       - utils/ci
       - ci
-      - requirements.txt
+      - requirements-build.txt
+      - requirements-utest.txt
 
 jobs:
 

.github/workflows/linting.yml

@@ -26,7 +26,7 @@ jobs:
           python-version: '3'
       - uses: isort/isort-action@master
         with:
-          requirementsFiles: "requirements.txt utils/cq/requirements.txt"
+          requirementsFiles: "requirements.txt"
       - name: Run on SConstruct file.
         run: isort --check-only SConstruct
       - name: Run on build files.

.github/workflows/version-checks.yml

@@ -19,6 +19,9 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
       - name: Install extra python packages
         run: python3 -m pip install --requirement utils/cq/requirements.txt
       - name: Check ${{ matrix.package }} version

SConstruct

@@ -480,9 +480,8 @@ def scons():
     prereqs.save_build_info()
     # also install to $PREFIX/lib to work with existing avocado test code
     if prereqs.test_requested():
-        env.Install('$PREFIX/lib/daos', ['.build_vars.sh', '.build_vars.json'])
         env.Install('$PREFIX/lib/daos/TESTING/ftest/util', ['site_scons/env_modules.py'])
-        env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh'])
+        env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh', "requirements-ftest.txt"])
 
     env.Install("$PREFIX/lib64/daos", "VERSION")
 

ci/gha_helper.py

@@ -16,6 +16,8 @@
                '.github/workflows/landing-builds.yml',
                '.dockerignore',
                'requirements.txt',
+               'requirements-build.txt',
+               'requirements-utest.txt',
                'ci/gha_helper.py']
 
 COMMIT_CMD = ['git', 'rev-parse', '--short', 'HEAD']

ci/provisioning/post_provision_config_nodes_EL_8.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-#  (C) Copyright 2021-2023 Intel Corporation.
+#  (C) Copyright 2021-2024 Intel Corporation.
 #
 #  SPDX-License-Identifier: BSD-2-Clause-Patent
 
@@ -15,19 +15,12 @@ group_repo_post() {
 }
 
 distro_custom() {
-    # install avocado
-    local avocado_rpms=(python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}})
-    if [ -z "$(dnf repoquery "${avocado_rpms[@]}")" ]; then
-        avocado_rpms=()
-        pip install "avocado-framework<83.0"
-        pip install "avocado-framework-plugin-result-html<83.0"
-        pip install "avocado-framework-plugin-varianter-yaml-to-mux<83.0"
-    fi
-    dnf -y install "${avocado_rpms[@]}" clustershell
-
-    # for Launchable's pip install
-    dnf -y install python3-setuptools.noarch
+    # TODO: This code is not exiting on failure.
 
+    # Use a more recent python version for unit testing, this allows us to also test installing
+    # pydaos into virtual environments.
+    dnf -y install python39 python39-devel
+    dnf -y install python3.11 python3.11-devel
 }
 
 install_mofed() {

ci/provisioning/post_provision_config_nodes_LEAP_15.sh

@@ -16,13 +16,4 @@ distro_custom() {
         sed -e '/MODULEPATH=/s/$/:\/usr\/share\/modules/'                     \
                /etc/profile.d/lmod.sh;                                        \
     fi
-
-    # force install of avocado 69.x
-    dnf -y erase avocado{,-common}                                              \
-                 python2-avocado{,-plugins-{output-html,varianter-yaml-to-mux}}
-    python3 -m pip install --upgrade pip
-    python3 -m pip install "avocado-framework<70.0"
-    python3 -m pip install "avocado-framework-plugin-result-html<70.0"
-    python3 -m pip install "avocado-framework-plugin-varianter-yaml-to-mux<70.0"
-
 }

ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh

@@ -48,9 +48,7 @@ post_provision_config_nodes() {
         fi
     fi
 
-    apt-get -y install avocado python3-avocado-plugins-output-html   \
-                       python3-avocado-plugins-varianter-yaml-to-mux \
-                       lsb-core
+    apt-get -y install lsb-core
 
     # shellcheck disable=2086
     if [ -n "$INST_RPMS" ] &&
@@ -63,10 +61,6 @@ post_provision_config_nodes() {
         return "$rc"
     fi
 
-    # temporary hack until Python 3 is supported by Functional testing
-    # possible TODO: support testing non-RPM testing
-    sed -ie '1s/2/3/' /usr/lib/daos/TESTING/ftest/launch.py
-
     # change the default shell to bash -- we write a lot of bash
     chsh -s /bin/bash
 

ci/rpm/test_daos_node.sh

@@ -101,17 +101,29 @@ sudo chmod 0755 /tmp/daos_sockets
 sudo chown "$me:$me" /tmp/daos_sockets
 
 FTEST=/usr/lib/daos/TESTING/ftest
-sudo PYTHONPATH="$FTEST/util"                               \
-     $FTEST/config_file_gen.py -n "$HOSTNAME"               \
-                               -a /etc/daos/daos_agent.yml  \
-                               -s /etc/daos/daos_server.yml
+
+python3 -m venv venv
+# shellcheck disable=SC1091
+source venv/bin/activate
+pip install --upgrade pip
+pip install -r $FTEST/requirements-ftest.txt
+
+sudo PYTHONPATH="$FTEST/util"                        \
+     "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py -n "$HOSTNAME" \
+        -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml
 sudo bash -c 'echo "system_ram_reserved: 4" >> /etc/daos/daos_server.yml'
 sudo PYTHONPATH="$FTEST/util"                        \
-     $FTEST/config_file_gen.py -n "$HOSTNAME"        \
-                               -d /etc/daos/daos_control.yml
+     "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py \
+     -n "$HOSTNAME" -d /etc/daos/daos_control.yml
 cat /etc/daos/daos_server.yml
 cat /etc/daos/daos_agent.yml
 cat /etc/daos/daos_control.yml
+
+# python3.6 does not like deactivate with -u set, later versions are OK with it however.
+set +u
+deactivate
+set -u
+
 if ! module load "$OPENMPI"; then
     echo "Unable to load OpenMPI module: $OPENMPI"
     module avail

ci/unit/required_packages.sh

@@ -2,17 +2,13 @@
 
 set -eux
 
-distro="$1"
 # No longer used but provided by pipeline-lib
+# distro="$1"
 # quick_build="${2:-false}"
 
-if [[ "$distro" = *7 ]]; then
-    OPENMPI_VER="3"
-    PY_MINOR_VER="6"
-elif [[ "$distro" = *8 ]]; then
-    OPENMPI_VER=""
-    PY_MINOR_VER=""
-fi
+OPENMPI_VER=""
+PY_MINOR_VER=""
+
 pkgs="argobots                         \
       boost-python3$PY_MINOR_VER-devel \
       capstone                         \
@@ -32,9 +28,6 @@ pkgs="argobots                         \
       patchelf                         \
       pmix                             \
       protobuf-c                       \
-      python3$PY_MINOR_VER-junit_xml   \
-      python3$PY_MINOR_VER-pyxattr     \
-      python3$PY_MINOR_VER-tabulate    \
       spdk-devel                       \
       valgrind-devel"
 

ci/unit/test_main_node.sh

@@ -73,5 +73,18 @@ fi
 
 rm -rf "$test_log_dir"
 
+# Use default python as that's where storage_estimator is installed.
+python3 -m venv venv
+# shellcheck disable=SC1091
+source venv/bin/activate
+# touch venv/pip.conf
+# pip config set global.progress_bar off
+# pip config set global.no_color true
+
+pip install --upgrade pip
+pip install --requirement requirements-utest.txt
+
+pip install /opt/daos/lib/daos/python/
+
 utils/run_utest.py $RUN_TEST_VALGRIND --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" \
                    $SUDO_ARG

ci/unit/test_nlt.sh

@@ -11,7 +11,7 @@ NODE=${NODELIST%%,*}
 mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
 # Copy over the install tree and some of the build tree.
-rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils jenkins@"$NODE":build/
+rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils requirements-utest.txt jenkins@"$NODE":build/
 
 # shellcheck disable=SC2029
 ssh -tt "$SSH_KEY_ARGS" jenkins@"$NODE" "$(cat "$mydir/test_nlt_node.sh")"

ci/unit/test_nlt_node.sh

@@ -24,5 +24,18 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh"
 # ./utils/node_local_test.py --max-log-size ????MiB --dfuse-dir /localhome/jenkins/ \
 #			   --server-valgrind all
 
+# Use the latest version that CI has available.
+python3.11 -m venv venv
+# shellcheck disable=SC1091
+source venv/bin/activate
+touch venv/pip.conf
+pip config set global.progress_bar off
+pip config set global.no_color true
+
+pip install --upgrade pip
+pip install --requirement requirements-utest.txt
+
+pip install /opt/daos/lib/daos/python/
+
 ./utils/node_local_test.py --max-log-size 1700MiB --dfuse-dir /localhome/jenkins/ \
     --log-usage-save nltir.xml --log-usage-export nltr.json all

debian/changelog

@@ -1,3 +1,9 @@
+daos (2.5.101-3) unstable; urgency=medium
+  [ Ashley M. Pittman ]
+  * Updated pydaos install process
+
+ -- Ashley M. Pittman <[email protected]>  Thu, 04 Apr 2024 09:15:00 -0800
+
 daos (2.5.101-2) unstable; urgency=medium
   [ Jan Michalski ]
   * Add dtx_tests to the server-tests package

debian/daos-client-tests.install

@@ -21,6 +21,3 @@ usr/bin/crt_launch
 usr/bin/daos_gen_io_conf
 usr/bin/daos_run_io_conf
 usr/lib64/libdpar.so
-# For avocado tests
-usr/lib/daos/.build_vars.json
-usr/lib/daos/.build_vars.sh

debian/libdaos-dev.dirs

@@ -1,4 +1,5 @@
 usr/lib64
 usr/include
 usr/include/cart
-usr/include/gurt
+usr/include/gurt
+usr/lib/daos/python

debian/libdaos-dev.install

@@ -5,3 +5,4 @@ usr/lib64/libdaos.so
 usr/lib64/libgurt.so
 usr/lib64/libcart.so
 usr/lib64/*.a
+usr/lib/daos/python/*

requirements-build.txt

@@ -0,0 +1,5 @@
+scons
+ninja
+meson
+distro
+pyelftools

requirements-ftest.txt

@@ -0,0 +1,6 @@
+avocado-framework==82
+avocado-framework-plugin-result-html==82
+avocado-framework-plugin-varianter-yaml-to-mux==82
+clustershell
+paramiko
+distro

requirements-utest.txt

@@ -0,0 +1,6 @@
+tabulate
+junit_xml
+pyxattr
+PyYAML
+# Disabled for now because of DAOS-14688
+# pytest

requirements.txt

@@ -1,13 +1,8 @@
-# Packages required to build/test DAOS.
-defusedxml
-distro
-jira
-junit_xml
-meson
-ninja
-pyelftools
-pyxattr
-pyyaml
-scons
-tabulate
-wheel
+# Packages required to build DAOS.
+-r requirements-build.txt
+# Packages required to run ftest.
+-r requirements-ftest.txt
+# Packages required to run unit testing.
+-r requirements-utest.txt
+# Packages used for linting.
+-r utils/cq/requirements.txt