Skip to content

Commit

Permalink
DAOS-15066 test: Update dfuse/bash and more to vms. (#13631)
Browse files Browse the repository at this point in the history 
Move daos_build test vms.
Add dfs tag and move slow test to daily.
Do not iterate 25 times but just run once.
Bump command timeout from 30 to 120 seconds.

Signed-off-by: Ashley Pittman <[email protected]>
  • Loading branch information
@ashleypittman
ashleypittman authored Feb 2, 2024
1 parent 98317fd commit 132b00d
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 136 deletions.
224 changes: 98 additions & 126 deletions src/tests/ftest/dfuse/bash.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
"""
(C) Copyright 2020-2023 Intel Corporation.
(C) Copyright 2020-2024 Intel Corporation.
SPDX-License-Identifier: BSD-2-Clause-Patent
"""
import os

import general_utils
from ClusterShell.NodeSet import NodeSet
from dfuse_test_base import DfuseTestBase
from exception_utils import CommandFailure
from run_utils import run_remote


class Cmd(DfuseTestBase):
Expand Down Expand Up @@ -43,19 +41,11 @@ def run_bashcmd(self, il_lib=None, compatible_mode=False):
Remove a directory
and more
"""
dir_name = self.params.get("dirname", '/run/bashcmd/*')
file_name1 = self.params.get("filename1", '/run/bashcmd/*')
file_name2 = self.params.get("filename2", '/run/bashcmd/*')
dd_count = self.params.get("dd_count", '/run/bashcmd/*')
dd_blocksize = self.params.get("dd_blocksize", '/run/bashcmd/*')
pool_count = self.params.get("pool_count", '/run/pool/*')
cont_count = self.params.get("cont_count", '/run/container/*')
dd_count = 512
dd_blocksize = 512

if il_lib is not None:
# no need to run multiple pools and containers with interception lib loaded
pool_count = 1
cont_count = 1
lib_path = os.path.join(self.prefix, 'lib64', il_lib)
lib_path = os.path.join(self.prefix, "lib64", il_lib)
if compatible_mode:
env_str = f"export LD_PRELOAD={lib_path}; export D_IL_COMPATIBLE=1; "
else:
Expand All @@ -64,109 +54,91 @@ def run_bashcmd(self, il_lib=None, compatible_mode=False):
env_str = ""

# Create a pool if one does not already exist.
for _ in range(pool_count):
self.add_pool(connect=False)
# perform test for multiple containers.
for count in range(cont_count):
self.add_container(self.pool)
mount_dir = f"/tmp/{self.pool.uuid}_daos_dfuse{count}"
self.start_dfuse(
self.hostlist_clients, self.pool, self.container, mount_dir=mount_dir)
if il_lib is not None:
# unmount dfuse and mount again with caching disabled
self.dfuse.unmount(tries=1)
self.dfuse.update_params(disable_caching=True)
self.dfuse.update_params(disable_wb_cache=True)
self.dfuse.run()

fuse_root_dir = self.dfuse.mount_dir.value
abs_dir_path = os.path.join(fuse_root_dir, dir_name)
abs_file_path1 = os.path.join(abs_dir_path, file_name1)
abs_file_path2 = os.path.join(abs_dir_path, file_name2)

with open(os.path.join(fuse_root_dir, 'src.c'), 'w') as fd:
fd.write('#include <stdio.h>\n\nint main(void) {\nprintf("Hello World!");\n}\n')
link_name = os.path.join(fuse_root_dir, 'link_c')

with open(os.path.join(fuse_root_dir, 'src_a.c'), 'w') as fd:
fd.write('#include <stdio.h>\n\nvoid fun_a(void) {\nprintf("fun_a()");\n}\n')
with open(os.path.join(fuse_root_dir, 'src_b.c'), 'w') as fd:
fd.write('#include <stdio.h>\n\nvoid fun_b(void) {\nprintf("fun_b()");\n}\n')
# list of commands to be executed.
commands = [f"mkdir -p {abs_dir_path}",
f"touch {abs_file_path1}",
f"ls -a {abs_file_path1}",
f"rm {abs_file_path1}",
f"dd if=/dev/zero of={abs_file_path1} count={dd_count}"
f" bs={dd_blocksize}",
f"ls -al {abs_file_path1}",
f"filesize=$(stat -c%s '{abs_file_path1}');"
f"if (( filesize != {dd_count}*{dd_blocksize} )); then exit 1; fi",
f"cp -r {abs_file_path1} {abs_file_path2}",
f"diff {abs_file_path1} {abs_file_path2}",
f"cmp --silent {abs_file_path1} {abs_file_path2}",
f"rm {abs_file_path2}",
f"mv {abs_file_path1} {abs_file_path2}",
f"ls -al {abs_file_path2}",
f"ls -al {abs_dir_path}/.",
f"ls -al {abs_dir_path}/..",
f"rm {abs_file_path2}",
f"rmdir {abs_dir_path}",
f"wc -l {fuse_root_dir}/src.c",
f"xxd {fuse_root_dir}/src.c",
f'ln -s src.c {link_name}',
f'readlink {link_name}',
f'realpath {link_name}',
f'head {fuse_root_dir}/src.c',
f'tail {fuse_root_dir}/src.c',
# f'more {fuse_root_dir}/src.c', # more hangs over ssh somehow
f'dos2unix {fuse_root_dir}/src.c',
f"gcc -o {fuse_root_dir}/output {fuse_root_dir}/src.c",
f'size {fuse_root_dir}/output',
f'readelf -s {fuse_root_dir}/output',
f'strip -s {fuse_root_dir}/output',
f"g++ -o {fuse_root_dir}/output {fuse_root_dir}/src.c",
f"gcc -c -o {fuse_root_dir}/obj_a.o {fuse_root_dir}/src_a.c",
f"gcc -c -o {fuse_root_dir}/obj_b.o {fuse_root_dir}/src_b.c",
f"ar -rc {fuse_root_dir}/lib.a {fuse_root_dir}/obj_a.o "
f"{fuse_root_dir}/obj_b.o",
f"objdump -d {fuse_root_dir}/obj_a.o",
f"grep print {fuse_root_dir}/src.c",
"awk '{print $1}' " + fuse_root_dir + '/src.c',
f"base64 {fuse_root_dir}/src.c",
f"md5sum {fuse_root_dir}/src.c",
f"cksum {fuse_root_dir}/src.c",
f"bzip2 -z {fuse_root_dir}/lib.a",
f"chmod u-r {fuse_root_dir}/lib.a.bz2",
"fio --readwrite=randwrite --name=test --size=\"2M\" --directory "\
f"{fuse_root_dir}/ --bs=1M --numjobs=\"4\" --ioengine=psync "\
"--group_reporting --exitall_on_error --continue_on_error=none",
f"curl \"https://www.google.com\" -o {fuse_root_dir}/download.html"]
for cmd in commands:
try:
# execute bash cmds
ret_code = general_utils.pcmd(
self.hostlist_clients, env_str + cmd, timeout=30)
if 0 not in ret_code:
error_hosts = NodeSet(
",".join(
[str(node_set) for code, node_set in
list(ret_code.items()) if code != 0]))
raise CommandFailure(
f"Error running '{cmd}' on the following hosts: {error_hosts}")
# report error if any command fails
except CommandFailure as error:
self.log.error("BashCmd Test Failed: %s",
str(error))
self.fail("Test was expected to pass but "
"it failed.\n")

# stop dfuse
self.stop_dfuse()
# destroy container
self.container.destroy()
# destroy pool
self.pool.destroy()
self.add_pool(connect=False)
self.add_container(self.pool)
mount_dir = f"/tmp/{self.pool.identifier}_daos_dfuse"
self.start_dfuse(self.hostlist_clients, self.pool, self.container, mount_dir=mount_dir)
if il_lib is not None:
# unmount dfuse and mount again with caching disabled
self.dfuse.unmount(tries=1)
self.dfuse.update_params(disable_caching=True)
self.dfuse.update_params(disable_wb_cache=True)
self.dfuse.run()

fuse_root_dir = self.dfuse.mount_dir.value
abs_dir_path = os.path.join(fuse_root_dir, "test")
abs_file_path1 = os.path.join(abs_dir_path, "testfile1.txt")
abs_file_path2 = os.path.join(abs_dir_path, "testfile2.txt")

with open(os.path.join(fuse_root_dir, "src.c"), "w") as fd:
fd.write('#include <stdio.h>\n\nint main(void) {\nprintf("Hello World!");\n}\n')
link_name = os.path.join(fuse_root_dir, "link_c")

with open(os.path.join(fuse_root_dir, "src_a.c"), "w") as fd:
fd.write('#include <stdio.h>\n\nvoid fun_a(void) {\nprintf("fun_a()");\n}\n')
with open(os.path.join(fuse_root_dir, "src_b.c"), "w") as fd:
fd.write('#include <stdio.h>\n\nvoid fun_b(void) {\nprintf("fun_b()");\n}\n')
# list of commands to be executed.
commands = [
f"mkdir -p {abs_dir_path}",
f"touch {abs_file_path1}",
f"ls -a {abs_file_path1}",
f"rm {abs_file_path1}",
f"dd if=/dev/zero of={abs_file_path1} count={dd_count}" f" bs={dd_blocksize}",
f"ls -al {abs_file_path1}",
f"filesize=$(stat -c%s '{abs_file_path1}');"
f"if (( filesize != {dd_count}*{dd_blocksize} )); then exit 1; fi",
f"cp -r {abs_file_path1} {abs_file_path2}",
f"diff {abs_file_path1} {abs_file_path2}",
f"cmp --silent {abs_file_path1} {abs_file_path2}",
f"rm {abs_file_path2}",
f"mv {abs_file_path1} {abs_file_path2}",
f"ls -al {abs_file_path2}",
f"ls -al {abs_dir_path}/.",
f"ls -al {abs_dir_path}/..",
f"rm {abs_file_path2}",
f"rmdir {abs_dir_path}",
f"wc -l {fuse_root_dir}/src.c",
f"xxd {fuse_root_dir}/src.c",
f"ln -s src.c {link_name}",
f"readlink {link_name}",
f"realpath {link_name}",
f"head {fuse_root_dir}/src.c",
f"tail {fuse_root_dir}/src.c",
# f'more {fuse_root_dir}/src.c', # more hangs over ssh somehow
f"dos2unix {fuse_root_dir}/src.c",
f"gcc -o {fuse_root_dir}/output {fuse_root_dir}/src.c",
f"size {fuse_root_dir}/output",
f"readelf -s {fuse_root_dir}/output",
f"strip -s {fuse_root_dir}/output",
f"g++ -o {fuse_root_dir}/output {fuse_root_dir}/src.c",
f"gcc -c -o {fuse_root_dir}/obj_a.o {fuse_root_dir}/src_a.c",
f"gcc -c -o {fuse_root_dir}/obj_b.o {fuse_root_dir}/src_b.c",
f"ar -rc {fuse_root_dir}/lib.a {fuse_root_dir}/obj_a.o " f"{fuse_root_dir}/obj_b.o",
f"objdump -d {fuse_root_dir}/obj_a.o",
f"grep print {fuse_root_dir}/src.c",
"awk '{print $1}' " + fuse_root_dir + "/src.c",
f"base64 {fuse_root_dir}/src.c",
f"md5sum {fuse_root_dir}/src.c",
f"cksum {fuse_root_dir}/src.c",
f"bzip2 -z {fuse_root_dir}/lib.a",
f"chmod u-r {fuse_root_dir}/lib.a.bz2",
'fio --readwrite=randwrite --name=test --size="2M" --directory '
f'{fuse_root_dir}/ --bs=1M --numjobs="4" --ioengine=psync '
"--group_reporting --exitall_on_error --continue_on_error=none",
f'curl "https://www.google.com" -o {fuse_root_dir}/download.html',
]
for cmd in commands:
result = run_remote(self.log, self.hostlist_clients, env_str + cmd)
if not result.passed:
self.fail(f'"{cmd}" failed on {result.failed_hosts}')

# stop dfuse
self.stop_dfuse()
# destroy container
self.container.destroy()
# destroy pool
self.pool.destroy()

def test_bashcmd(self):
"""
Expand All @@ -176,9 +148,9 @@ def test_bashcmd(self):
for different container and pool sizes and perform basic bash
commands.
:avocado: tags=all,pr,daily_regression
:avocado: tags=hw,medium
:avocado: tags=dfuse
:avocado: tags=all,daily_regression
:avocado: tags=vm
:avocado: tags=dfuse,dfs
:avocado: tags=Cmd,test_bashcmd
"""
self.run_bashcmd()
Expand All @@ -192,8 +164,8 @@ def test_bashcmd_ioil(self):
commands.
:avocado: tags=all,pr,daily_regression
:avocado: tags=hw,medium
:avocado: tags=dfuse,il
:avocado: tags=vm
:avocado: tags=dfuse,il,dfs
:avocado: tags=Cmd,test_bashcmd_ioil
"""
self.run_bashcmd(il_lib="libioil.so")
Expand All @@ -206,9 +178,9 @@ def test_bashcmd_pil4dfs(self):
for different container and pool sizes and perform basic bash
commands.
:avocado: tags=all
:avocado: tags=hw,medium
:avocado: tags=dfuse,pil4dfs
:avocado: tags=all,daily_regression
:avocado: tags=vm
:avocado: tags=dfuse,pil4dfs,dfs
:avocado: tags=Cmd,test_bashcmd_pil4dfs
"""
self.run_bashcmd(il_lib="libpil4dfs.so")
14 changes: 4 additions & 10 deletions src/tests/ftest/dfuse/bash.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,16 @@ server_config:
engines_per_host: 1
engines:
0:
targets: 4
nr_xs_helpers: 0
storage:
0:
class: dcpm
scm_list: ["/dev/pmem0"]
class: ram
scm_mount: /mnt/daos
system_ram_reserved: 1
pool:
scm_size: 1000000000
control_method: dmg
pool_count: 5
container:
type: POSIX
control_method: daos
cont_count: 5
bashcmd:
dirname: test
filename1: testfile1.txt
filename2: testfile2.txt
dd_count: 512
dd_blocksize: 512

0 comments on commit 132b00d

Please sign in to comment.