Consolidate ansible directory hierarchy

We remove the extra level of `pbench/agent` under `agent/ansible`, while
moving the playbooks and inventory into their own directory.

We also take the opportunity to remove support for collecting `ara`
system information.  While removing references to it, we also corrected
the references to `pbench-sysinfo-dump` in the documentation tree.

As a result of this refactoring, additional ansible & yaml lint issues
we identified and corrected.

A note about `ansible-galaxy` being too "smart" for its own good:

The collection build process was taking in the entire directory
hierarchy as part of the collection even if files and directories are
not part of the documented structure.

To fix this, we move all the actual galaxy collection components into
their own "collection" directory, and keep the supporting cast separate.
This has the benefit of not requiring ignore files.

Further, we move the generated artifacts out of the source tree and into
a temporary directory.  The collection path needs to be configured to
use the temporary directory, so this is unfortunately hard-coded to use
`/var/tmp/agent-ansible`.  If really important we can address this
later.

agent/Makefile

@@ -299,8 +299,6 @@ install-tool-scripts: install-destdir
 
 install-ansible: install-destdir
 	${INSTALL} ${INSTALLOPTS} ${ANSIBLEDIR}
-	cd ansible; \
-	    ${COPY} ara ${ANSIBLEDIR}
 
 clean:
 	rm -rf ${DESTDIR}

agent/ansible/Makefile

@@ -2,31 +2,33 @@
 apiserver = https://galaxy.ansible.com/api/
 apikey = ${APIKEY}
 
-# Collection info
 namespace = pbench
 collection = agent
-version = $(shell ./bin/yamlconf -C ./pbench/agent/galaxy.yml version)
+version = $(shell bin/yamlconf -C collection/galaxy.yml version)
+
+tmpdir = /var/tmp/agent-ansible
 
 # Tar ball name and collections directory path
 tb = ${namespace}-${collection}-${version}.tar.gz
-colldir = collections/ansible_collections/${namespace}/${collection}
+colldir = ${tmpdir}/build/ansible_collections/${namespace}/${collection}
 
 .DEFAULT_GOAL := test
 
 lint:
-	make -C pbench/agent
+	cd collection; ansible-lint -c ../ansible-lint.yml
+	yaml-lint $$(find . -type f -name '*.yml')
+	cd collection; ../bin/check-readmes
 
 build:	lint
-	ansible-galaxy collection build ${namespace}/${collection}
+	mkdir -p ${tmpdir}
+	ansible-galaxy collection build --output-path ${tmpdir} --force collection
 
 test:   build
-	ansible-galaxy collection install ${tb} -p ./collections
-	git submodule update --init --recursive
+	ansible-galaxy collection install --force ${tmpdir}/${tb} -p ${tmpdir}/build
 	cd ${colldir}; ansible-test sanity
 
 publish: clean build
-	ansible-galaxy collection publish -vvv -s "${apiserver}"  --api-key "${apikey}" ${tb}
+	ansible-galaxy collection publish -vvv -s "${apiserver}"  --api-key "${apikey}" ${tmpdir}/${tb}
 
 clean:
-	sudo rm -rf collections
-	rm -f ${tb} ${namespace}-${collection}-*.tar.gz
+	rm -rf ${tmpdir}

agent/ansible/ansible.cfg

@@ -0,0 +1,2 @@
+[defaults]
+collections_paths = /var/tmp/agent-ansible/build/ansible_collections

agent/ansible/pbench/agent/README.md → agent/ansible/collection/README.md

@@ -30,3 +30,8 @@ An example inventory file that can be edited to suit your circumstances can be o
 The raw link for that is
 
     https://raw.githubusercontent.com/distributed-system-analysis/pbench/master/agent/ansible/Inventory/pbench_agent.hosts.example
+
+## Build Requirements
+
+    dnf install python3-ansible-lint rubygem-yaml-lint
+    ansible-galaxy collection install ansible.posix

agent/ansible/pbench/agent/roles/pbench_agent_config/tasks/main.yml → agent/ansible/collection/roles/pbench_agent_config/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 # pbench agent configuration
-- name: "pbench agent configuration - install config file(s)"
+- name: "Pbench agent configuration - install config file(s)"
   ansible.builtin.include_role:
     name: pbench_agent_files_install
   vars:
@@ -9,7 +9,7 @@
     mode: "0644"
     files: "{{ pbench_config_files }}"
 
-- name: "pbench agent configuration - install ssh key"
+- name: "Pbench agent configuration - install ssh key"
   ansible.builtin.include_role:
     name: pbench_agent_files_install
   vars:

agent/ansible/pbench/agent/roles/pbench_agent_files_install/tasks/main.yml → agent/ansible/collection/roles/pbench_agent_files_install/tasks/main.yml

@@ -1,36 +1,36 @@
 ---
 # Install file(s) for pbench agent.
-- name: create temp dir to store the file(s) locally
+- name: Create temp dir to store the file(s) locally
   delegate_to: localhost
   become: false
   ansible.builtin.tempfile:
     state: directory
     prefix: "{{ inventory_hostname }}-"
   register: tempdir_1
 
-- name: relax perms
+- name: Relax perms
   delegate_to: localhost
   become: false
   ansible.builtin.file:
     path: "{{ tempdir_1.path }}"
     mode: 0755
 
-- name: install the file(s) locally
+- name: Install the file(s) locally
   delegate_to: localhost
   become: false
   ansible.builtin.get_url:
     url: "{{ source }}/{{ item }}"
     dest: "{{ tempdir_1.path }}/{{ item }}"
   with_items: "{{ files }}"
 
-- name: copy the file(s) to the remote
+- name: Copy the file(s) to the remote
   ansible.builtin.copy:
     src: "{{ tempdir_1.path }}/{{ item }}"
     dest: "{{ dest }}"
     mode: "{{ mode }}"
   with_items: "{{ files }}"
 
-- name: delete local temp dir
+- name: Delete local temp dir
   delegate_to: localhost
   become: false
   ansible.builtin.file:

agent/ansible/pbench/agent/roles/pbench_firewall_open_ports/tasks/main.yml → agent/ansible/collection/roles/pbench_firewall_open_ports/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 # pbench agent configuration
-- name: "pbench agent configuration - open firewall ports for tool meister"
+- name: "Pbench agent configuration - open firewall ports for tool meister"
   ansible.posix.firewalld:
     port: "{{ item }}/tcp"
     permanent: true

agent/ansible/pbench/agent/roles/pbench_repo_install/tasks/main.yml → agent/ansible/collection/roles/pbench_repo_install/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 # Install pbench.repo
-- name: ensure we have the pbench.repo file properly in place
+- name: Ensure we have the pbench.repo file properly in place
   ansible.builtin.template:
     src: etc/yum.repos.d/pbench.repo.j2
     dest: /etc/yum.repos.d/pbench.repo

agent/ansible/README → agent/ansible/playbooks/README

@@ -16,7 +16,7 @@ N.B. Installation is incomplete until we install a key file so that
 config file so that it knows which server to use.
 
 The playbooks use variables defined in the inventory file (for an
-example inventory file, see =Inventory/pbench-agent.hosts.example=).
+example inventory file, see =./pbench-agent.hosts.example=).
 It must be modified for a given environment: see comments in the
 file for guidance.
 

agent/ansible/pbench_agent_config.yml → agent/ansible/playbooks/pbench_agent_config.yml

@@ -1,5 +1,5 @@
 ---
-- name: install pbench-agent config and key files
+- name: Install pbench-agent config and key files
   hosts: servers
   remote_user: root
 

agent/ansible/pbench_agent_install.yml → agent/ansible/playbooks/pbench_agent_install.yml

@@ -1,5 +1,5 @@
 ---
-- name: install pbench-agent
+- name: Install pbench-agent
   hosts: servers
   remote_user: root
 

agent/ansible/pbench_agent_tool_meister_firewall.yml → agent/ansible/playbooks/pbench_agent_tool_meister_firewall.yml

@@ -1,5 +1,5 @@
 ---
-- name: open ports for redis and tool data sink
+- name: Open ports for redis and tool data sink
   hosts: controllers
   remote_user: root
 

agent/ansible/pbench_repo_install.yml → agent/ansible/playbooks/pbench_repo_install.yml

@@ -1,8 +1,7 @@
 ---
-- name: install pbench-agent repo
+- name: Install pbench-agent repo
   hosts: servers
   remote_user: root
 
   roles:
     - pbench_repo_install
-

agent/ansible/requirements.yml

@@ -0,0 +1,2 @@
+collections:
+  - ansible.posix

agent/containers/images/Makefile

@@ -56,7 +56,7 @@ IMAGE_REPO = docker://quay.io/pbench
 
 # Convenience reference to the repo template in the pbench tree.
 # Not intended to be overridden with an environment variable.
-_PBENCH_REPO_TEMPLATE = ../../ansible/pbench/agent/roles/pbench_repo_install/templates/etc/yum.repos.d/pbench.repo.j2
+_PBENCH_REPO_TEMPLATE = ../../ansible/roles/pbench_repo_install/templates/etc/yum.repos.d/pbench.repo.j2
 
 # This is for use on CentOS; on Fedora, the package is available without a
 # custom .repo file.

agent/util-scripts/tool-meister/pbench-sysinfo-dump

@@ -163,28 +163,6 @@ function collect_sos {
 	fi
 }
 
-function collect_ara {
-	# collect data only when ara is installed
-	if python -c "import ara" &> /dev/null; then
-		ara_pb="${install_dir}/ansible/ara/ara.yml"
-		if [[ ! -d "${install_dir}" || ! -f ${ara_pb} ]]; then
-			printf -- "ERROR: ara playbook not found, '%s'\n" "${ara_pb}" >&2
-			return 1
-		fi
-
-		mkdir ${dir}/ara
-
-		# generate inventory file with controller and remotes
-		INVENTORY=${dir}/ara/inventory
-		generate_inventory > ${INVENTORY}
-
-		ansible-playbook -i ${INVENTORY} --extra-vars '{"SYSINFO_DIR":"'${dir}'"}' "${ara_pb}"
-		printf -- "done collection ara data\n"
-	else
-		printf -- "skipping, ara python module not installed\n"
-	fi
-}
-
 function collect_insights {
 	mkdir ${dir}/insights-client
 	script -c 'insights-client --offline --logging-file ${dir}/insights-client/output.log' /dev/null | sed -n 's/.*\sat\s//p' | tr -d "$'\r" | xargs -I '{}' mv {} ${dir}/insights-client
@@ -197,7 +175,7 @@ function collect_insights {
 
 for item in ${sysinfo//,/ }; do
 	case ${item} in
-	kernel_config|security_mitigations|libvirt|topology|sos|block|ara|insights)
+	kernel_config|security_mitigations|libvirt|topology|sos|block|insights)
 		printf -- "collecting %s\n" "${item}"
 		if [[ "${mode}" == "parallel" ]]; then
 			collect_${item} &

docs/CONFIG_DATA.md

@@ -19,50 +19,46 @@ The structure of the sysinfo directory in a pbench result, for example,  `/var/l
 
 The file contains kernel configuration data. 
 
-The data is collected using [pbench-sysinfo-dump#L38](https://github.com/distributed-system-analysis/pbench/blob/master/agent/util-scripts/pbench-sysinfo-dump#L38). The script uses `uname` system utility (`systemcall` is a term used for all the APIs provided by the kernel) to collect kernel release information and then checks if a corresponding kernel configuration file exists on the system. If it does, the script simply copies the file, located in `/boot` directory, to the `sysinfo` directory.
+The data is collected using [pbench-sysinfo-dump#L43](https://github.com/distributed-system-analysis/pbench/blob/main/agent/util-scripts/tool-meister/pbench-sysinfo-dump#L43). The script uses `uname` system utility (`systemcall` is a term used for all the APIs provided by the kernel) to collect kernel release information and then checks if a corresponding kernel configuration file exists on the system. If it does, the script simply copies the file, located in `/boot` directory, to the `sysinfo` directory.
 
 The file contains data in a key value format where the key is a metric name and the value can be a string literal or a number. The keys and the values are separated by an equality sign.
 
 ## security-mitigation-data.txt
 
 The file contains CPU vulnerabilities data and RHEL-specific flag settings. 
 
-The data is collected using [pbench-sysinfo-dump#L44](https://github.com/distributed-system-analysis/pbench/blob/master/agent/util-scripts/pbench-sysinfo-dump#L44). The script checks if `/sys/devices/system/cpu/vulnerabilities` directory exists. If it does, the script prints the filenames and the contents of all the files located in the directory. After that, it repeats the same steps for the `/sys/kernel/debug/x86` directory.
+The data is collected using [pbench-sysinfo-dump#L50](https://github.com/distributed-system-analysis/pbench/blob/main/agent/util-scripts/tool-meister/pbench-sysinfo-dump#L50). The script checks if `/sys/devices/system/cpu/vulnerabilities` directory exists. If it does, the script prints the filenames and the contents of all the files located in the directory. After that, it repeats the same steps for the `/sys/kernel/debug/x86` directory.
 
 The file contains data in a key value format where the key is a file name and the value is the content of the file.
 
 ## libvirt/
 
 The directory provides information about libvirt, an open-source API, daemon and management tool for managing platform virtualization.
 
-The data is collected using [pbench-sysinfo-dump#L60](https://github.com/distributed-system-analysis/pbench/blob/master/agent/util-scripts/pbench-sysinfo-dump#L60). The script copies libvirt files located at `/var/log/libvirt` and `/etc/libvirt` directories to the `sysinfo/libvirt/log` and `sysinfo/libvirt/etc` directories respectively. Only the files whose name follows the regex `*.log` are copied from the `/var/log/libvirt` directory.
+The data is collected using [pbench-sysinfo-dump#L67](https://github.com/distributed-system-analysis/pbench/blob/main/agent/util-scripts/tool-meister/pbench-sysinfo-dump#L67). The script copies libvirt files located at `/var/log/libvirt` and `/etc/libvirt` directories to the `sysinfo/libvirt/log` and `sysinfo/libvirt/etc` directories respectively. Only the files whose name follows the regex `*.log` are copied from the `/var/log/libvirt` directory.
 
 ## lstopo.txt
 
 The file provides information about the topology of the system.
 
-The data is collected using [pbench-sysinfo-dump#L71](https://github.com/distributed-system-analysis/pbench/blob/master/agent/util-scripts/pbench-sysinfo-dump#L71). The script executes the command `lstopo --of txt` and dumps its output into a text file only if `/usr/bin/lstopo` file exists on the system.
+The data is collected using [pbench-sysinfo-dump#L77](https://github.com/distributed-system-analysis/pbench/blob/main/agent/util-scripts/tool-meister/pbench-sysinfo-dump#L77). The script executes the command `lstopo --of txt` and dumps its output into a text file only if `/usr/bin/lstopo` file exists on the system.
 
 ## block-params.log
 
 The file provides information about block devices.
 
-The data is collected using [pbench-sysinfo-dump#L79](https://github.com/distributed-system-analysis/pbench/blob/master/agent/util-scripts/pbench-sysinfo-dump#L79). The script loops over all the files which satisfy the regex: `/sys/block/[s,h,v]d\*[a-z]/` and prints each file name along with the contents of the file.
+The data is collected using [pbench-sysinfo-dump#L84](https://github.com/distributed-system-analysis/pbench/blob/main/agent/util-scripts/tool-meister/pbench-sysinfo-dump#L84). The script loops over all the files which satisfy the regex: `/sys/block/[s,h,v]d\*[a-z]/` and prints each file name along with the contents of the file.
 
 The file contains data in a key value format where the key is the file name and the value is the content of the file..
 
 ## sosreport tarball (e.g. sosreport-localhost-localhost-pbench-2019-05-29-rtvzlke.tar.xz)
 
 The tarball contains system configuration and diagnostic information collected by invoking the `sosreport` command.
 
-The data is collected using [pbench-sysinfo-dump#L87](https://github.com/distributed-system-analysis/pbench/blob/master/agent/util-scripts/pbench-sysinfo-dump#L87). The script uses `sosreport` command with different plugins to get the required system information. The resulting tarball contains a number of files copied from the system as well as the output of several commands executed on the system.
-
-## ara
-
-This specific file is not in the scope of my internship because ara works with `python2` and Fedora 30, which is installed on my system, works with `python3` only.
+The data is collected using [pbench-sysinfo-dump#L91](https://github.com/distributed-system-analysis/pbench/blob/main/agent/util-scripts/tool-meister/pbench-sysinfo-dump#L91). The script uses `sosreport` command with different plugins to get the required system information. The resulting tarball contains a number of files copied from the system as well as the output of several commands executed on the system.
 
 ## insights tarball
 
 The tarball contains system information gathered by the [insights-client](https://github.com/RedHatInsights/insights-client).
 
-The data is collected using [pbench-sysinfo-dump#L189](https://github.com/distributed-system-analysis/pbench/blob/master/agent/util-scripts/pbench-sysinfo-dump#L189). The script uses `insights-client` command with different options to get the system information. The resulting tarball contains a number of files copied from the system as well as the output of several commands executed on the system.
+The data is collected using [pbench-sysinfo-dump#L188](https://github.com/distributed-system-analysis/pbench/blob/main/agent/util-scripts/tool-meister/pbench-sysinfo-dump#L188). The script uses `insights-client` command with different options to get the system information. The resulting tarball contains a number of files copied from the system as well as the output of several commands executed on the system.