MGMT-1112 Test-infra should support running on RHEL, Fedora and non r…

…oot user

Makefile

@@ -39,7 +39,7 @@ REMOTE_INVENTORY_URL := $(or $(REMOTE_INVENTORY_URL), "")
 all:
 	./install_env_and_run_full_flow.sh
 
-destroy: destroy_nodes delete_minikube kill_all_port_forwardings
+destroy: destroy_nodes delete_minikube
 	rm -rf build/terraform/*
 
 ###############
@@ -71,7 +71,7 @@ start_minikube:
 
 delete_minikube:
 	minikube delete
-	/usr/local/bin/skipper run discovery-infra/virsh_cleanup.py -m
+	skipper run discovery-infra/virsh_cleanup.py -m
 
 #############
 # Terraform #
@@ -86,14 +86,14 @@ run_terraform_from_skipper:
 		cd build/terraform/ && terraform init  -plugin-dir=/root/.terraform.d/plugins/ && terraform apply -auto-approve -input=false -state=terraform.tfstate -state-out=terraform.tfstate -var-file=terraform.tfvars.json
 
 run_terraform: copy_terraform_files
-	/usr/local/bin/skipper make run_terraform_from_skipper $(SKIPPER_PARAMS)
+	skipper make run_terraform_from_skipper $(SKIPPER_PARAMS)
 
 _destroy_terraform:
 	cd build/terraform/  && terraform destroy -auto-approve -input=false -state=terraform.tfstate -state-out=terraform.tfstate -var-file=terraform.tfvars.json || echo "Failed cleanup terraform"
 	discovery-infra/virsh_cleanup.py -f test-infra
 
 destroy_terraform:
-	/usr/local/bin/skipper make _destroy_terraform $(SKIPPER_PARAMS)
+	skipper make _destroy_terraform $(SKIPPER_PARAMS)
 
 #######
 # Run #
@@ -126,7 +126,7 @@ _install_cluster:
 	discovery-infra/install_cluster.py -id $(CLUSTER_ID) -ps '$(PULL_SECRET)'
 
 install_cluster:
-	/usr/local/bin/skipper make _install_cluster $(SKIPPER_PARAMS)
+	skipper make _install_cluster $(SKIPPER_PARAMS)
 
 
 #########
@@ -137,13 +137,13 @@ _deploy_nodes:
 	discovery-infra/start_discovery.py -i $(IMAGE) -n $(NUM_MASTERS) -p $(STORAGE_POOL_PATH) -k '$(SSH_PUB_KEY)' -mm $(MASTER_MEMORY) -wm $(WORKER_MEMORY) -nw $(NUM_WORKERS) -ps '$(PULL_SECRET)' -bd $(BASE_DOMAIN) -cN $(CLUSTER_NAME) -vN $(NETWORK_CIDR) -nN $(NETWORK_NAME) -nB $(NETWORK_BRIDGE) -ov $(OPENSHIFT_VERSION) -rv $(RUN_WITH_VIPS) -iU $(REMOTE_INVENTORY_URL) -id $(CLUSTER_ID) $(ADDITIONAL_PARAMS)
 
 deploy_nodes_with_install:
-	/usr/local/bin/skipper make _deploy_nodes ADDITIONAL_PARAMS=-in $(SKIPPER_PARAMS)
+	skipper make _deploy_nodes ADDITIONAL_PARAMS=-in $(SKIPPER_PARAMS)
 
 deploy_nodes:
-	/usr/local/bin/skipper make _deploy_nodes $(SKIPPER_PARAMS)
+	skipper make _deploy_nodes $(SKIPPER_PARAMS)
 
 destroy_nodes:
-	/usr/local/bin/skipper run 'discovery-infra/delete_nodes.py -iU $(REMOTE_INVENTORY_URL) -id $(CLUSTER_ID)' $(SKIPPER_PARAMS)
+	skipper run 'discovery-infra/delete_nodes.py -iU $(REMOTE_INVENTORY_URL) -id $(CLUSTER_ID)' $(SKIPPER_PARAMS)
 
 redeploy_nodes: destroy_nodes deploy_nodes
 
@@ -170,7 +170,7 @@ create_inventory_client: bring_bm_inventory
 	$(CONTAINER_COMMAND) run -it --rm -u $(CURRENT_USER) -v $(PWD)/build:/swagger-api/out -v $(PWD)/bm-inventory/swagger.yaml:/swagger.yaml:ro,Z -v $(PWD)/build/code-gen-config.json:/config.json:ro,Z jimschubert/swagger-codegen-cli:2.3.1 generate --lang python --config /config.json --output ./bm-inventory-client/ --input-spec /swagger.yaml
 
 delete_all_virsh_resources: destroy_nodes delete_minikube
-	/usr/local/bin/skipper run 'discovery-infra/delete_nodes.py -a' $(SKIPPER_PARAMS)
+	skipper run 'discovery-infra/delete_nodes.py -a' $(SKIPPER_PARAMS)
 
 build_and_push_image: create_inventory_client
 	$(CONTAINER_COMMAND) build -t $(IMAGE_NAME):$(IMAGE_TAG) -f Dockerfile.test-infra .
@@ -185,7 +185,7 @@ _download_iso:
 	discovery-infra/start_discovery.py -k '$(SSH_PUB_KEY)'  -ps '$(PULL_SECRET)' -bd $(BASE_DOMAIN) -cN $(CLUSTER_NAME) -ov $(OPENSHIFT_VERSION) -pU $(PROXY_URL) -iU $(REMOTE_INVENTORY_URL) -id $(CLUSTER_ID) -iO
 
 download_iso:
-	/usr/local/bin/skipper make _download_iso $(SKIPPER_PARAMS)
+	skipper make _download_iso $(SKIPPER_PARAMS)
 
 download_iso_for_remote_use: deploy_bm_inventory
-	/usr/local/bin/skipper make _download_iso $(SKIPPER_PARAMS)
+	skipper make _download_iso $(SKIPPER_PARAMS)

create_full_environment.sh

@@ -6,24 +6,8 @@ function error () {
     echo $@ 1>&2
 }
 
-# Check OS
-OS=$(awk -F= '/^ID=/ { print $2 }' /etc/os-release | tr -d '"')
-if [[ ! ${OS} =~ ^(centos)$ ]]; then
-  error "\"${OS}\" is an unsupported OS. We support only CentOS."
-  exit 1
-fi
-
-#Check CentOS version
-VER=$(awk -F= '/^VERSION_ID=/ { print $2 }' /etc/os-release | tr -d '"' | cut -f1 -d'.')
-VER_SUPPORTED=8
-
-if [[ ${VER} -ne ${VER_SUPPORTED} ]]; then
-  error "CentOS version ${VER_SUPPORTED} is required."
-  exit 1
-fi
-
 echo "Installing environment"
-sudo scripts/install_environment.sh
+scripts/install_environment.sh
 echo "Done installing"
 
 echo "Creating image"

discovery-infra/bm_inventory_api.py

@@ -132,5 +132,6 @@ def create_client(inventory_url=None, wait_for_url=True):
         i_url = utils.get_service_url("bm-inventory")
     log.info("Inventory URL %s", i_url)
     client = InventoryClient(inventory_url=i_url)
-    client.wait_for_api_readiness()
+    if not wait_for_url:
+        client.wait_for_api_readiness()
     return client

discovery-infra/consts.py

@@ -2,7 +2,8 @@
 
 TF_FOLDER = "build/terraform"
 TFVARS_JSON_FILE = os.path.join(TF_FOLDER, "terraform.tfvars.json")
-IMAGE_PATH = "/tmp/installer-image.iso"
+IMAGE_FOLDER = "/tmp/images"
+IMAGE_PATH = "%s/installer-image.iso" % IMAGE_FOLDER
 STORAGE_PATH = "/var/lib/libvirt/openshift-images"
 SSH_KEY = "ssh_key/key.pub"
 NODES_REGISTERED_TIMEOUT = 180

discovery-infra/start_discovery.py

@@ -62,13 +62,14 @@ def create_nodes_and_wait_till_registered(inventory_client, cluster, image_path,
     create_nodes(image_path, storage_path=storage_path, master_count=master_count, nodes_details=nodes_details)
 
     # TODO: Check for only new nodes
-    utils.wait_till_nodes_are_ready(nodes_count=nodes_count, cluster_name=nodes_details["cluster_name"])
+    utils.wait_till_nodes_are_ready(nodes_count=nodes_count, network_name=nodes_details["libvirt_network_name"])
     if not inventory_client:
         log.info("No inventory url, will not wait till nodes registration")
         return
 
     log.info("Wait till nodes will be registered")
-    waiting.wait(lambda: utils.are_all_libvirt_nodes_in_cluster_hosts(inventory_client, cluster.id),
+    waiting.wait(lambda: utils.are_all_libvirt_nodes_in_cluster_hosts(inventory_client, cluster.id,
+                                                                      nodes_details["libvirt_network_name"]),
                  timeout_seconds=consts.NODES_REGISTERED_TIMEOUT,
                  sleep_seconds=10, waiting_for="Nodes to be registered in inventory service")
     log.info("Registered nodes are:")
@@ -77,9 +78,9 @@ def create_nodes_and_wait_till_registered(inventory_client, cluster, image_path,
 
 # Set nodes roles by vm name
 # If master in name -> role will be master, same for worker
-def set_hosts_roles(client, cluster_id):
+def set_hosts_roles(client, cluster_id, network_name):
     added_hosts = []
-    libvirt_nodes = utils.get_libvirt_nodes_mac_role_ip_and_name()
+    libvirt_nodes = utils.get_libvirt_nodes_mac_role_ip_and_name(network_name)
     inventory_hosts = client.get_cluster_hosts(cluster_id)
 
     for libvirt_mac, libvirt_metadata in libvirt_nodes.items():
@@ -147,7 +148,7 @@ def nodes_flow(client, cluster_name, cluster):
                                           nodes_details=nodes_details)
     if client:
         cluster_info = client.cluster_get(cluster.id)
-        macs = utils.get_libvirt_nodes_macs()
+        macs = utils.get_libvirt_nodes_macs(nodes_details["libvirt_network_name"])
 
         if not (cluster_info.api_vip and cluster_info.ingress_vip):
             utils.wait_till_hosts_with_macs_are_in_status(client=client, cluster_id=cluster.id, macs=macs,
@@ -156,7 +157,7 @@ def nodes_flow(client, cluster_name, cluster):
         else:
             log.info("VIPs already configured")
 
-        set_hosts_roles(client, cluster.id)
+        set_hosts_roles(client, cluster.id, nodes_details["libvirt_network_name"])
         utils.wait_till_hosts_with_macs_are_in_status(client=client, cluster_id=cluster.id, macs=macs,
                                                       statuses=[consts.NodesStatus.KNOWN])
         log.info("Printing after setting roles")
@@ -173,9 +174,9 @@ def main():
     client = None
     cluster = {}
     cluster_name = args.cluster_name or consts.CLUSTER_PREFIX + str(uuid.uuid4())[:8]
-
     # If image is passed, there is no need to create cluster and download image, need only to spawn vms with is image
     if not args.image:
+        utils.recreate_folder(consts.IMAGE_FOLDER)
         client = bm_inventory_api.create_client(args.inventory_url)
         if args.cluster_id:
             cluster = client.cluster_get(cluster_id=args.cluster_id)

discovery-infra/utils.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import itertools
 import subprocess
 from pathlib import Path
@@ -8,8 +9,9 @@
 from retry import retry
 import consts
 from logger import log
+import libvirt
 
-VIRSH_LEASES_COMMAND = "virsh -q net-dhcp-leases"
+conn = libvirt.open('qemu:///system')
 
 
 def run_command(command, shell=False):
@@ -43,46 +45,47 @@ def get_service_url(service_name):
         raise
 
 
-def wait_till_nodes_are_ready(nodes_count, cluster_name):
+def get_network_leases(network_name):
+    net = conn.networkLookupByName(network_name)
+    return net.DHCPLeases()
+
+
+def wait_till_nodes_are_ready(nodes_count, network_name):
     log.info("Wait till %s nodes will be ready and have ips", nodes_count)
-    cmd = "%s %s| grep %s | wc -l" % (VIRSH_LEASES_COMMAND, consts.TEST_NETWORK, cluster_name)
     try:
-        waiting.wait(lambda: int(run_command(cmd, shell=True).strip()) >= nodes_count,
+        waiting.wait(lambda: len(get_network_leases(network_name)) >= nodes_count,
                      timeout_seconds=consts.NODES_REGISTERED_TIMEOUT * nodes_count,
                      sleep_seconds=10, waiting_for="Nodes to have ips")
         log.info("All nodes have booted and got ips")
     except:
-        cmd = "%s %s" % (VIRSH_LEASES_COMMAND, consts.TEST_NETWORK)
-        log.error("Not all nodes are ready. Current dhcp leases are %s", run_command(cmd, shell=False).strip())
+        log.error("Not all nodes are ready. Current dhcp leases are %s", get_network_leases)
         raise
 
 
 # Require wait_till_nodes_are_ready has finished and all nodes are up
-def get_libvirt_nodes_mac_role_ip_and_name():
-    cmd = "%s %s" % (VIRSH_LEASES_COMMAND, consts.TEST_NETWORK)
+def get_libvirt_nodes_mac_role_ip_and_name(network_name):
     nodes_data = {}
     try:
-        output = run_command(cmd, shell=False).splitlines()
-        for node in output:
-            nic_data = node.split()
-            nodes_data[nic_data[2].lower()] = {"ip": nic_data[4].split("/")[0],
-                                               "name": nic_data[5],
-                                               "role": consts.NodeRoles.WORKER if
-                                               consts.NodeRoles.WORKER in nic_data[5] else consts.NodeRoles.MASTER}
+        leases = get_network_leases(network_name)
+        for lease in leases:
+            nodes_data[lease["mac"]] = {"ip": lease["ipaddr"],
+                                        "name": lease["hostname"],
+                                        "role": consts.NodeRoles.WORKER if
+                                        consts.NodeRoles.WORKER in lease["hostname"] else consts.NodeRoles.MASTER}
         return nodes_data
     except:
-        cmd = "%s %s" % (VIRSH_LEASES_COMMAND, consts.TEST_NETWORK)
-        log.error("Failed to get nodes macs from libvirt. Output is %s", run_command(cmd, shell=False))
+        log.error("Failed to get nodes macs from libvirt. Output is %s", get_network_leases(network_name))
         raise
 
 
-def get_libvirt_nodes_macs():
-    return get_libvirt_nodes_mac_role_ip_and_name().keys()
+def get_libvirt_nodes_macs(network_name):
+    return get_libvirt_nodes_mac_role_ip_and_name(network_name).keys()
 
 
-def are_all_libvirt_nodes_in_cluster_hosts(client, cluster_id):
+def are_all_libvirt_nodes_in_cluster_hosts(client, cluster_id, network_name):
     hosts_macs = client.get_hosts_id_with_macs(cluster_id)
-    return all(mac.lower() in map(str.lower, itertools.chain(*hosts_macs.values())) for mac in get_libvirt_nodes_macs())
+    return all(mac.lower() in
+               map(str.lower, itertools.chain(*hosts_macs.values())) for mac in get_libvirt_nodes_macs(network_name))
 
 
 def get_cluster_hosts_with_mac(client, cluster_id, macs):
@@ -165,3 +168,15 @@ def folder_exists(file_path):
 
 def file_exists(file_path):
     return Path(file_path).exists()
+
+
+def create_folder_if_not_exists(folder):
+    if not os.path.exists(folder):
+        os.makedirs(folder)
+
+
+def recreate_folder(folder):
+    if os.path.exists(folder):
+        shutil.rmtree(folder)
+    create_folder_if_not_exists(folder)
+    run_command("chmod ugo+rx %s" % folder)

discovery-infra/virsh_cleanup.py

@@ -16,43 +16,43 @@ def run_command(command, check=False, resource_filter=None):
 
 
 def clean_domains(skip_list, resource_filter):
-    domains = run_command("virsh list --all --name", resource_filter=resource_filter)
+    domains = run_command("virsh -c qemu:///system list --all --name", resource_filter=resource_filter)
     domains = domains.splitlines()
     for domain in domains:
         log.info("Deleting domain %s", domain)
         if domain and domain not in skip_list:
-            run_command("virsh destroy %s" % domain, check=False)
-            run_command("virsh undefine %s" % domain, check=False)
+            run_command("virsh -c qemu:///system destroy %s" % domain, check=False)
+            run_command("virsh -c qemu:///system undefine %s" % domain, check=False)
 
 
 def clean_volumes(pool):
-    volumes_with_path = run_command("virsh vol-list %s | tail -n +3" % pool).splitlines()
+    volumes_with_path = run_command("virsh -c qemu:///system vol-list %s | tail -n +3" % pool).splitlines()
     for volume_with_path in volumes_with_path:
         volume, _ = volume_with_path.split()
         if volume:
             log.info("Deleting volume %s in pool %s", volume, pool)
-            run_command("virsh vol-delete --pool %s %s" % (pool, volume), check=False)
+            run_command("virsh -c qemu:///system vol-delete --pool %s %s" % (pool, volume), check=False)
 
 
 def clean_pools(skip_list, resource_filter):
-    pools = run_command("virsh pool-list --all --name", resource_filter=resource_filter)
+    pools = run_command("virsh -c qemu:///system pool-list --all --name", resource_filter=resource_filter)
     pools = pools.splitlines()
     for pool in pools:
         if pool and pool not in skip_list:
             clean_volumes(pool)
             log.info("Deleting pool %s", pool)
-            run_command("virsh pool-destroy %s" % pool, check=False)
-            run_command("virsh pool-undefine %s" % pool, check=False)
+            run_command("virsh -c qemu:///system pool-destroy %s" % pool, check=False)
+            run_command("virsh -c qemu:///system pool-undefine %s" % pool, check=False)
 
 
 def clean_networks(skip_list, resource_filter):
-    networks = run_command("virsh net-list --all --name", resource_filter=resource_filter)
+    networks = run_command("virsh -c qemu:///system net-list --all --name", resource_filter=resource_filter)
     networks = networks.splitlines()
     for net in networks:
         if net and net not in skip_list:
             log.info("Deleting network %s", net)
-            run_command("virsh net-destroy %s" % net, check=False)
-            run_command("virsh net-undefine %s" % net, check=False)
+            run_command("virsh -c qemu:///system net-destroy %s" % net, check=False)
+            run_command("virsh -c qemu:///system net-undefine %s" % net, check=False)
 
 
 def clean_virsh_resources(skip_list, resource_filter):

scripts/assisted_deployment.sh

@@ -10,7 +10,7 @@ function set_dns() {
   if ! [ -f "$FILE" ]; then
     echo -e "[main]\ndns=dnsmasq" | sudo tee $FILE
   fi
-  echo "" > /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf
+  sudo truncate -s0 /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf
   echo "server=/api.${CLUSTER_NAME}.${BASE_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf
   sudo systemctl reload NetworkManager
 }

scripts/deploy_bm_inventory.sh

@@ -3,27 +3,23 @@ set -euo pipefail
 
 source scripts/utils.sh
 
-export INVENTORY_URL=$(get_main_ip)
-export INVENTORY_PORT=${INVENTORY_PORT:-6000}
+
 export KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config}
+export NO_EXTERNAL_PORT=${NO_EXTERNAL_PORT:-n}
 export SERVICE_NAME=bm-inventory
+export INVENTORY_URL=$(get_main_ip)
+export INVENTORY_PORT=${INVENTORY_PORT:-6000}
 
-print_log "Config firewall"
-sudo systemctl start firewalld
-sudo firewall-cmd --zone=public --permanent --add-port=${INVENTORY_PORT}/tcp
-sudo firewall-cmd --zone=libvirt --permanent --add-port=${INVENTORY_PORT}/tcp
-sudo firewall-cmd --reload
+mkdir -p build
 
 print_log "Updating bm_inventory params"
-/usr/local/bin/skipper run discovery-infra/update_bm_inventory_cm.py
-/usr/local/bin/skipper run	"make -C bm-inventory/ deploy-all" ${SKIPPER_PARAMS} DEPLOY_TAG=${DEPLOY_TAG}
+skipper run discovery-infra/update_bm_inventory_cm.py
+skipper run	"make -C bm-inventory/ deploy-all" ${SKIPPER_PARAMS} DEPLOY_TAG=${DEPLOY_TAG}
 
 print_log "Wait till ${SERVICE_NAME} api is ready"
 wait_for_url_and_run "$(minikube service ${SERVICE_NAME} --url -n assisted-installer)" "echo \"waiting for ${SERVICE_NAME}\""
 
 print_log "Starting port forwarding for deployment/${SERVICE_NAME}"
-
 wait_for_url_and_run "http://${INVENTORY_URL}:${INVENTORY_PORT}" "spawn_port_forwarding_command ${SERVICE_NAME} ${INVENTORY_PORT}"
-
 print_log "${SERVICE_NAME} can be reached at http://${INVENTORY_URL}:${INVENTORY_PORT} "
 print_log "Done"