Merge pull request #1992 from nuagenetworks/dev

Dev

Documentation/RELEASE_NOTES.md

@@ -2,21 +2,21 @@
 
 ## Release info
 
-* MetroAE Version 5.4.0
+* MetroAE Version 5.5.0
 * Nuage Release Alignment 20.10.R1
-* Date of Release 22-December-2022
+* Date of Release 21-August-2023
 
 ## Release Contents
 
 ### Feature Enhancements
-* Add Support for TPM server Deployment (METROAE-461)
-* Add lowercase proxy env support (METROAE-662)
-* Remove {{ }} from when clauses in metro (METROAE-643)
+* Add support for multiple external interfaces in NUH (METROAE-670)
 
 ### Resolved Issues
-* Update the DevOps Website (devops.nuagenetworks.net) with the latest MetroAE container information (METROAE-627)
-* Fix internal network option for NUH deployment on OpenStack (METROAE-663)
-* Update the title for nsg_template_name to reflect the correct description (METROAE-501)
+* Fix DNS restart issue for TPM server HA deployment (METROAE-668)
+* Update/Renew VSD ejabberd license for 20.10.Rx version (METROAE-669)
+* VSC R13 installation fails due to ssh hardening (METROAE-672)
+* Fix SD-WAN Portal standalone deployment (METROAE-667)
+* VSTATs(ES) upgrade fails while checking for ES version after the upgrade is complete (METROAE-671)
 
 ## Test Matrix
 

README.md

@@ -1,6 +1,6 @@
 # Nuage Networks Metro Automation Engine (MetroAE)
 
-Version: 5.4.0
+Version: 5.5.0
 
 MetroAE is an automation engine that can be used to
 

deployments/default/upgrade.yml

@@ -41,5 +41,10 @@
 #
 # vsd_preupgrade_db_check_script_path: ""
 
+# < VSD Ejabberd rpm License File >
+# Path to the rpm file for the VSD Patch upgrade to 20.10.R4+ (include the rpm file name itself too)
+#
+# vsd_ejabberd_rpm_file: ""
+
 #############
 

examples/aws_sdwan_upgrade/upgrade.yml

@@ -43,4 +43,9 @@ upgrade_to_version: "5.2.3"
 #
 # vsd_preupgrade_db_check_script_path: ""
 
+# < VSD Ejabberd rpm License File >
+# Path to the rpm file for the VSD Patch upgrade to 20.10.R4+ (include the rpm file name itself too)
+#
+# vsd_ejabberd_rpm_file: ""
+
 #############

examples/blank_deployment/tpms.yml

@@ -55,6 +55,11 @@
     #
     # target_server: ""
 
+    # < TPM Forwarding Port >
+    # TPM port information to be configured on VSD; specifically on 'Server Base URL' parameter along with TPM hostname
+    #
+    # forwarding_port: 0
+
     # < KVM cpuset information >
     # Cpuset information for cpu pinning on KVM. For example, TPM requires 2 cores and sample values will be of the form [1, 2]
     #

examples/blank_deployment/upgrade.yml

@@ -41,5 +41,10 @@
 #
 # vsd_preupgrade_db_check_script_path: ""
 
+# < VSD Ejabberd rpm License File >
+# Path to the rpm file for the VSD Patch upgrade to 20.10.R4+ (include the rpm file name itself too)
+#
+# vsd_ejabberd_rpm_file: ""
+
 #############
 

examples/kvm_vsc_upgrade/upgrade.yml

@@ -43,4 +43,9 @@ upgrade_to_version: "5.2.3"
 #
 # vsd_preupgrade_db_check_script_path: ""
 
+# < VSD Ejabberd rpm License File >
+# Path to the rpm file for the VSD Patch upgrade to 20.10.R4+ (include the rpm file name itself too)
+#
+# vsd_ejabberd_rpm_file: ""
+
 #############

examples/upgrade_datacenter_vcenter/upgrade.yml

@@ -43,4 +43,9 @@ upgrade_to_version: "5.2.3"
 #
 # vsd_preupgrade_db_check_script_path: ""
 
+# < VSD Ejabberd rpm License File >
+# Path to the rpm file for the VSD Patch upgrade to 20.10.R4+ (include the rpm file name itself too)
+#
+# vsd_ejabberd_rpm_file: ""
+
 #############

metroae

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-METROAE_VERSION="5.4.0"
+METROAE_VERSION="5.5.0"
 SCRIPT_VERSION="1.5.2"
 MENU=()
 
@@ -197,7 +197,12 @@ function  check_if_internal_user {
             echo ""
 
             declare -l continue_confirm
-            continue_confirm="init"
+            if [[ -z SKIP_PROMPTS ]]
+            then
+                continue_confirm="init"
+            else
+                continue_confirm=$SKIP_PROMPTS
+            fi
             while [[ $continue_confirm != "y" ]] && [[ $continue_confirm != "n" ]] && [[ $continue_confirm != "" ]]
             do
                 read -p " Would like to share usage data with MetroAE team? (y/N): " continue_confirm

metroae-container

@@ -70,6 +70,7 @@ docker run \
     $TI \
     $SSHMOUNT \
     $VAULTPASS \
+    -e SKIP_PROMPTS \
     --env-file <(env | grep -Ev '^(PWD|PATH|HOME|USER|SHELL|MAIL|SSH_CONNECTION|LOGNAME|OLDPWD|LESSOPEN|_XDG_RUNTIME_DIR|HISTCONTROL)') \
     -v \
     "`pwd`:/metroae" \

schemas/tpms.json

@@ -73,12 +73,18 @@
                 "description": "Hostname or IP address of the hypervisor where VM will be instantiated",
                 "propertyOrder": 80
             },
+            "forwarding_port": {
+                "type": "integer",
+                "title": "TPM Forwarding Port",
+                "description": "TPM port information to be configured on VSD; specifically on 'Server Base URL' parameter along with TPM hostname",
+                "propertyOrder": 90
+            },
             "cpuset": {
                 "type": "array",
                 "title": "KVM cpuset information",
                 "description": "Cpuset information for cpu pinning on KVM. For example, TPM requires 2 cores and sample values will be of the form [1, 2]",
                 "sectionEnd": "VM and Target Server",
-                "propertyOrder": 90,
+                "propertyOrder": 100,
                 "advanced": true,
                 "items": {
                     "type": "integer"
@@ -88,7 +94,7 @@
                 "type": "boolean",
                 "title": "Preserve TPM VM",
                 "description": "Shuts down the vm during destroy process, instead of actually deleting them.",
-                "propertyOrder": 100,
+                "propertyOrder": 110,
                 "advanced": true
             },
             "openstack_image": {
@@ -97,49 +103,49 @@
                 "description": "Name of image installed on OpenStack for VSD",
                 "target_server_type": "openstack",
                 "sectionBegin": "OpenStack Parameters",
-                "propertyOrder": 110
+                "propertyOrder": 120
             },
             "openstack_flavor": {
                 "type": "string",
                 "title": "OpenStack Flavor",
                 "description": "Name of instance flavor installed on OpenStack for VSD",
                 "target_server_type": "openstack",
-                "propertyOrder": 120
+                "propertyOrder": 130
             },
             "openstack_availability_zone": {
                 "type": "string",
                 "title": "OpenStack Availability Zone",
                 "description": "Name of availability zone on OpenStack for VSD",
                 "target_server_type": "openstack",
-                "propertyOrder": 130
+                "propertyOrder": 140
             },
             "openstack_network": {
                 "type": "string",
                 "title": "OpenStack Network",
                 "description": "Name of network on OpenStack for VSD",
                 "target_server_type": "openstack",
-                "propertyOrder": 140
+                "propertyOrder": 150
             },
             "openstack_subnet": {
                 "type": "string",
                 "title": "OpenStack Subnet",
                 "description": "Name of subnet on OpenStack for VSD",
                 "target_server_type": "openstack",
-                "propertyOrder": 150
+                "propertyOrder": 160
             },
             "openstack_port_name": {
                 "type": "string",
                 "title": "OpenStack Port Name",
                 "description": "Name for Mgmt interface",
                 "target_server_type": "openstack",
-                "propertyOrder": 160
+                "propertyOrder": 170
             },
             "openstack_port_security_groups": {
                 "type": "array",
                 "title": "OpenStack Port Security Groups",
                 "description": "Set of security groups to associate with Mgmt interface",
                 "target_server_type": "openstack",
-                "propertyOrder": 170,
+                "propertyOrder": 180,
                 "items": {
                     "type": "string"
                 }
@@ -150,7 +156,7 @@
                 "description": "Project name for OpenStack",
                 "default": "",
                 "target_server_type": "openstack",
-                "propertyOrder": 180
+                "propertyOrder": 190
             },
             "openstack_auth_url": {
                 "type": "string",
@@ -159,7 +165,7 @@
                 "default": "",
                 "target_server_type": "openstack",
                 "sectionEnd": "OpenStack Parameters",
-                "propertyOrder": 190
+                "propertyOrder": 200
             }
         },
         "required": ["hostname", "mgmt_ip", "mgmt_ip_prefix", "mgmt_gateway"]

schemas/upgrade.json

@@ -79,8 +79,17 @@
             "description": "Path on the MetroAE host to the VSD pre-upgrade database check script",
             "default": "",
             "workflow": "Upgrade",
-            "sectionEnd": "Upgrade",
             "propertyOrder": 80
+        },
+        "vsd_ejabberd_rpm_file": {
+            "type": "string",
+            "title": "VSD Ejabberd rpm License File",
+            "default": "",
+            "description": "Path to the rpm file for the VSD Patch upgrade to 20.10.R4+ (include the rpm file name itself too)",
+            "sectionEnd": "Upgrade",
+            "workflow": "Upgrade",
+            "propertyOrder": 90,
+            "advanced": true
         }
     }
 }

src/convert_schemas_to_excel.py

@@ -348,8 +348,8 @@ def add_data_validation(self, worksheet, field, row, col):
     def read_schema(self, schema_name):
         file_name = schema_name + ".json"
         file_path = os.path.join(self.settings["schema_directory"], file_name)
-        with open(file_path, "rb") as f:
-            schema_str = f.read().decode("utf-8")
+        with open(file_path, "r", encoding="utf-8") as f:
+            schema_str = f.read()
 
         try:
             return json.loads(schema_str)
@@ -364,8 +364,8 @@ def read_example(self, example_dir, schema_name):
         file_name = os.path.join(example_dir, schema_name + ".yml")
         if os.path.isfile(file_name):
             try:
-                with open(file_name, "rb") as f:
-                    return yaml.safe_load(f.read().decode("utf-8"))
+                with open(file_name, "r", encoding="utf-8") as f:
+                    return yaml.safe_load(f.read())
             except Exception as e:
                 raise Exception("Could not parse example: %s\n%s" % (
                     file_name, str(e)))

src/deployment_templates/tpms.j2

@@ -75,6 +75,15 @@
     # target_server: ""
     {%- endif %}
 
+    # < TPM Forwarding Port >
+    # TPM port information to be configured on VSD; specifically on 'Server Base URL' parameter along with TPM hostname
+    #
+    {%- if item.forwarding_port is defined %}
+    forwarding_port: {{ item.forwarding_port }}
+    {%- else %}
+    # forwarding_port: 0
+    {%- endif %}
+
     # < KVM cpuset information >
     # Cpuset information for cpu pinning on KVM. For example, TPM requires 2 cores and sample values will be of the form [1, 2]
     #

src/deployment_templates/upgrade.j2

@@ -71,5 +71,14 @@ vsd_preupgrade_db_check_script_path: "{{ vsd_preupgrade_db_check_script_path }}"
 # vsd_preupgrade_db_check_script_path: ""
 {%- endif %}
 
+# < VSD Ejabberd rpm License File >
+# Path to the rpm file for the VSD Patch upgrade to 20.10.R4+ (include the rpm file name itself too)
+#
+{%- if vsd_ejabberd_rpm_file is defined %}
+vsd_ejabberd_rpm_file: "{{ vsd_ejabberd_rpm_file }}"
+{%- else %}
+# vsd_ejabberd_rpm_file: ""
+{%- endif %}
+
 #############
 

src/generate_all_from_schemas.sh

@@ -78,16 +78,16 @@ for dir in $EXAMPLE_DATA_DIR/*/; do
     done
 
     mkdir -p $EXAMPLE_DIR/excel
-    python src/convert_schemas_to_excel.py $EXAMPLE_DIR/excel/${example_name}.xlsx $dir
+    python3 src/convert_schemas_to_excel.py $EXAMPLE_DIR/excel/${example_name}.xlsx $dir
 done
 
 echo "Generate deployment spreadsheet template CSV"
 python src/generate_deployment_spreadsheet_template.py > deployment_spreadsheet_template.csv
 
 
 echo "Generate deployment excel spreadsheet template"
-python src/convert_schemas_to_excel.py sample_deployment.xlsx
-python src/convert_schemas_to_excel.py $EXAMPLE_DIR/excel/blank_deployment.xlsx
+python3 src/convert_schemas_to_excel.py sample_deployment.xlsx
+python3 src/convert_schemas_to_excel.py $EXAMPLE_DIR/excel/blank_deployment.xlsx
 
 echo "Generate deployment spreadsheet example"
 mkdir $CSV_DEPLOYMENT_DIR

src/playbooks/with_build/vsd_stats_deploy.yml

@@ -9,12 +9,6 @@
         name: enable-stats-out
         tasks_from: main
 
-    - name: Apply inplace patch if needed 
-      include_role:
-        name: vsd-upgrade-inplace
-        tasks_from: main.yml
-      when: vsd_install_inplace_upgrade | default(False)
-
     - include_role:
         name: vsd-deploy
         tasks_from: vsd_security_hardening.yml

src/roles/common/tasks/vstat-enable-stats.yml

@@ -107,6 +107,12 @@
           timeout_seconds: 1200
           test_interval_seconds: 30
 
+      - name: Upgrade stats vsds
+        include_role:
+          name: vsd-upgrade-inplace
+          tasks_from: main.yml
+        when: vsd_install_inplace_upgrade | default (False)
+
       - name: Verify port 39039 is open for listening in stats-out deployment
         shell: lsof -i:39090 | grep LISTEN
 

src/roles/common/templates/group_vars.all.j2

@@ -28,7 +28,7 @@ webfilter_ram: "{{ common.webfilter_ram | default(8) }}"
 tpm_ram: "{{ common.tpm_ram | default(8) }}"
 portal_ram: "{{ common.portal_ram | default(24) }}"
 portal_license_file: "{{ common.portal_license_file | default('NONE') }}"
-portal_allocate_size_gb: "{{ portal_allocate_size_gb | default(16) }}"
+portal_allocate_size_gb: "{{ common.portal_allocate_size_gb | default(16) }}"
 vsd_license_required_days_left: {{ common.vsd_license_required_days_left | default(-1) }}
 vsd_fqdn: {{ common.vsd_fqdn_global | default('NONE') }}
 vsd_port: {{ common.vsd_port_global | default('8443') }}
@@ -350,6 +350,10 @@ vsd_cluster_license_file: {{ common.vsd_cluster_license_file | relative_path_to_
 vsd_ejabberd_license_file: {{ common.vsd_ejabberd_license_file | relative_path_to_absolute_path  }}
 {% endif %}
 
+{% if upgrade.vsd_ejabberd_rpm_file is defined %}
+vsd_ejabberd_rpm_file: {{ upgrade.vsd_ejabberd_rpm_file | relative_path_to_absolute_path  }}
+{% endif %}
+
 {% if common.nuh_license_file is defined %}
 nuh_license_file: {{ common.nuh_license_file | relative_path_to_absolute_path  }}
 {% endif %}

src/roles/common/templates/tpm.j2

@@ -40,3 +40,5 @@ tpm_sa_or_ha: sa
 {% else %}
 tpm_sa_or_ha: ha
 {% endif %}
+
+forwarding_port: {{ item.forwarding_port | default(8443) }}

src/roles/nuh-deploy/tasks/nuh_copy_certificates.yml

@@ -92,8 +92,8 @@
       - name: Check if the eth1 is configured on Internal interface
         shell: set -o pipefail && ip netns exec Internal ip addr list | grep eth1
 
-      - name: Check if the eth2 is configured on External interfaces
-        shell: set -o pipefail && ip netns exec {{ item.name }} ip addr list | grep eth2
+      - name: Check if the eth2/eth3 is configured on External interfaces
+        shell: set -o pipefail && ip netns exec {{ item.name }} ip addr list | grep {{ item.dev }}
         with_items: "{{ external_interfaces }}"
         when: external_interfaces is defined and external_interfaces | length > 0
 

src/roles/sdwan-portal-deploy/templates/config.j2

@@ -3,6 +3,7 @@ advanced.client.timeout: '180'
 advanced.es.timeout: '60000'
 vnsportal.default.ui: 1
 {% if portal_sa_or_ha == 'sa' %}
+gr.enabled: false
 ha.enabled: false
 ha.node1.address: {{ portal1.mgmt_ip }}
 ha.node2.address: ''