vwan_nva.azcli

# Variables (jomore AIRS)
subscription_id=e7da9914-9b05-4891-893c-546cb7b0422e
vendor="Jose_generic_test_nva"
version=latest
deploy_mode=rest        # arm and rest supported at this time
vpn_endpoint=public     # if "private", deploys ER to connect to onprem too. Use "public" if you dont have access to ER

# Variables (common)
rg=josenva
location=westcentralus
vwan_name=josevwan
vhub_name=josehub
vhub_space=192.168.0.0/16
nva_name=josenva
nva_location=westcentralus
nva_asn=65100
gnva_cloudinit="/tmp/nva-cloudinit.txt"
username=$(whoami)
public_ssh_key=$(more ~/.ssh/id_rsa.pub)

# Select subscription (only whitelisted subs will work)
az account set -s $subscription_id

# Create RG, VWAN and Vhub
rg_location=$(az group show -n $rg --query id -o tsv)
if [[ -n "$rg_location" ]]
then
      echo "Resource group $rg found in $rg_location"
else
      echo "Creating new resource group $rg in $location..."
      az group create -n $rg -l $location
fi
vwan_id=$(az network vwan show -n $vwan_name -g $rg --query name -o tsv)
if [[ -n "$vwan_id" ]]
then
      echo "Virtual WAN $vwan_name found in resource group $rg"
else
      echo "Creating new Virtual WAN in resource group $rg..."
      az network vwan create -n $vwan_name -l $location -g $rg
fi
hub_id=$(az network vhub show -n $vhub_name -g $rg --query id -o tsv)
if [[ -n "hub_id" ]]
then
      echo "Virtual Hub $vhub_name found in Virtual WAN $vwan_name"
else
      echo "Creating new Virtual Hub in VWAN $vwan in $location..."
      az network vhub create --vwan $vwan_name -n $vhub_name --address-prefix $vhub_space -l $location -g $rg
fi

# Create cloudinit file
cat <<EOF > $gnva_cloudinit
#cloud-config
users:
  - default
  - name: $username
    sudo: ["ALL=(ALL) NOPASSWD:ALL"]
    ssh-authorized-keys:
     - $public_ssh_key
packages:
  - jq
runcmd:
  - apt update
  - UCF_FORCE_CONFOLD=1 DEBIAN_FRONTEND=noninteractive apt install -y bird strongswan
  - sysctl -w net.ipv4.ip_forward=1
  - sysctl -w net.ipv4.conf.all.accept_redirects = 0 
  - sysctl -w net.ipv4.conf.all.send_redirects = 0
EOF
cloudinit_string=$(cat $gnva_cloudinit | python3 -c 'import json, sys; print( json.dumps( sys.stdin.read() ) )')

########################
#   Helper functions   #
########################

# Wait for resource to be created
function wait_until_finished {
     wait_interval=15
     resource_id=$1
     resource_name=$(echo $resource_id | cut -d/ -f 9)
     echo "Waiting for resource $resource_name to finish provisioning..."
     start_time=`date +%s`
     state=$(az resource show --id $resource_id --query properties.provisioningState -o tsv)
     until [[ "$state" == "Succeeded" ]] || [[ "$state" == "Failed" ]] || [[ -z "$state" ]]
     do
        sleep $wait_interval
        state=$(az resource show --id $resource_id --query properties.provisioningState -o tsv)
     done
     if [[ -z "$state" ]]
     then
        echo "Something really bad happened..."
     else
        run_time=$(expr `date +%s` - $start_time)
        ((minutes=${run_time}/60))
        ((seconds=${run_time}%60))
        echo "Resource $resource_name provisioning state is $state, wait time $minutes minutes and $seconds seconds"
     fi
}

#######################
# Deploy NVA with CLI #
#######################

# Not working!!!!
# az network virtual-appliance create -n $nva_name -g $rg \
#      --scale-unit 2 \
#      --vendor $vendor \
#      --version $version \
#      --vhub $hub_id \
#      --asn $nva_asn \
#      --init-config $cloudinit_string \
#      --tags tagExample="xyz"

# Diagnostics
# az network virtual-appliance list --query '[].{Name:name,ResourceGroup:resourceGroup,ProvisioningState:provisioningState,AddressPrefix:addressPrefix}' -o table
# pip1=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[1].publicIpAddress' -o tsv) && echo $pip1
# pip2=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[3].publicIpAddress' -o tsv) && echo $pip2
# az network virtual-appliance show -n $nva_name -g $rg --query cloudInitConfiguration -o tsv
# nva_id=$(az network virtual-appliance show -n $nva_name -g $rg --query id -o tsv)

# SSH
# ssh -n -o StrictHostKeyChecking=no -o BatchMode=yes "$pip2" "ip a"

# Cleanup
# az network virtual-appliance delete -n $nva_name -g $rg -y

######################
# Powershell command #
######################

# $rg = "josenva"
# $nva_name = "josenva-pwsh"
# $nva_location = "westcentralus"
# $vhub_name = "josehub"
# $vhub_id = $(get-azvirtualhub -resourcegroupname $rg -name $vhub_name).id
# $cloudinit_string = "#!/bin/bash\nuseradd -m jose\necho \"jose:Microsoft123!\" | chpasswd \n"
# New-AzNetworkVirtualAppliance -Name $nva_name -ResourceGroupName $rg `
#   -CloudInitConfiguration $cloudinit_string `
#   -Location $nva_location
#   -Sku <Microsoft.Azure.Commands.Network.Models.PSVirtualApplianceSkuProperties> -VirtualHubId $vhub_id

#######################
# Deploy NVA with ARM #
#######################

if [[ "$deploy_mode" == "arm" ]]; then
      # Create ARM template and parameters
      arm_template_file=/tmp/nva_template.json
      cat <<'EOF' > $arm_template_file
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
      "name": {
            "defaultValue": "mynva",
            "type": "String"
      },
      "vhub_id": {
            "type": "String"
      },
      "vendor": {
            "defaultValue": "Jose_generic_test_nva",
            "type": "String"
      },
      "version": {
            "defaultValue": "latest",
            "type": "String"
      },
      "cloudinit": {
            "defaultValue": "echo helloworld",
            "type": "String"
      },
      "asn": {
            "defaultValue": 65001,
            "type": "Int"
      }
},
"variables": {},
"resources": [
      {
            "type": "Microsoft.Network/NetworkVirtualAppliances",
            "apiVersion": "2020-11-01",
            "name": "[parameters('name')]",
            "location": "[resourceGroup().location]",
            "tags": {
            "tagexample1": "tagvalue1"
            },
            "properties": {
            "nvaSku": {
                  "vendor": "[parameters('vendor')]",
                  "bundledScaleUnit": "2",
                  "marketPlaceVersion": "[parameters('version')]"
            },
            "virtualHub": {
                  "id": "[parameters('vhub_id')]"
            },
            "virtualApplianceAsn": "[parameters('asn')]",
            "cloudInitConfiguration": "[parameters('cloudinit')]"
            }
      }
]
}
EOF
      arm_params_file=/tmp/nva_params.json
      cat <<EOF > $arm_params_file
{
"\$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
      "name": {
            "value": "$nva_name"
      },
      "vhub_id": {
            "value": "$hub_id"
      },
      "cloudinit": {
            "value": $cloudinit_string
      },
      "asn": {
            "value": $nva_asn
      },
      "vendor": {
            "value": "$vendor"
      },
      "version": {
            "value": "$version"
      },
}
}
EOF

      # Send ARM deployment
      # Option 1: Parameters inline (there seems to be some problem with the cloudinit string, probably with quoting)
      # az deployment group create -n vwannva-$RANDOM --template-file $arm_template_file -g $rg \
      #       --parameters "name=$nva_name" \
      #       --parameters "vhub_id=$hub_id" \
      #       --parameters "cloudinit=$cloudinit_string" \
      #       --parameters "asn=$nva_asn" \
      #       --parameters "version=$version"
      # Option 2: Parameters in file (working fine)
      az deployment group create -n vwannva-$RANDOM --template-file $arm_template_file --parameters @$arm_params_file -g $rg
fi

##############################
# Creating NVA over REST API #
##############################

if [[ "$deploy_mode" == "rest" ]]; then
      # REST payload
      json_payload='{
            "properties": {
            "nvaSku": {
                  "vendor": "'$vendor'",
                  "bundledScaleUnit": "2",
                  "marketPlaceVersion": "'$version'"
            },
            "virtualHub": {
                  "id": "'$hub_id'"
            },
            "virtualApplianceAsn": '$nva_asn',
            "cloudInitConfiguration": '$cloudinit_string'
            },
            "location": "'$location'",
            "tags": {
            "tagexample1": "tagvalue1"
            }
      }'
      uri="/subscriptions/${subscription_id}/resourceGroups/${rg}/providers/Microsoft.Network/NetworkVirtualAppliances/${nva_name}?api-version=2021-02-01"
      az rest --method PUT --uri $uri --body "$json_payload"
fi

############################
# NVA creation diagnostics #
############################

# Wait until provisioning state is Successful
# az network virtual-appliance show -n $nva_name -g $rg --query provisioningState -o tsv

# Wait for NVA to be created
nva_id=$(az network virtual-appliance show -n $nva_name -g $rg --query id -o tsv)
wait_until_finished $nva_id

# Diagnostics
az network virtual-appliance list --query '[].{Name:name,ResourceGroup:resourceGroup,ProvisioningState:provisioningState,AddressPrefix:addressPrefix}' -o table
az network virtual-appliance show -n $nva_name -g $rg --query cloudInitConfiguration -o tsv

# Get PIPs
pip1=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[1].publicIpAddress' -o tsv) && echo $pip1
pip2=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[3].publicIpAddress' -o tsv) && echo $pip2

# Verify SSH
ssh -n -o StrictHostKeyChecking=no -o BatchMode=yes "$pip1" "ip a"
ssh -n -o StrictHostKeyChecking=no -o BatchMode=yes "$pip2" "ip a"

# Cleanup
# az network virtual-appliance delete -n $nva_name -g $rg -y

# Some other REST calls
subscription_id=$(az account show --query id -o tsv)
api_version=2021-02-01
# hub_rg=$(az network vhub list --query '[0].resourceGroup' -o tsv) && echo $hub_rg

# List NVA SKUs (not working: NOT FOUND)
# url="https://management.azure.com/subscriptions/${subscription_id}/resourceGroups/${rg}/providers/Microsoft.Network/networkVirtualApplianceSKUs?api-version=${api_version}" 
# az rest --method GET --url $url

# Show NVA SKU (not working: NOT FOUND)
# url="https://management.azure.com/subscriptions/${subscription_id}/resourceGroups/${rg}/providers/Microsoft.Network/networkVirtualApplianceSKUs/Contoso_generic_test_nva?api-version=${api_version}" 
# az rest --method GET --url $url

# Show NVA
# url="https://management.azure.com/subscriptions/${subscription_id}/resourceGroups/${rg}/providers/Microsoft.Network/networkVirtualAppliances/${nva_name}?api-version=${api_version}" 
# az rest --method GET --url $url

#############################
#  Provision BGP to the RS  #
#############################

# Helper function to calculate the default gateway for a subnet
# Example: default_gw 172.16.1.31 255.255.255.248
function default_gw(){
    IP=$1
    MASK=$2
    IP_HEX=$(printf '%.2X%.2X%.2X%.2X\n' `echo $IP | sed -e 's/\./ /g'`)
    MASK_HEX=$(printf '%.2X%.2X%.2X%.2X\n' `echo $MASK | sed -e 's/\./ /g'`)
    IP_DEC=$(echo "ibase=16; $IP_HEX" | bc)
    MASK_DEC=$(echo "ibase=16; $MASK_HEX" | bc)
    SUBNET_DEC=$(( IP_DEC&MASK_DEC ))
    GW_DEC=$(( $SUBNET_DEC + 1 ))
    GW_HEX=$(printf '%x\n' $GW_DEC)
    GW=$(printf '%d.%d.%d.%d\n' `echo $GW_HEX | sed -r 's/(..)/0x\1 /g'`)
    echo "$GW"
}

# Get RS IPs
rs_ip1=$(az network vhub show -n $vhub_name -g $rg --query 'virtualRouterIps[0]' -o tsv) && echo $rs_ip1
rs_ip2=$(az network vhub show -n $vhub_name -g $rg --query 'virtualRouterIps[1]' -o tsv) && echo $rs_ip2
rs_asn=$(az network vhub show -n $vhub_name -g $rg --query 'virtualRouterAsn' -o tsv) && echo $rs_asn

# Get NVA private IPs
nva_ip1=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[0].privateIpAddress' -o tsv) && echo $nva_ip1
nva_ip2=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[2].privateIpAddress' -o tsv) && echo $nva_ip2

# Find out the gateway for the private ip
nva_default_gw=$(default_gw $nva_ip1 "255.255.255.128") && echo $nva_default_gw

# Static route for the whole hub pointing to the private IPs
ssh -n -o StrictHostKeyChecking=no -o BatchMode=yes "$pip1" "sudo ip route add $vhub_space via $nva_default_gw"
ssh -n -o StrictHostKeyChecking=no -o BatchMode=yes "$pip2" "sudo ip route add $vhub_space via $nva_default_gw"

# Create BGP file
bird_config_file=/tmp/bird.conf
cat <<EOF > $bird_config_file
log syslog all;
#router id $nva_ip1;
protocol device {
        scan time 10;
}
protocol direct {
      disabled;
}
protocol kernel {
      preference 254;
      learn;
      merge paths on;
      export all;
      #disabled;
}
protocol static {
      import all;
      # route $vhub_space via $linuxnva_default_gw;
}
filter DROP_LONG {
      # Drop long prefixes
      if ( net ~ [ 0.0.0.0/0{30,32} ] ) then { reject; }
      else accept;
}
protocol bgp RS1 {
      description "RS1";
      multihop;
      #local $nva_ip1 as $nva_asn;
      local as $nva_asn;
      neighbor $rs_ip1 as $rs_asn;
          import filter {accept;};
          # export filter {accept;};
          export filter DROP_LONG;
}
protocol bgp RS2 {
      description "RS1";
      multihop;
      #local $nva_ip1 as $nva_asn;
      local as $nva_asn;
      neighbor $rs_ip2 as $rs_asn;
          import filter {accept;};
          # export filter {accept;};
          export filter DROP_LONG;
}
EOF

# Deploy file
username=$(whoami)
scp $bird_config_file "${pip1}:/home/${username}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo mv /home/${username}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo systemctl restart bird"
scp $bird_config_file "${pip2}:/home/${username}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo mv /home/${username}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo systemctl restart bird"

# BGP Diagnostics NVA_0
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show protocols RS1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show protocols RS2"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show route"
# BGP Diagnostics NVA_1
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show protocols RS1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show protocol all RS1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show route"

#########################
#   Create spoke VNet   #
#########################

# Variables
vnet_name=spoke1
vnet_prefix=10.1.0.0/16
vm_subnet_name=vm1
vm_subnet_prefix=10.1.1.0/24
vm_size=Standard_B1s

# Create Vnet
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix --subnet-name $vm_subnet_name --subnet-prefix $vm_subnet_prefix

# Create VM
azurevm_name=azurevm
azurevm_pip_name="${azurevm_name}-pip"
az network nsg create -n "${azurevm_name}-nsg" -g $rg
az network nsg rule create -n SSH --nsg-name "${azurevm_name}-nsg" -g $rg --priority 1000 --destination-port-ranges 22 --access Allow --protocol Tcp
az network nsg rule create -n ICMP --nsg-name "${azurevm_name}-nsg" -g $rg --priority 1030 --destination-port-ranges '*' --access Allow --protocol Icmp
az vm create -n $azurevm_name -g $rg -l $location --image ubuntuLTS --generate-ssh-keys --nsg "${azurevm_name}-nsg" \
    --public-ip-address $azurevm_pip_name --vnet-name $vnet_name --size $vm_size --subnet $vm_subnet_name
azurevm_pip_ip=$(az network public-ip show -n $azurevm_pip_name --query ipAddress -o tsv -g $rg) && echo $azurevm_pip_ip
azurevm_nic_id=$(az vm show -n $azurevm_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv) && echo $azurevm_nic_id
azurevm_private_ip=$(az network nic show --ids $azurevm_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $azurevm_private_ip

# Create VMSS
# az vmss create -n vmsstest -g $rg -l $location --image UbuntuLTS --generate-ssh-keys --public-ip-per-vm --lb "" --vnet-name $vnet_name --subnet $vm_subnet_name --nsg "${azurevm_name}-nsg" --vm-sku $vm_size

# Connect to Virtual Hub
hub_default_rt_id=$(az network vhub route-table show --vhub-name $vhub_name -g $rg -n defaultRouteTable --query id -o tsv)
az network vhub connection create -n $vnet_name -g $rg --vhub-name $vhub_name --remote-vnet $vnet_name \
    --internet-security true --associated-route-table $hub_default_rt_id --propagated-route-tables $hub_default_rt_id --labels default

# Diagnostics
az network vhub connection list -g $rg --vhub-name $vhub_name -o table

# New route should be visible in the NVA
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show route"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show route"

# Effective routes of the VM
az network nic show-effective-route-table --ids $azurevm_nic_id -o table

########################
# Create onprem branch #
########################

# Simulation of onprem with a different Azure VNet and NVA

# Variables
onprem_vnet_name=onprem
onprem_vnet_prefix=172.16.0.0/16
onprem_nva_subnet_name=onprem
onprem_nva_subnet_prefix=172.16.1.0/24
onprem_vng_subnet_name=GatewaySubnet
onprem_vng_subnet_prefix=172.16.0.0/24
onprem_ergw_name=onprem-ergw
onprem_ergw_pip=onprem-ergw-pip
vwan_ergw_name=ergw
er_pop=Dallas
er_provider=Megaport
er_circuit_sku=Standard
er_circuit_name="er-$er_pop"

# Create onprem VNet
az network vnet create -n $onprem_vnet_name -g $rg --address-prefixes $onprem_vnet_prefix --subnet-name $onprem_nva_subnet_name --subnet-prefixes $onprem_nva_subnet_prefix

# If using the vpn private endpoint, we need an ER
if [[ $vpn_endpoint == "private" ]]
then
      az network vnet subnet create -g $rg -n $onprem_vng_subnet_name --vnet-name $onprem_vnet_name --address-prefix $onprem_vng_subnet_prefix
      az network public-ip create -g $rg -n $onprem_ergw_pip --allocation-method Dynamic --sku Basic
      az network vnet-gateway create -g $rg -n $onprem_ergw_name --gateway-type ExpressRoute --sku Standard -l $location --vnet $onprem_vnet_name --public-ip-addresses $onprem_ergw_pip --no-wait
      az network express-route gateway create -g $rg -n $vwan_ergw_name --virtual-hub $hub_id
      az network express-route create -n $er_circuit_name --peering-location $er_pop -g $rg \
            --bandwidth 50 Mbps --provider $er_provider -l $location --sku-family MeteredData --sku-tier $er_circuit_sku
      service_key=$(az network express-route show -n $er_circuit_name -g $rg --query serviceKey -o tsv) && echo "Service Key is $service_key"
      # Provision Megaport MCR in location
      script_location="/home/jose/repos/azcli/megaport.sh"
      if [[ -e "$script_location" ]]
      then
            $script_location -s=jomore-${er_pop} -a=create_mcr -k=$service_key
            sleep 60  # Wait 1 minute before creating the connections. This could be replaced with a loop checking ./megaport.sh -a=list_live
            $script_location -s=jomore-${er_pop} -a=create_vxc -k=$service_key
      else
            echo "Sorry, I cannot seem to find the script $script_location to interact with the Megaport API"
      fi
      # Connect circuit to ER gateways (onprem/)
      circuit_id=$(az network express-route show -n $er_circuit_name -g $rg -o tsv --query id) && echo "Circuit ID is $circuit_id"
      circuit_state=$(az network express-route show -n $er_circuit_name -g $rg -o tsv --query serviceProviderProvisioningState) && echo "Circuit state is $circuit_state"
      # Wait until circuit is provisioned
      until [[ "$circuit_state" == "Provisioned" ]]
      do
            sleep 15
            circuit_state=$(az network express-route show -n $er_circuit_name -g $rg -o tsv --query serviceProviderProvisioningState) && echo "Circuit state is $circuit_state"
      done
      peering_id=$(az network express-route peering show -n "AzurePrivatePeering" --circuit-name $er_circuit_name -g $rg -o tsv --query id) && echo "Private peering ID is $peering_id" # For this to work, a Refresh might have to be done...
      hub_default_rt_id=$(az network vhub route-table show --vhub-name $vhub_name -g $rg -n defaultRouteTable --query id -o tsv) && echo $hub_default_rt_id
      az network vpn-connection create -n "${onprem_ergw_name}-${er_pop}" -g $rg -l $location --vnet-gateway1 $onprem_ergw_name --express-route-circuit2 $circuit_id
      az network express-route gateway connection create --gateway-name $vwan_ergw_name -n "${vwan_ergw_name}-${er_pop}" -g $rg --peering $peering_id \
            --associated-route-table $hub_default_rt_id --propagated-route-tables $hub_default_rt_id --labels default
fi

# Create onprem Linux NVA
onprem_linuxnva_asn=65002
onprem_linuxnva_name=onpremnva
onprem_linuxnva_pip=${onprem_linuxnva_name}-pip
onprem_linuxnva_ip=172.16.1.10
linuxnva_cloudinit_file=/tmp/linuxnva_cloudinit.txt
cat <<EOF > $linuxnva_cloudinit_file
#cloud-config
runcmd:
  - apt update && apt install -y bird strongswan
  - sysctl -w net.ipv4.ip_forward=1
  - sysctl -w net.ipv4.conf.all.accept_redirects = 0 
  - sysctl -w net.ipv4.conf.all.send_redirects = 0
EOF
az network nsg create -n "${onprem_linuxnva_name}-nsg" -g $rg
az network nsg rule create -n SSH --nsg-name "${onprem_linuxnva_name}-nsg" -g $rg --priority 1000 --destination-port-ranges 22 --access Allow --protocol Tcp
az network nsg rule create -n IKE --nsg-name "${onprem_linuxnva_name}-nsg" -g $rg --priority 1010 --destination-port-ranges 4500 --access Allow --protocol Udp
az network nsg rule create -n IPsec --nsg-name "${onprem_linuxnva_name}-nsg" -g $rg --priority 1020 --destination-port-ranges 500 --access Allow --protocol Udp
az network nsg rule create -n ICMP --nsg-name "${onprem_linuxnva_name}-nsg" -g $rg --priority 1030 --destination-port-ranges '*' --access Allow --protocol Icmp
az vm create -n $onprem_linuxnva_name -g $rg -l $location --image ubuntuLTS --generate-ssh-keys \
    --public-ip-address $onprem_linuxnva_pip --vnet-name $onprem_vnet_name --size $vm_size --subnet $onprem_nva_subnet_name \
    --custom-data $linuxnva_cloudinit_file --private-ip-address "$onprem_linuxnva_ip" --nsg "${onprem_linuxnva_name}-nsg"
onprem_linuxnva_nic_id=$(az vm show -n $onprem_linuxnva_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic update --ids $onprem_linuxnva_nic_id --ip-forwarding
onprem_linuxnva_pip_ip=$(az network public-ip show -n $onprem_linuxnva_pip -g $rg --query ipAddress -o tsv) && echo $onprem_linuxnva_pip_ip
onprem_linuxnva_private_ip=$(az network nic show --ids $onprem_linuxnva_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $onprem_linuxnva_private_ip
onprem_linuxnva_default_gw=$(default_gw "$onprem_linuxnva_ip" "255.255.255.0") && echo $onprem_linuxnva_default_gw

# Private IPs of the external interfaces of the VWAN NVA
nva_ip21=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[1].privateIpAddress' -o tsv) && echo $nva_ip21
nva_ip22=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[3].privateIpAddress' -o tsv) && echo $nva_ip22

# Private IPs VWAN NVAs
nva_private_ip1=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[0].privateIpAddress' -o tsv) && echo $nva_private_ip1
nva_private_ip2=$(az network virtual-appliance show -n $nva_name -g $rg --query 'virtualApplianceNics[2].privateIpAddress' -o tsv) && echo $nva_private_ip2

# If running on ER, connectivity to Azure VM and hub NVAs over ER can be verified
if [[ $vpn_endpoint == "private" ]]
then
      az network nic show-effective-route-table --ids $azurevm_nic_id -o table
      az network nic show-effective-route-table --ids $onprem_linuxnva_nic_id -o table
      ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping -c 5 $azurevm_private_ip"
      ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping -c 5 $nva_ip21"
      ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping -c 5 $nva_ip22"
      ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping -c 5 $nva_private_ip1"
      ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping -c 5 $nva_private_ip2"
      ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no -J $onprem_linuxnva_pip_ip $nva_private_ip1
      ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no -J $onprem_linuxnva_pip_ip $nva_private_ip2
fi

# Configure StrongSwan VPN, depending on whether using the public or the private IPs
# See https://blog.sys4.de/routing-based-vpn-with-strongswan-de.html
# See https://wiki.strongswan.org/projects/strongswan/wiki/RouteBasedVPN
# Note: routing breaks when endpoint_public = endpoint_private (the private endpoint is redirected to the VTI, which is still down)
if [[ "$vpn_endpoint" == "public" ]]
then
      endpoint_a_public=$pip1
      endpoint_a_private=$nva_ip21
      endpoint_b_public=$pip2
      endpoint_b_private=$nva_ip22
      endpoint_c_public=$onprem_linuxnva_pip_ip
      endpoint_c_private=$onprem_linuxnva_private_ip
else
      endpoint_a_public=$nva_private_ip1
      endpoint_a_private=$nva_private_ip1
      endpoint_b_public=$nva_private_ip2
      endpoint_b_private=$nva_private_ip2
      endpoint_c_public=$onprem_linuxnva_private_ip
      endpoint_c_private=$onprem_linuxnva_private_ip
fi

echo "Configuring VPN between A:${endpoint_a_public}/${endpoint_a_private} and C:${endpoint_c_public}/${endpoint_c_private}"
echo "Configuring VPN between B:${endpoint_b_public}/${endpoint_b_private} and C:${endpoint_c_public}/${endpoint_c_private}"

# NVA 1
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo ip tunnel add vti0 local $endpoint_a_private remote $endpoint_c_public mode vti key 11"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo sysctl -w net.ipv4.conf.vti0.disable_policy=1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo ip link set up dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo ip route add ${endpoint_c_private}/32 dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo sed -i 's/# install_routes = yes/install_routes = no/' /etc/strongswan.d/charon.conf"
# NVA 2
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo ip tunnel add vti0 local $endpoint_b_private remote $endpoint_c_public mode vti key 11"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo sysctl -w net.ipv4.conf.vti0.disable_policy=1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo ip link set up dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo ip route add ${endpoint_c_private}/32 dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo sed -i 's/# install_routes = yes/install_routes = no/' /etc/strongswan.d/charon.conf"
# Onprem
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip tunnel add vti0 local $endpoint_c_private remote $endpoint_a_public mode vti key 11"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo sysctl -w net.ipv4.conf.vti0.disable_policy=1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip link set up dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip route add ${nva_ip21}/32 dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip tunnel add vti1 local $endpoint_c_private remote $endpoint_b_public mode vti key 12"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo sysctl -w net.ipv4.conf.vti1.disable_policy=1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip link set up dev vti1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip route add ${nva_ip22}/32 dev vti1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo sed -i 's/# install_routes = yes/install_routes = no/' /etc/strongswan.d/charon.conf"

# Troubleshooting: remove static routes
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo ip route delete ${endpoint_c_private}/32 dev vti0"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo ip route delete ${endpoint_c_private}/32 dev vti0"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip route delete ${nva_ip21}/32 dev vti0"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip route delete ${nva_ip22}/32 dev vti1"

# IPsec Config files
vpn_psk=$(openssl rand -base64 64)
vpn_psk=${vpn_psk//$'\n'/}  # Remove line breaks
psk_file_a=/tmp/ipsec.secrets.a
psk_file_b=/tmp/ipsec.secrets.b
psk_file_c=/tmp/ipsec.secrets.c
cat <<EOF > $psk_file_a
$endpoint_a_public $endpoint_c_public : PSK "$vpn_psk"
EOF
cat <<EOF > $psk_file_b
$endpoint_b_public $endpoint_c_public : PSK "$vpn_psk"
EOF
cat <<EOF > $psk_file_c
$endpoint_c_public $endpoint_a_public : PSK "$vpn_psk"
$endpoint_c_public $endpoint_b_public : PSK "$vpn_psk"
EOF
ipsec_file_a=/tmp/ipsec.conf.a
ipsec_file_b=/tmp/ipsec.conf.b
ipsec_file_c=/tmp/ipsec.conf.c
cat <<EOF > $ipsec_file_a
config setup
        charondebug="all"
        uniqueids=yes
        strictcrlpolicy=no
conn to-onprem
  authby=secret
  leftid=$endpoint_a_public
  leftsubnet=0.0.0.0/0
  right=$endpoint_c_public
  rightsubnet=0.0.0.0/0
  ike=aes256-sha2_256-modp1024!
  esp=aes256-sha2_256!
  keyingtries=0
  ikelifetime=1h
  lifetime=8h
  dpddelay=30
  dpdtimeout=120
  dpdaction=restart
  auto=start
  mark=11
EOF
cat <<EOF > $ipsec_file_b
config setup
        charondebug="all"
        uniqueids=yes
        strictcrlpolicy=no
conn to-onprem
  authby=secret
  leftid=$endpoint_b_public
  leftsubnet=0.0.0.0/0
  right=$endpoint_c_public
  rightsubnet=0.0.0.0/0
  ike=aes256-sha2_256-modp1024!
  esp=aes256-sha2_256!
  keyingtries=0
  ikelifetime=1h
  lifetime=8h
  dpddelay=30
  dpdtimeout=120
  dpdaction=restart
  auto=start
  mark=11
EOF
cat <<EOF > $ipsec_file_c
config setup
        charondebug="all"
        uniqueids=yes
        strictcrlpolicy=no
conn to-azure1
  authby=secret
  leftid=$endpoint_c_public
  leftsubnet=0.0.0.0/0
  right=$endpoint_a_public
  rightsubnet=0.0.0.0/0
  ike=aes256-sha2_256-modp1024!
  esp=aes256-sha2_256!
  keyingtries=0
  ikelifetime=1h
  lifetime=8h
  dpddelay=30
  dpdtimeout=120
  dpdaction=restart
  auto=start
  mark=11
conn to-azure2
  authby=secret
  leftid=$endpoint_c_public
  leftsubnet=0.0.0.0/0
  right=$endpoint_b_public
  rightsubnet=0.0.0.0/0
  ike=aes256-sha2_256-modp1024!
  esp=aes256-sha2_256!
  keyingtries=0
  ikelifetime=1h
  lifetime=8h
  dpddelay=30
  dpdtimeout=120
  dpdaction=restart
  auto=start
  mark=12
EOF
username=$(whoami)
# Deploy files to NVA1
scp $psk_file_a $pip1:/home/$username/ipsec.secrets
scp $ipsec_file_a $pip1:/home/$username/ipsec.conf
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo mv ./ipsec.* /etc/"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo systemctl restart ipsec"
# Deploy files to NVA2
scp $psk_file_b $pip2:/home/$username/ipsec.secrets
scp $ipsec_file_b $pip2:/home/$username/ipsec.conf
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo mv ./ipsec.* /etc/"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo systemctl restart ipsec"
# Deploy files to onprem
scp $psk_file_c $onprem_linuxnva_pip_ip:/home/$username/ipsec.secrets
scp $ipsec_file_c $onprem_linuxnva_pip_ip:/home/$username/ipsec.conf
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo mv ./ipsec.* /etc/"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo systemctl restart ipsec"

# IPsec Diagnostics
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "ip a"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "netstat -rnv"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "systemctl status ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo ipsec status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "ping $onprem_linuxnva_private_ip -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "ip a"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "netstat -rnv"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "systemctl status ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo ipsec status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "ping $onprem_linuxnva_private_ip -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ip a"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "netstat -rnv"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "systemctl status ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ipsec status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping $endpoint_a_private -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping $endpoint_b_private -c 5"

# Configure BGP with Bird
bird_config_file_a=/tmp/bird.conf.a  # NVA 1/2
bird_config_file_c=/tmp/bird.conf.c  # onprem
# NVA 1/2
cat <<EOF > $bird_config_file_a
log syslog all;
protocol device {
        scan time 10;
}
protocol direct {
      disabled;
}
protocol kernel {
      preference 254;
      learn;
      merge paths on;
      import filter {
          if net ~ ${endpoint_c_private}/32 then accept;
          else reject;
      };
      export filter {
          if net ~ ${endpoint_c_private}/32 then reject;
          else accept;
      };
}
protocol static {
      import all;
      route $vhub_space via $nva_default_gw;
}
filter DROP_LONG {
      # Drop long prefixes
      if ( net ~ [ 0.0.0.0/0{30,32} ] ) then { reject; }
      else accept;
}
protocol bgp RS1 {
      description "RS1";
      multihop;
      local as $nva_asn;
      neighbor $rs_ip1 as $rs_asn;
          import filter {accept;};
          # export filter {accept;};
          export filter DROP_LONG;
}
protocol bgp RS2 {
      description "RS1";
      multihop;
      local as $nva_asn;
      neighbor $rs_ip2 as $rs_asn;
          import filter {accept;};
          # export filter {accept;};
          export filter DROP_LONG;
}
protocol bgp onprem {
      description "BGP to Onprem";
      multihop;
      local as $nva_asn;
      neighbor $endpoint_c_private as $onprem_linuxnva_asn;
          import filter {accept;};
          # export filter {accept;};
          export filter DROP_LONG;
}
EOF
# Configure BGP with Bird (onprem)
cat <<EOF > $bird_config_file_c
log syslog all;
router id $endpoint_c_private;
protocol device {
        scan time 10;
}
protocol direct {
      disabled;
}
protocol kernel {
      preference 254;
      learn;
      merge paths on;
      import filter {
          if net ~ ${nva_ip21}/32 then accept;
          if net ~ ${nva_ip22}/32 then accept;
          else reject;
      };
      export filter {
          if net ~ ${nva_ip21}/32 then reject;
          if net ~ ${nva_ip22}/32 then reject;
          else accept;
      };
}
protocol static {
      import all;
      route 1.1.1.1/32 via $onprem_linuxnva_default_gw;
      route $onprem_vnet_prefix via $onprem_linuxnva_default_gw;
}
filter DROP_LONG {
      # Drop long prefixes
      if ( net ~ [ 0.0.0.0/0{30,32} ] ) then { reject; }
      else accept;
}
protocol bgp NVA1 {
      description "BGP to NVA1";
      multihop;
      local $endpoint_c_private as $onprem_linuxnva_asn;
      neighbor $endpoint_a_private as $nva_asn;
          import filter {accept;};
          #export filter {accept;};
          export filter DROP_LONG;
}
protocol bgp NVA2 {
      description "BGP to NVA2";
      multihop;
      local $endpoint_c_private as $onprem_linuxnva_asn;
      neighbor $endpoint_b_private as $nva_asn;
          import filter {accept;};
          #export filter {accept;};
          export filter DROP_LONG;
}
EOF

# Deploy BGP config files
username=$(whoami)
# NVA1
scp $bird_config_file_a "${pip1}:/home/${username}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo mv /home/${username}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo systemctl restart bird"
# NVA2
scp $bird_config_file_a "${pip2}:/home/${username}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo mv /home/${username}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo systemctl restart bird"
# Onprem
scp $bird_config_file_c "${onprem_linuxnva_pip_ip}:/home/${username}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo mv /home/${username}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo systemctl restart bird"

# BGP Diagnostics
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show protocols RS1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show protocols RS2"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show route"

ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip1 "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show protocols RS1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show protocol all RS1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $pip2 "sudo birdc show route"

ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show protocols NVA1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show protocol all NVA1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show protocols NVA2"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show protocol all NVA2"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show route"

# Connectivity from spoke VM to onprem
az network nic show-effective-route-table --ids $azurevm_nic_id -o table
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $azurevm_pip_ip "ping -c 5 $onprem_linuxnva_private_ip"

###############
#   Cleanup   #
###############

# Delete RG
# az group delete -n $rg -y --no-wait

###############################
# Some sample cloudinit files #
###############################

# public SSH key
# cat <<EOF > $gnva_cloudinit
# #cloud-config
# users:
#   - default
#   - name: $username
#     sudo: ["ALL=(ALL) NOPASSWD:ALL"]
#     ssh-authorized-keys:
#      - $public_ssh_key
# packages:
#   - jq
# EOF

# New user/password
cat <<EOF > $gnva_cloudinit
#!/bin/bash
apt install -y jq
useradd -m $username
echo "$username:Microsoft123!" | chpasswd $username
usermod -aG sudo $username
host_name=\$(curl -s -H Metadata:true "http://169.254.169.254/metadata/instance?api-version=2020-09-01" | jq -r '.compute.name')
if [[ "\${HOSTNAME: -1}" == "0" ]]; then echo "0123456789" > ~/license.txt; else echo "9876543210" > ~/license.txt; fi
EOF

# How to identify whether a VM is running on a VMSS
vmss_name=$(curl -s -H Metadata:true "http://169.254.169.254/metadata/instance?api-version=2020-09-01" | jq -r '.compute.vmScaleSetName') && echo $vmss_name
if [[ -n "$vmss_name" ]]
then
      echo "It seems I am an instance of a VMSS"
else
      echo "It seems I am a standalone VM"
fi