diff --git a/azure_jumpstart_ag/artifacts/PowerShell/AgConfig-contoso-motors.psd1 b/azure_jumpstart_ag/artifacts/PowerShell/AgConfig-contoso-motors.psd1 index f8b4e2c433..3b42892eda 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/AgConfig-contoso-motors.psd1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/AgConfig-contoso-motors.psd1 @@ -57,17 +57,18 @@ @{name="customlocation"; version="latest"}, @{name="kusto"; version="latest"}, @{name="storage-preview"; version="latest"}, - @{name="azure-iot-ops"; version="0.5.1b1"} + @{name="azure-iot-ops"; version="latest"} ) # PowerShell modules PowerShellModules = @( - @{name='Az.ConnectedKubernetes'; version="0.10.3"}, + @{name='Az.ConnectedKubernetes'; version="latest"}, @{name='Az.KubernetesConfiguration'; version="latest"}, @{name='Az.Kusto'; version="latest"}, @{name='Az.EventGrid'; version="latest"}, @{name='Az.Storage'; version="latest"}, - @{name='Az.EventHub'; version="latest"} + @{name='Az.EventHub'; version="latest"}, + @{name='powershell-yaml'; version="latest"} ) # Winget packages list @@ -92,7 +93,8 @@ 'FireDaemon.OpenSSL', 'thomasnordquist.MQTT-Explorer', 'GitHub.cli', - 'Python.Python.3.12' + 'Python.Python.3.12', + 'Derailed.k9s' ) # Pip packages list @@ -143,51 +145,29 @@ natSubnet = "192.168.46.0/24" # This value is the subnet is the NAT router will use to route to AzSMGMT to access the Internet. It can be any /24 subnet and is only used for routing. natDNS = "%staging-natDNS%" # Do not change - can be configured by passing the optional natDNS parameter to the ARM deployment. - # AKS Edge Essentials variables + # Site Kubernetes cluster configurations SiteConfig = @{ Detroit = @{ - ArcClusterName = "Ag-ArcK8s-Detroit" - NetIPAddress = "172.20.1.2" - DefaultGateway = "172.20.1.1" - PrefixLength = "24" - DNSClientServerAddress = "168.63.129.16" - ServiceIPRangeStart = "172.20.1.31" - ServiceIPRangeSize = "10" - ControlPlaneEndpointIp = "172.20.1.21" - LinuxNodeIp4Address = "172.20.1.11" - Subnet = "172.20.1.0/24" + ArcClusterName = "Ag-K3s-Detroit" FriendlyName = "Detroit" - IsProduction = $true - Type = "AKSEE" + GrafanaDataSource = "detroit" + Type = "k3s" Branch = "main" + HelmValuesFile = "prometheus-additional-scrape-config.yaml" HelmSetValue = "alertmanager.enabled=false,grafana.enabled=false,prometheus.service.type=LoadBalancer" HelmService = "service/prometheus-kube-prometheus-prometheus" - GrafanaDataSource = "detroit" - HelmValuesFile = "prometheus-additional-scrape-config.yaml" - clusterLogSize = "1024" - AKSEEReleaseUseLatest = $true # If set to true, the latest AKSEE release will be used. If set to false, the n-1 version will be used + IsProduction = $true } Monterrey = @{ - ArcClusterName = "Ag-ArcK8s-Monterrey" - NetIPAddress = "172.20.1.3" - DefaultGateway = "172.20.1.1" - PrefixLength = "24" - DNSClientServerAddress = "168.63.129.16" - ServiceIPRangeStart = "172.20.1.71" - ServiceIPRangeSize = "10" - ControlPlaneEndpointIp = "172.20.1.61" - LinuxNodeIp4Address = "172.20.1.51" - Subnet = "172.20.1.0/24" + ArcClusterName = "Ag-K3s-Monterrey" FriendlyName = "Monterrey" - IsProduction = $true - Type = "AKSEE" + GrafanaDataSource = "monterrey" + Type = "k3s" Branch = "main" + HelmValuesFile = "prometheus-additional-scrape-config.yaml" HelmSetValue = "alertmanager.enabled=false,grafana.enabled=false,prometheus.service.type=LoadBalancer" HelmService = "service/prometheus-kube-prometheus-prometheus" - GrafanaDataSource = "monterrey" - HelmValuesFile = "prometheus-additional-scrape-config.yaml" - clusterLogSize = "1024" - AKSEEReleaseUseLatest = $true # If set to true, the latest AKSEE release will be used. If set to false, the n-1 version will be used + IsProduction = $true } } @@ -221,7 +201,7 @@ inferencing_deployment = @{ GitOpsConfigName = "contoso-motors" KustomizationName = "contoso-motors" - KustomizationPath="./contoso_manufacturing/operations" + KustomizationPath="./agora/contoso_motors" Namespace = "contoso-motors" Order = 1 } diff --git a/azure_jumpstart_ag/artifacts/PowerShell/AgLogonScript.ps1 b/azure_jumpstart_ag/artifacts/PowerShell/AgLogonScript.ps1 index c18b6ad9ec..84c3b99a57 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/AgLogonScript.ps1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/AgLogonScript.ps1 @@ -45,13 +45,16 @@ if ($scenario -eq "contoso_supermarket") { $global:workflowStatus = "" $global:appClonedRepo = "https://github.com/$githubUser/jumpstart-apps" }elseif ($scenario -eq "contoso_motors") { - $global:appUpstreamRepo = "https://github.com/microsoft/jumpstart-agora-apps" + $global:appUpstreamRepo = "https://github.com/azure/jumpstart-apps" $global:aioNamespace = "azure-iot-operations" - $global:mqListenerService = "aio-mq-dmqtt-frontend" + $global:mqListenerService = "aio-broker-insecure" $global:mqttExplorerReleasesUrl = $websiteUrls["mqttExplorerReleases"] $global:stagingStorageAccountName = $Env:stagingStorageAccountName $global:aioStorageAccountName = $Env:aioStorageAccountName $global:spnObjectId = $Env:spnObjectId + $global:k3sArcDataClusterName = $Env:k3sArcDataClusterName + $global:k3sArcClusterName = $Env:k3sArcClusterName + $global:tenantId = $Env:tenantId }elseif ($scenario -eq "contoso_hypermarket"){ $global:appUpstreamRepo = "https://github.com/Azure/jumpstart-apps" $global:tenantId = $Env:tenantId @@ -77,7 +80,9 @@ Import-Module "$AgPowerShellDir\contoso_motors.psm1" -Force -DisableNameChecking Import-Module "$AgPowerShellDir\contoso_hypermarket.psm1" -Force -DisableNameChecking Start-Transcript -Path ($AgLogsDir + "\AgLogonScript.log") -Write-Host "Executing Jumpstart Agora automation scripts" +Write-Host "Executing Jumpstart Agora automation scripts..." +Write-Host "Selected global scenario:" $global:scenario +Write-Host "Selected scenario:" $scenario $startTime = Get-Date # Remove registry keys that are used to automatically logon the user (only used for first-time setup) @@ -108,7 +113,7 @@ $global:Credentials = New-Object System.Management.Automation.PSCredential($AgCo ##################################################################### Write-Host "[$(Get-Date -Format t)] INFO: Configuring Azure CLI (Step 1/17)" -ForegroundColor DarkGreen Write-Host "[$(Get-Date -Format t)] INFO: Logging into Az CLI" -ForegroundColor Gray -if($scenario -eq "contoso_hypermarket"){ +if($scenario -eq "contoso_hypermarket" -or $scenario -eq "contoso_motors"){ az login --identity | Out-File -Append -FilePath ($AgConfig.AgDirectories["AgLogsDir"] + "\AzCLI.log") }else{ az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId | Out-File -Append -FilePath ($AgConfig.AgDirectories["AgLogsDir"] + "\AzCLI.log") @@ -149,7 +154,7 @@ if ($scenario -eq "contoso_supermarket") { ##################################################################### # Configure L1 virtualization infrastructure ##################################################################### -if ($scenario -eq "contoso_supermarket" -or $scenario -eq "contoso_motors") { +if ($scenario -eq "contoso_supermarket") { Write-Host "[$(Get-Date -Format t)] INFO: Configuring L1 virtualization infrastructure (Step 6/17)" -ForegroundColor DarkGreen Deploy-VirtualizationInfrastructure } @@ -164,9 +169,9 @@ if ($scenario -eq "contoso_supermarket") { ##################################################################### # Get clusters config files ##################################################################### -if($scenario -eq "contoso_hypermarket"){ - Get-K3sConfigFile - Merge-K3sConfigFiles +if($scenario -eq "contoso_hypermarket" -or $scenario -eq "contoso_motors"){ + Get-K3sConfigFileContosoMotors + Merge-K3sConfigFilesContosoMotors Set-K3sClusters } @@ -194,7 +199,7 @@ if ($scenario -eq "contoso_supermarket") { ##################################################################### # Connect the AKS Edge Essentials clusters and hosts to Azure Arc ##################################################################### -if($scenario -eq "contoso_supermarket" -or $scenario -eq "contoso_motors"){ +if($scenario -eq "contoso_supermarket"){ Write-Host "[$(Get-Date -Format t)] INFO: Connecting AKS Edge clusters to Azure with Azure Arc (Step 10/17)" -ForegroundColor DarkGreen Deploy-AzArcK8sAKSEE } @@ -234,11 +239,10 @@ if ($scenario -eq "contoso_supermarket") { } if ($scenario -eq "contoso_motors") { - Update-AzureIoTOpsExtension - Deploy-AIO - Deploy-MotorsConfigs + Deploy-AIO-M3 $mqttIpArray=Set-MQTTIpAddress Deploy-MQTTExplorer -mqttIpArray $mqttIpArray + Deploy-MotorsConfigs }elseif($scenario -eq "contoso_hypermarket"){ Deploy-AIO-M3 $mqttIpArray=Set-MQTTIpAddress diff --git a/azure_jumpstart_ag/artifacts/PowerShell/Bootstrap.ps1 b/azure_jumpstart_ag/artifacts/PowerShell/Bootstrap.ps1 index 7dfbb814a4..61a3210467 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/Bootstrap.ps1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/Bootstrap.ps1 @@ -34,7 +34,8 @@ param ( [string]$speachToTextEndpoint, [object]$azureOpenAIModel, [string]$openAIDeploymentName, - [string]$acrName + [string]$acrName, + [string]$AZCOPY_AUTO_LOGIN_TYPE ) ############################################################## @@ -78,10 +79,11 @@ param ( [System.Environment]::SetEnvironmentVariable('speachToTextEndpoint', $speachToTextEndpoint, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('azureOpenAIModel', $azureOpenAIModel, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('openAIDeploymentName', $openAIDeploymentName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('AZCOPY_AUTO_LOGIN_TYPE', 'MSI', [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('acrName', $acrName, [System.EnvironmentVariableTarget]::Machine) $ErrorActionPreference = 'Continue' - +Write-Host "Scenario from the configuration is $scenario" ############################################################## # Change RDP Port ############################################################## @@ -314,6 +316,9 @@ if($scenario -eq "contoso_supermarket"){ Invoke-WebRequest ($templateBaseUrl + "artifacts/monitoring/grafana-freezer-monitoring.json") -OutFile "$AgMonitoringDir\grafana-freezer-monitoring.json" } elseif ($scenario -eq "contoso_motors") { + Invoke-WebRequest ($templateBaseUrl + "artifacts/kubernetes/K3s/longhorn.yaml") -OutFile "$AgToolsDir\longhorn.yaml" + Invoke-WebRequest ($templateBaseUrl + "artifacts/kubernetes/K3s/kubeVipRbac.yml") -OutFile "$AgToolsDir\kubeVipRbac.yml" + Invoke-WebRequest ($templateBaseUrl + "artifacts/kubernetes/K3s/kubeVipDaemon.yml") -OutFile "$AgToolsDir\kubeVipDaemon.yml" Invoke-WebRequest ($templateBaseUrl + "artifacts/settings/Bookmarks-contoso-motors") -OutFile "$AgToolsDir\Bookmarks" Invoke-WebRequest ($templateBaseUrl + "artifacts/settings/mq_cloudConnector.yml") -OutFile "$AgToolsDir\mq_cloudConnector.yml" Invoke-WebRequest ($templateBaseUrl + "artifacts/settings/mqtt_explorer_settings_motors.json") -OutFile "$AgToolsDir\mqtt_explorer_settings.json" @@ -412,7 +417,7 @@ Get-ScheduledTask -TaskName ServerManager | Disable-ScheduledTask ############################################################## # Install Hyper-V, WSL and reboot ############################################################## -if($scenario -eq "contoso_supermarket" -or $scenario -eq "contoso_motors"){ +if($scenario -eq "contoso_supermarket"){ Write-Header "Installing Hyper-V" Enable-WindowsOptionalFeature -Online -FeatureName Containers -All -NoRestart Enable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform -NoRestart diff --git a/azure_jumpstart_ag/artifacts/PowerShell/Modules/common.psm1 b/azure_jumpstart_ag/artifacts/PowerShell/Modules/common.psm1 index 57d583762d..23d4018390 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/Modules/common.psm1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/Modules/common.psm1 @@ -983,6 +983,9 @@ function Deploy-Prometheus { $apiServerPort = ($apiServerAddress -split ":")[1] do { + Write-Host "apiServerFqdn=$apiServerFqdn" + Write-Host "apiServerPort=$apiServerPort" + Write-Host "apiServer=$apiServer" $result = Test-NetConnection -ComputerName $apiServerFqdn -Port $apiServerPort -WarningAction SilentlyContinue if ($result.TcpTestSucceeded) { Write-Host "[$(Get-Date -Format t)] INFO: Kubernetes API server $apiServer is available" -ForegroundColor Gray @@ -1142,20 +1145,6 @@ function Deploy-Prometheus { Write-Host } -function Update-AzureIoTOpsExtension { - try { - Write-Host "Starting patching of azure-iot-ops extension..." -ForegroundColor Green - & "C:\Program Files\Microsoft SDKs\Azure\CLI2\python.exe" -m pip install -U --target "C:\Program Files\Microsoft SDKs\Azure\CLI2\Lib\site-packages\azure-cli-extensions\azure-iot-ops" azure-identity==1.17.1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Installation of azure-iot-ops extension completed successfully." -ForegroundColor Green - } else { - Write-Host "Installation of azure-iot-ops extension failed with exit code $LASTEXITCODE." -ForegroundColor Red - } - } catch { - Write-Host "An error occurred during the patching of the azure-iot-ops extension." -ForegroundColor Red - Write-Host $_.Exception.Message -ForegroundColor Red - } -} # Deploys Azure IoT Operations on all k8s clusters in the config file function Deploy-AIO { @@ -1270,7 +1259,6 @@ function Deploy-AIO { --features cluster-connect custom-locations ` --custom-locations-oid $customLocationRPOID ` --only-show-errors - Start-Sleep -Seconds 10 do { @@ -1285,7 +1273,7 @@ function Deploy-AIO { } else { $aioStatus = "deployed" - } + } } until ($aioStatus -eq "deployed" -or $retryCount -eq $maxRetries) $kvIndex++ } diff --git a/azure_jumpstart_ag/artifacts/PowerShell/Modules/contoso_hypermarket.psm1 b/azure_jumpstart_ag/artifacts/PowerShell/Modules/contoso_hypermarket.psm1 index 316bb7a478..9fbb36fac3 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/Modules/contoso_hypermarket.psm1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/Modules/contoso_hypermarket.psm1 @@ -199,11 +199,14 @@ function Deploy-AIO-M3 { Write-Host "`n" $schemaName = "${clusterName}-$($Env:namingGuid)-schema" $schemaId = $(az iot ops schema registry create --name $schemaName ` - --resource-group $resourceGroup ` + --resource-group $Env:resourceGroup ` --registry-namespace "$clusterName-$($Env:namingGuid)-namespace" ` - --sa-resource-id $(az storage account show --name $aioStorageAccountName --resource-group $resourceGroup -o tsv --query id) ` + --sa-resource-id $(az storage account show --name $Env:aioStorageAccountName --resource-group $Env:resourceGroup -o tsv --query id) ` --query id -o tsv) + Write-Host "[$(Get-Date -Format t)] INFO: The aio storage account name is: $aioStorageAccountName" -ForegroundColor DarkGray + Write-Host "[$(Get-Date -Format t)] INFO: the schemaId is '$schemaId' - verify this" -ForegroundColor DarkGray + $customLocationName = $arcClusterName.toLower() + "-cl" # Initialize the Azure IoT Operations instance on the Arc-enabled cluster @@ -220,8 +223,8 @@ function Deploy-AIO-M3 { Write-Host "[$(Get-Date -Format t)] Error: An error occured while deploying AIO on the cluster...Retrying" -ForegroundColor DarkRed Write-Host "`n" az iot ops init --cluster $arcClusterName.toLower() ` - --resource-group $resourceGroup ` - --subscription $subscriptionId ` + --resource-group $Env:resourceGroup ` + --subscription $Env:subscriptionId ` --only-show-errors $retryCount++ } @@ -238,8 +241,8 @@ function Deploy-AIO-M3 { do { az iot ops create --name $arcClusterName.toLower() ` --cluster $arcClusterName.toLower() ` - --resource-group $resourceGroup ` - --subscription $subscriptionId ` + --resource-group $Env:resourceGroup ` + --subscription $Env:subscriptionId ` --custom-location $customLocationName ` --sr-resource-id $schemaId ` --enable-rsync true ` diff --git a/azure_jumpstart_ag/artifacts/PowerShell/Modules/contoso_motors.psm1 b/azure_jumpstart_ag/artifacts/PowerShell/Modules/contoso_motors.psm1 index ed4fee4ed1..ad16c92297 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/Modules/contoso_motors.psm1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/Modules/contoso_motors.psm1 @@ -1,14 +1,137 @@ -function Deploy-MotorsConfigs { - Write-Host "[$(Get-Date -Format t)] INFO: Configuring OVMS prerequisites on Kubernetes nodes." -ForegroundColor Gray - $VMs = (Get-VM).Name - foreach ($VM in $VMs) { - Invoke-Command -VMName $VM -Credential $Credentials -ScriptBlock { - Invoke-AksEdgeNodeCommand -NodeType Linux -command "curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/v0.27.0/install.sh | bash -s v0.27.0" +function Get-K3sConfigFileContosoMotors { + # Downloading k3s Kubernetes cluster kubeconfig file + Write-Host "Downloading k3s Kubeconfigs" + $Env:AZCOPY_AUTO_LOGIN_TYPE = "MSI" + foreach ($cluster in $AgConfig.SiteConfig.GetEnumerator()) { + $clusterName = $cluster.Name.ToLower() + $arcClusterName = $AgConfig.SiteConfig[$clusterName].ArcClusterName + "-$namingGuid" + $containerName = $arcClusterName.toLower() + $sourceFile = "https://$stagingStorageAccountName.blob.core.windows.net/$containerName/config" + azcopy cp $sourceFile "C:\Users\$adminUsername\.kube\ag-k3s-$clusterName" --check-length=false + $sourceFile = "https://$stagingStorageAccountName.blob.core.windows.net/$containerName/*" + azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$AgLogsDir\" --include-pattern "*.log" + } +} + +function Merge-K3sConfigFilesContosoMotors{ + + $mergedKubeconfigPath = "C:\Users\$adminUsername\.kube\config" + + $kubeconfig1Path = "C:\Users\$adminUsername\.kube\ag-k3s-detroit" + $kubeconfig2Path = "C:\Users\$adminUsername\.kube\ag-k3s-monterrey" + + # Extract base file names (without extensions) to use as new names + $suffix1 = [System.IO.Path]::GetFileNameWithoutExtension($kubeconfig1Path) + $suffix2 = [System.IO.Path]::GetFileNameWithoutExtension($kubeconfig2Path) + + # Load the kubeconfig files, ensuring no empty lines or structures + $kubeconfig1 = get-content $kubeconfig1Path | ConvertFrom-Yaml + $kubeconfig2 = get-content $kubeconfig2Path | ConvertFrom-Yaml + + # Function to replace cluster, user, and context names with the file name, while keeping original server addresses + function Set-NamesWithFileName { + param ( + [hashtable]$kubeconfigData, + [string]$newName + ) + + # Replace cluster names but keep the server addresses + foreach ($cluster in $kubeconfigData.clusters) { + if ($cluster.name -and $cluster.cluster.server) { + $cluster.name = "$newName" + } + } + + # Replace user names + foreach ($user in $kubeconfigData.users) { + if ($user.name) { + $user.name = "$newName" + } + } + + # Replace context names, but retain the correct mapping to cluster and user + foreach ($context in $kubeconfigData.contexts) { + if ($context.name -and $context.context.cluster -and $context.context.user) { + $context.name = "$newName" + $context.context.cluster = "$newName" + $context.context.user = "$newName" + } } - kubectx $VM.ToLower() - kubectl create -f https://operatorhub.io/install/ovms-operator.yaml + + return $kubeconfigData + } + + # Apply renaming using file names + $kubeconfig1 = Set-NamesWithFileName -kubeconfigData $kubeconfig1 -newName $suffix1 + $kubeconfig2 = Set-NamesWithFileName -kubeconfigData $kubeconfig2 -newName $suffix2 + + # Merge the clusters, users, and contexts from both kubeconfigs + $mergedClusters = $kubeconfig1.clusters + $kubeconfig2.clusters + $mergedUsers = $kubeconfig1.users + $kubeconfig2.users + $mergedContexts = $kubeconfig1.contexts + $kubeconfig2.contexts + + # Prepare the merged kubeconfig ensuring no empty or null fields + $mergedKubeconfig = @{ + apiVersion = $kubeconfig1.apiVersion + kind = $kubeconfig1.kind + clusters = $mergedClusters | Where-Object { $_.name -and $_.cluster.server } + users = $mergedUsers | Where-Object { $_.name } + contexts = $mergedContexts | Where-Object { $_.name -and $_.context.cluster -and $_.context.user } + "current-context" = $kubeconfig1."current-context" # Retain the current context of the first file } + # Convert the merged data back to YAML and save to a new file + $mergedKubeconfig | ConvertTo-Yaml | Set-Content -Path $mergedKubeconfigPath + + Write-Host "Kubeconfig files successfully merged into $mergedKubeconfigPath" + kubectx detroit="ag-k3s-detroit" + kubectx monterrey="ag-k3s-monterrey" + +} + +function Set-K3sClusters { + Write-Host "Configuring kube-vip on K3s clusters" + #az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId + az login --identity + az account set -s $subscriptionId + foreach ($cluster in $AgConfig.SiteConfig.GetEnumerator()) { + if ($cluster.Value.Type -eq "k3s") { + $clusterName = $cluster.Value.FriendlyName.ToLower() + $vmName = $cluster.Value.ArcClusterName + "-$namingGuid" + kubectx $clusterName + $k3sVIP = $(az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $vmName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv) + Write-Host "Assigning kube-vip-role on k3s cluster" + $kubeVipRbac = "$($Agconfig.AgDirectories.AgToolsDir)\kubeVipRbac.yml" + kubectl apply -f $kubeVipRbac + + $kubeVipDaemonset = "$($Agconfig.AgDirectories.AgToolsDir)\kubeVipDaemon.yml" + (Get-Content -Path $kubeVipDaemonset) -replace 'k3sVIPPlaceholder', "$k3sVIP" | Set-Content -Path $kubeVipDaemonset + kubectl apply -f $kubeVipDaemonset + + Write-Host "Deploying Kube vip cloud controller on k3s cluster" + kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + + $serviceIpRange = $(az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $vmName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv) + $sortedIps = $serviceIpRange | Sort-Object { [System.Version]$_ } + $lowestServiceIp = $sortedIps[0] + $highestServiceIp = $sortedIps[-1] + + kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp + Start-Sleep -Seconds 30 + + # Write-Host "Creating longhorn storage on K3scluster" + # kubectl apply -f "$($Agconfig.AgDirectories.AgToolsDir)\longhorn.yaml" + # Start-Sleep -Seconds 30 + # Write-Host "`n" + } + } +} + +function Deploy-MotorsConfigs { + Write-Host "[$(Get-Date -Format t)] INFO: Beginning Contoso Motors GitOps Deployment" -ForegroundColor Gray + + + # Loop through the clusters and deploy the configs in AppConfig hashtable in AgConfig-contoso-motors.psd foreach ($cluster in $AgConfig.SiteConfig.GetEnumerator()) { Start-Job -Name gitops -ScriptBlock { @@ -102,7 +225,7 @@ function Deploy-MotorsConfigs { --scope cluster ` --url $appClonedRepo ` --branch $branch ` - --sync-interval 3s ` + --sync-interval 5m ` --kustomization name=$appName path=$appPath prune=true ` --timeout 30m ` --namespace $namespace ` diff --git a/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 b/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 index b216627956..2beb0d38f9 100644 --- a/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 +++ b/azure_jumpstart_ag/artifacts/PowerShell/Modules/manufacturing.psm1 @@ -58,7 +58,7 @@ function Deploy-ManufacturingConfigs { --scope cluster ` --url $appClonedRepo ` --branch $branch ` - --sync-interval 3s ` + --sync-interval 5m ` --kustomization name=$appName path=$appPath prune=true retry_interval=1m ` --timeout 10m ` --namespace $namespace ` @@ -578,17 +578,3 @@ function Deploy-ManufacturingBookmarks { $quickAccess.Namespace($AgConfig.AgDirectories.AgLogsDir).Self.InvokeVerb("pintohome") } -function Update-AzureIoTOpsExtension { - try { - Write-Host "Starting patching of azure-iot-ops extension..." -ForegroundColor Green - & "C:\Program Files\Microsoft SDKs\Azure\CLI2\python.exe" -m pip install -U --target "C:\Program Files\Microsoft SDKs\Azure\CLI2\Lib\site-packages\azure-cli-extensions\azure-iot-ops" azure-identity==1.17.1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Installation of azure-iot-ops extension completed successfully." -ForegroundColor Green - } else { - Write-Host "Installation of azure-iot-ops extension failed with exit code $LASTEXITCODE." -ForegroundColor Red - } - } catch { - Write-Host "An error occurred during the patching of the azure-iot-ops extension." -ForegroundColor Red - Write-Host $_.Exception.Message -ForegroundColor Red - } -} \ No newline at end of file diff --git a/azure_jumpstart_ag/artifacts/kubernetes/K3s/installK3s.sh b/azure_jumpstart_ag/artifacts/kubernetes/K3s/installK3s.sh index dc68270bd4..3f38f4b1a8 100644 --- a/azure_jumpstart_ag/artifacts/kubernetes/K3s/installK3s.sh +++ b/azure_jumpstart_ag/artifacts/kubernetes/K3s/installK3s.sh @@ -6,30 +6,33 @@ sudo adduser staginguser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --d sudo echo "staginguser:ArcPassw0rd" | sudo chpasswd # Injecting environment variables -echo '#!/bin/bash' >> vars.sh -echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh -echo $subscriptionId:$2 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$3 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $templateBaseUrl:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $storageContainerName:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $k3sControlPlane:$9 | awk '{print substr($1,2); }' >> vars.sh -echo $resourceGroup:$10| awk '{print substr($1,2); }' >> vars.sh -echo $deployGPUNodes:$11| awk '{print substr($1,2); }' >> vars.sh - -sed -i '2s/^/export adminUsername=/' vars.sh -sed -i '3s/^/export subscriptionId=/' vars.sh -sed -i '4s/^/export vmName=/' vars.sh -sed -i '5s/^/export location=/' vars.sh -sed -i '6s/^/export stagingStorageAccountName=/' vars.sh -sed -i '7s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '8s/^/export templateBaseUrl=/' vars.sh -sed -i '9s/^/export storageContainerName=/' vars.sh -sed -i '10s/^/export k3sControlPlane=/' vars.sh -sed -i '11s/^/export resourceGroup=/' vars.sh -sed -i '12s/^/export deployGPUNodes=/' vars.sh + +echo '#!/bin/bash' > vars.sh + + +adminUsername=$1 +subscriptionId=$2 +vmName=$3 +location=$4 +stagingStorageAccountName=$5 +logAnalyticsWorkspace=$6 +templateBaseUrl=$7 +storageContainerName=$8 +k3sControlPlane=$9 +resourceGroup=${10} +scenario=${11} + +echo "export adminUsername=$adminUsername" >> vars.sh +echo "export subscriptionId=$subscriptionId" >> vars.sh +echo "export vmName=$vmName" >> vars.sh +echo "export location=$location" >> vars.sh +echo "export stagingStorageAccountName=$stagingStorageAccountName" >> vars.sh +echo "export logAnalyticsWorkspace=$logAnalyticsWorkspace" >> vars.sh +echo "export templateBaseUrl=$templateBaseUrl" >> vars.sh +echo "export storageContainerName=$storageContainerName" >> vars.sh +echo "export k3sControlPlane=$k3sControlPlane" >> vars.sh +echo "export resourceGroup=$resourceGroup" >> vars.sh +echo "export scenario=$scenario" >> vars.sh export vmName=$3 @@ -78,8 +81,7 @@ sudo chmod +x /usr/local/bin/azcopy # Authorize azcopy by using a system-wide managed identity export AZCOPY_AUTO_LOGIN_TYPE=MSI - -# Run the lock check before attempting the installation +d devthe lock check before attempting the installation check_dpkg_lock # Installing Azure CLI & Azure Arc extensions @@ -309,6 +311,23 @@ fi # Installing NVIDIA container toolkit + + +# check the scenario and run the curl command if scenario is contoso-motors +if [ "$scenario" == "contoso_motors" ]; then + echo "Running curl command to install OpenVINO Toolkit Operator" + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/v0.31.0/install.sh | bash -s v0.31.0 + sleep 10 + echo "Installing operator via kubectl" + kubectl create -f https://operatorhub.io/install/ovms-operator.yaml + + echo "Installing OVMS and InfluxDB Helm charts" + helm install ovms --create-namespace -n contoso-motors oci://mcr.microsoft.com/jumpstart/agora/helm/ovms --version 0.1.0 + sleep 10 + helm install influxdb -n contoso-motors oci://mcr.microsoft.com/jumpstart/agora/helm/influxdb --version 0.1.1 +fi + # Uploading this script log to staging storage for ease of troubleshooting echo "" echo "Uploading the script logs to staging storage" @@ -318,4 +337,5 @@ log="/home/$adminUsername/jumpstart_logs/installK3s-$vmName.log" storageContainerNameLower=$(echo $storageContainerName | tr '[:upper:]' '[:lower:]') azcopy cp $log "https://$stagingStorageAccountName.blob.core.windows.net/$storageContainerNameLower/installK3s-$vmName.log" --check-length=false >/dev/null 2>&1 + exit 0 \ No newline at end of file diff --git a/azure_jumpstart_ag/contoso_motors/bicep/clientVm/clientVm.bicep b/azure_jumpstart_ag/contoso_motors/bicep/clientVm/clientVm.bicep index 4ab4bf32f1..08727ff682 100644 --- a/azure_jumpstart_ag/contoso_motors/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_ag/contoso_motors/bicep/clientVm/clientVm.bicep @@ -56,9 +56,6 @@ param deployBastion bool = false @description('Storage account used for staging file artifacts') param storageAccountName string -@description('The login server name of the Azure Container Registry') -param acrName string - @description('The name of the Azure Data Explorer cluster') param adxClusterName string @@ -130,10 +127,13 @@ resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2023-02-01' = if ( resource vm 'Microsoft.Compute/virtualMachines@2022-11-01' = { name: vmName location: location + identity: { + type: 'SystemAssigned' + } tags: resourceTags properties: { hardwareProfile: { - vmSize: 'Standard_D32s_v5' + vmSize: 'Standard_D8s_v5' } storageProfile: { osDisk: { @@ -206,10 +206,45 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-11-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/PowerShell/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${encodedPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnObjectId ${spnObjectId} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azureLocation ${location} -stagingStorageAccountName ${storageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -acrName ${acrName} -rdpPort ${rdpPort} -githubAccount ${githubAccount} -githubBranch ${githubBranch} -namingGuid ${namingGuid} -adxClusterName ${adxClusterName} -customLocationRPOID ${customLocationRPOID} -scenario ${scenario} -aioStorageAccountName ${aioStorageAccountName} -vmAutologon ${vmAutologon}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${encodedPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnObjectId ${spnObjectId} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azureLocation ${location} -stagingStorageAccountName ${storageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -rdpPort ${rdpPort} -githubAccount ${githubAccount} -githubBranch ${githubBranch} -namingGuid ${namingGuid} -adxClusterName ${adxClusterName} -customLocationRPOID ${customLocationRPOID} -scenario ${scenario} -aioStorageAccountName ${aioStorageAccountName} -vmAutologon ${vmAutologon}' } } } +// Add role assignment for the VM: Azure Key Vault Secret Officer role +resource vmRoleAssignment_KeyVaultAdministrator 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Administrator') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '00482a5a-887f-4fb3-b363-3b7fe8e74483') + principalType: 'ServicePrincipal' + + } +} + +// Add role assignment for the VM: Storage Blob Data Contributor +resource vmRoleAssignment_Storage 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') + principalType: 'ServicePrincipal' + } +} + +// Add role assignment for the VM: Owner +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + principalType: 'ServicePrincipal' + } +} + + output adminUsername string = windowsAdminUsername output publicIP string = deployBastion == false ? concat(publicIpAddress.properties.ipAddress) : '' diff --git a/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/acr.bicep b/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/acr.bicep deleted file mode 100644 index c864f1db59..0000000000 --- a/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/acr.bicep +++ /dev/null @@ -1,25 +0,0 @@ -@description('The location of the Managed Cluster resource') -param location string = resourceGroup().location - -@description('Resource tag for Jumpstart Agora') -param resourceTags object = { - Project: 'Jumpstart_Agora' -} - -@description('Name of the Azure Container Registry') -param acrName string - -@description('Provide a tier of your Azure Container Registry.') -param acrSku string = 'Basic' - -resource acr 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' ={ - name: acrName - location: location - tags: resourceTags - sku: { - name: acrSku - } - properties: { - adminUserEnabled: true - } -} diff --git a/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/ubuntuRancher.bicep new file mode 100644 index 0000000000..f4a10f1f0f --- /dev/null +++ b/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/ubuntuRancher.bicep @@ -0,0 +1,182 @@ +@description('The name of you Virtual Machine') +param vmName string = 'Ag-K3s-${namingGuid}' + +@description('Username for the Virtual Machine') +param adminUsername string = 'agora' + +@description('RSA public key used for securing SSH access to ArcBox resources. This parameter is only needed when deploying the DataOps or DevOps flavors.') +@secure() +param sshRSAPublicKey string = '' + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version') +@allowed([ + '22_04-lts-gen2' +]) +param ubuntuOSVersion string = '22_04-lts-gen2' + +@description('Location for all resources.') +param azureLocation string = resourceGroup().location + +@description('The size of the VM') +param vmSize string = 'Standard_B4ms' + +@description('Resource Id of the subnet in the virtual network') +param subnetId string + +@description('Name for the staging storage account using to hold kubeconfig. This value is passed into the template as an output from mgmtStagingStorage.json') +param stagingStorageAccountName string + +@description('Name of the Log Analytics workspace used with cluster extensions') +param logAnalyticsWorkspace string + +@description('The base URL used for accessing artifacts and automation artifacts') +param templateBaseUrl string + +@description('Storage account container name for artifacts') +param storageContainerName string + +@description('The scenario to deploy') +param scenario string + +@maxLength(5) +@description('Random GUID') +param namingGuid string + +var publicIpAddressName = '${vmName}-PIP' +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +var k3sControlPlane = 'true' // deploy single-node k3s control plane +var diskSize = 512 +var numberOfIPAddresses = 15 // The number of IP addresses to create + +// Create multiple public IP addresses +resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { + name: '${publicIpAddressName}${i}' + location: azureLocation + properties: { + publicIPAllocationMethod: 'Static' + publicIPAddressVersion: 'IPv4' + idleTimeoutInMinutes: 4 + } + sku: { + name: 'Basic' + } +}] + +// Create multiple NIC IP configurations and assign the public IP to the IP configuration +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [for i in range(1, numberOfIPAddresses): { + name: 'ipconfig${i}' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + publicIPAddress: { + id: publicIpAddresses[i-1].id + } + primary: i == 1 ? true : false + } + }] + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { + name: vmName + location: azureLocation + identity: { + type: 'SystemAssigned' + } + properties: { + hardwareProfile: { + vmSize: vmSize + } + storageProfile: { + osDisk: { + name: '${vmName}-OSDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + diskSizeGB: diskSize + } + imageReference: { + publisher: 'canonical' + offer: '0001-com-ubuntu-server-jammy' + sku: ubuntuOSVersion + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: networkInterface.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + linuxConfiguration: { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: sshRSAPublicKey + } + ] + } + } + } + } +} + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + principalType: 'ServicePrincipal' + } +} + +// Add role assignment for the VM: Storage Blob Data Contributor +resource vmRoleAssignment_Storage 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') + principalType: 'ServicePrincipal' + } +} + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3s' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane} ${resourceGroup().name} ${scenario}' + fileUris: [ + '${templateBaseUrl}artifacts/kubernetes/K3s/installK3s.sh' + ] + } + } + dependsOn: [ + vmRoleAssignment_Owner + vmRoleAssignment_Storage + ] +} diff --git a/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/ubuntuRancherNodes.bicep new file mode 100644 index 0000000000..043a42ded3 --- /dev/null +++ b/azure_jumpstart_ag/contoso_motors/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -0,0 +1,182 @@ + +@description('The name of you Virtual Machine') +param vmName string = 'Ag-K3s-Node-${namingGuid}' + +@description('Username for the Virtual Machine') +param adminUsername string = 'agora' + +@description('RSA public key used for securing SSH access to ArcBox resources. This parameter is only needed when deploying the DataOps or DevOps flavors.') +@secure() +param sshRSAPublicKey string = '' + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version') +@allowed([ + '22_04-lts-gen2' +]) +param ubuntuOSVersion string = '22_04-lts-gen2' + +@description('Location for all resources.') +param azureLocation string = resourceGroup().location + +@description('Resource Id of the subnet in the virtual network') +param subnetId string + +@description('Name for the staging storage account using to hold kubeconfig. This value is passed into the template as an output from mgmtStagingStorage.json') +param stagingStorageAccountName string + +@description('Name of the Log Analytics workspace used with cluster extensions') +param logAnalyticsWorkspace string + +@description('The base URL used for accessing artifacts and automation artifacts') +param templateBaseUrl string + +@description('Storage account container name for artifacts') +param storageContainerName string + +@maxLength(5) +@description('Random GUID') +param namingGuid string + +@description('Option to deploy GPU-enabled nodes for the K3s Worker nodes.') +param deployGPUNodes bool = false + +@description('The sku name of the K3s cluster worker nodes.') +@allowed([ + 'Standard_D8s_v5' + 'Standard_NV6ads_A10_v5' + 'Standard_NV4as_v4' +]) +param k8sWorkerNodesSku string = deployGPUNodes ? 'Standard_NV4as_v4' : 'Standard_D8s_v5' + +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +var diskSize = 512 + +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [ + { + name: 'ipconfig1' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + } + } + ] + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { + name: vmName + location: azureLocation + identity: { + type: 'SystemAssigned' + } + properties: { + hardwareProfile: { + vmSize: k8sWorkerNodesSku + } + storageProfile: { + osDisk: { + name: '${vmName}-OSDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + diskSizeGB: diskSize + } + imageReference: { + publisher: 'canonical' + offer: '0001-com-ubuntu-server-jammy' + sku: ubuntuOSVersion + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: networkInterface.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + linuxConfiguration: { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: sshRSAPublicKey + } + ] + } + } + } + } +} + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + principalType: 'ServicePrincipal' + } +} + +// Add role assignment for the VM: Storage Blob Data Contributor +resource vmRoleAssignment_Storage 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Storage Blob Data Contributor') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') + principalType: 'ServicePrincipal' + } +} + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3s' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${deployGPUNodes}' + fileUris: [ + '${templateBaseUrl}artifacts/kubernetes/K3s/installK3s.sh' + ] + } + } + dependsOn: [ + vmRoleAssignment_Owner + vmRoleAssignment_Storage + gpuInstallationScript + ] +} + +resource gpuInstallationScript 'Microsoft.Compute/virtualMachines/extensions@2015-06-15' =if(deployGPUNodes) { + parent: vm + name: 'gpuInstallationScript' + location: azureLocation + properties: { + publisher: 'Microsoft.HpcCompute' + type: 'NvidiaGpuDriverLinux' + typeHandlerVersion: '1.6' + autoUpgradeMinorVersion: true + settings: {} + } +} diff --git a/azure_jumpstart_ag/contoso_motors/bicep/main.azd.bicep b/azure_jumpstart_ag/contoso_motors/bicep/main.azd.bicep index e4234c81d5..d896b7f61c 100644 --- a/azure_jumpstart_ag/contoso_motors/bicep/main.azd.bicep +++ b/azure_jumpstart_ag/contoso_motors/bicep/main.azd.bicep @@ -43,10 +43,10 @@ param windowsAdminPassword string param logAnalyticsWorkspaceName string = 'Ag-Workspace-${namingGuid}' @description('Target GitHub account') -param githubAccount string = 'microsoft' +param githubAccount string = 'nabeelmsft' @description('Target GitHub branch') -param githubBranch string = 'main' +param githubBranch string = 'contosomotorswork' @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false diff --git a/azure_jumpstart_ag/contoso_motors/bicep/main.bicep b/azure_jumpstart_ag/contoso_motors/bicep/main.bicep index 8392341da1..f60509753e 100644 --- a/azure_jumpstart_ag/contoso_motors/bicep/main.bicep +++ b/azure_jumpstart_ag/contoso_motors/bicep/main.bicep @@ -27,6 +27,9 @@ param windowsAdminUsername string @secure() param windowsAdminPassword string +@description('Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example \'ssh-rsa AAAAB...snip...UcyupgH azureuser@linuxvm\'') +param sshRSAPublicKey string + @description('Name for your log analytics workspace') param logAnalyticsWorkspaceName string = 'Ag-Workspace-${namingGuid}' @@ -43,10 +46,10 @@ param deployBastion bool = false param virtualNetworkNameCloud string = 'Ag-Vnet-Prod' @description('Name of the Staging AKS subnet in the cloud virtual network') -param subnetNameCloudAksStaging string = 'Ag-Subnet-Staging' +param subnetNameCloudK3s string = 'Ag-Subnet-K3s' @description('Name of the inner-loop AKS subnet in the cloud virtual network') -param subnetNameCloudAksInnerLoop string = 'Ag-Subnet-InnerLoop' +param subnetNameCloud string = 'Ag-Subnet-Cloud' @description('Name of the storage queue') param storageQueueName string = 'aioqueue' @@ -78,10 +81,11 @@ param adxClusterName string = 'agadx${namingGuid}' @description('The custom location RPO ID') param customLocationRPOID string -@minLength(5) -@maxLength(50) -@description('Name of the Azure Container Registry') -param acrName string = 'agacr${namingGuid}' +@description('The name of the Azure Arc K3s cluster') +param k3sArcDataClusterName string = 'Ag-K3s-Detroit-${namingGuid}' + +@description('The name of the Azure Arc K3s data cluster') +param k3sArcClusterName string = 'Ag-K3s-Monterrey-${namingGuid}' @description('Override default RDP port using this parameter. Default is 3389. No changes will be made to the client VM.') param rdpPort string = '3389' @@ -92,7 +96,17 @@ param vmAutologon bool = true @description('The agora scenario to be deployed') param scenario string = 'contoso_motors' +@description('The sku name of the K3s cluster worker nodes.') +@allowed([ + 'Standard_D8s_v5' + 'Standard_NV6ads_A10_v5' + 'Standard_NV4as_v4' +]) +param k8sWorkerNodesSku string = 'Standard_D8s_v5' +//param k8sWorkerNodesSku string = deployGPUNodes ? 'Standard_NV4as_v4' : 'Standard_D8s_v5' + var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_ag/' +var k3sClusterNodesCount = 2 // Number of nodes to deploy in the K3s cluster module mgmtArtifactsAndPolicyDeployment 'mgmt/mgmtArtifacts.bicep' = { name: 'mgmtArtifactsAndPolicyDeployment' @@ -106,8 +120,8 @@ module networkDeployment 'mgmt/network.bicep' = { name: 'networkDeployment' params: { virtualNetworkNameCloud: virtualNetworkNameCloud - subnetNameCloudAksStaging: subnetNameCloudAksStaging - subnetNameCloudAksInnerLoop: subnetNameCloudAksInnerLoop + subnetNameCloudK3s: subnetNameCloudK3s + subnetNameCloud: subnetNameCloud deployBastion: deployBastion location: location } @@ -120,8 +134,82 @@ module storageAccountDeployment 'mgmt/storageAccount.bicep' = { } } +module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = { + name: 'ubuntuRancherK3s2Deployment' + params: { + sshRSAPublicKey: sshRSAPublicKey + stagingStorageAccountName: toLower(storageAccountDeployment.outputs.storageAccountName) + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: networkDeployment.outputs.k3sSubnetId + azureLocation: location + vmName : k3sArcDataClusterName + storageContainerName: toLower(k3sArcDataClusterName) + namingGuid: namingGuid + } +} + +module ubuntuRancherK3sDeployment 'kubernetes/ubuntuRancher.bicep' = { + name: 'ubuntuRancherK3sDeployment' + params: { + sshRSAPublicKey: sshRSAPublicKey + stagingStorageAccountName: toLower(storageAccountDeployment.outputs.storageAccountName) + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: networkDeployment.outputs.k3sSubnetId + azureLocation: location + vmName : k3sArcClusterName + storageContainerName: toLower(k3sArcClusterName) + namingGuid: namingGuid + } +} + +module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): { + name: 'ubuntuRancherK3sNodesDeployment-${i}' + params: { + sshRSAPublicKey: sshRSAPublicKey + stagingStorageAccountName: toLower(storageAccountDeployment.outputs.storageAccountName) + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: networkDeployment.outputs.k3sSubnetId + azureLocation: location + vmName : '${k3sArcDataClusterName}-Node-0${i}' + storageContainerName: toLower(k3sArcDataClusterName) + namingGuid: namingGuid + //deployGPUNodes: deployGPUNodes + k8sWorkerNodesSku: k8sWorkerNodesSku + } + dependsOn: [ + ubuntuRancherK3sDataSvcDeployment + ] +}] + +module ubuntuRancherK3sNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): { + name: 'ubuntuRancherK3sNodes2Deployment-${i}' + params: { + sshRSAPublicKey: sshRSAPublicKey + stagingStorageAccountName: toLower(storageAccountDeployment.outputs.storageAccountName) + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: networkDeployment.outputs.k3sSubnetId + azureLocation: location + vmName : '${k3sArcClusterName}-Node-0${i}' + storageContainerName: toLower(k3sArcClusterName) + namingGuid: namingGuid + //deployGPUNodes: deployGPUNodes + k8sWorkerNodesSku: k8sWorkerNodesSku + } + dependsOn: [ + ubuntuRancherK3sDeployment + ] +}] + module clientVmDeployment 'clientVm/clientVm.bicep' = { name: 'clientVmDeployment' + dependsOn: [ + ubuntuRancherK3sNodesDeployment + ubuntuRancherK3sDataSvcNodesDeployment + ] params: { windowsAdminUsername: windowsAdminUsername windowsAdminPassword: windowsAdminPassword @@ -138,7 +226,6 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { //githubPAT: githubPAT location: location subnetId: networkDeployment.outputs.innerLoopSubnetId - acrName: acrName rdpPort: rdpPort namingGuid: namingGuid adxClusterName: adxClusterName @@ -191,14 +278,6 @@ module keyVault 'data/keyVault.bicep' = { } } -module acr 'kubernetes/acr.bicep' = { - name: 'acrDeployment' - params: { - acrName: acrName - location: location - } -} - module adx 'data/dataExplorer.bicep' = { name: 'adxDeployment' params: { diff --git a/azure_jumpstart_ag/contoso_motors/bicep/mgmt/network.bicep b/azure_jumpstart_ag/contoso_motors/bicep/mgmt/network.bicep index dbf41bcbb4..7e732e00f0 100644 --- a/azure_jumpstart_ag/contoso_motors/bicep/mgmt/network.bicep +++ b/azure_jumpstart_ag/contoso_motors/bicep/mgmt/network.bicep @@ -1,11 +1,12 @@ @description('Name of the Cloud VNet') param virtualNetworkNameCloud string -@description('Name of the Staging AKS subnet in the cloud virtual network') -param subnetNameCloudAksStaging string +@description('Name of the K3s subnet in the cloud virtual network') +param subnetNameCloudK3s string + +@description('Name of the inner-loop subnet in the cloud virtual network') +param subnetNameCloud string -@description('Name of the inner-loop AKS subnet in the cloud virtual network') -param subnetNameCloudAksInnerLoop string @description('Azure Region to deploy the Log Analytics Workspace') param location string = resourceGroup().location @@ -25,8 +26,8 @@ param networkSecurityGroupNameCloud string = 'Ag-NSG-Prod' param bastionNetworkSecurityGroupName string = 'Ag-NSG-Bastion' var addressPrefixCloud = '10.16.0.0/16' -var subnetAddressPrefixAksDev = '10.16.80.0/21' -var subnetAddressPrefixInnerLoop = '10.16.64.0/21' +var subnetAddressPrefixK3s = '10.16.80.0/21' +var subnetAddressPrefixCloud = '10.16.64.0/21' var bastionSubnetIpPrefix = '10.16.3.64/26' var bastionSubnetName = 'AzureBastionSubnet' var bastionSubnetRef = '${cloudVirtualNetwork.id}/subnets/${bastionSubnetName}' @@ -45,11 +46,11 @@ var bastionSubnet = [ } } ] -var cloudAKSDevSubnet = [ +var cloudK3sSubnet = [ { - name: subnetNameCloudAksStaging + name: subnetNameCloudK3s properties: { - addressPrefix: subnetAddressPrefixAksDev + addressPrefix: subnetAddressPrefixK3s privateEndpointNetworkPolicies: 'Enabled' privateLinkServiceNetworkPolicies: 'Enabled' networkSecurityGroup: { @@ -59,11 +60,11 @@ var cloudAKSDevSubnet = [ } ] -var cloudAKSInnerLoopSubnet = [ +var cloudSubnet = [ { - name: subnetNameCloudAksInnerLoop + name: subnetNameCloud properties: { - addressPrefix: subnetAddressPrefixInnerLoop + addressPrefix: subnetAddressPrefixCloud privateEndpointNetworkPolicies: 'Enabled' privateLinkServiceNetworkPolicies: 'Enabled' networkSecurityGroup: { @@ -83,7 +84,10 @@ resource cloudVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-07-01' = { addressPrefixCloud ] } - subnets: (deployBastion == false) ? union (cloudAKSDevSubnet,cloudAKSInnerLoopSubnet) : union(cloudAKSDevSubnet,cloudAKSInnerLoopSubnet,bastionSubnet) + subnets: (deployBastion == false) + ? union(cloudK3sSubnet, cloudSubnet) + : union(cloudK3sSubnet, cloudSubnet, bastionSubnet) + //subnets: (deployBastion == false) ? union (cloudAKSDevSubnet,cloudAKSInnerLoopSubnet) : union(cloudAKSDevSubnet,cloudAKSInnerLoopSubnet,bastionSubnet) } } @@ -363,6 +367,6 @@ resource bastionHost 'Microsoft.Network/bastionHosts@2023-02-01' = if (deployBas } output vnetId string = cloudVirtualNetwork.id -output devSubnetId string = cloudVirtualNetwork.properties.subnets[0].id +output k3sSubnetId string = cloudVirtualNetwork.properties.subnets[0].id output innerLoopSubnetId string = cloudVirtualNetwork.properties.subnets[1].id output virtualNetworkNameCloud string = cloudVirtualNetwork.name diff --git a/azure_jumpstart_ag/contoso_motors/bicep/storage/storageAccount.bicep b/azure_jumpstart_ag/contoso_motors/bicep/storage/storageAccount.bicep index baa298048b..f11df1e4a3 100644 --- a/azure_jumpstart_ag/contoso_motors/bicep/storage/storageAccount.bicep +++ b/azure_jumpstart_ag/contoso_motors/bicep/storage/storageAccount.bicep @@ -21,6 +21,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { } properties: { supportsHttpsTrafficOnly: true + isHnsEnabled: true } }