diff --git a/drops/arc_k8s_secret_store_extension.json b/drops/arc_k8s_secret_store_extension.json new file mode 100644 index 0000000..85bbb85 --- /dev/null +++ b/drops/arc_k8s_secret_store_extension.json @@ -0,0 +1,33 @@ +{ + "Title": "Using Secret Store extension to fetch secrets in Azure Arc-enabled Kubernetes cluster", + "Summary": "This drop includes bicep automation for deploying an Azure Arc-enabled k3s cluster with the Secret Store extension.", + "Description": "This Jumpstart drop includes automation to setup a K3s cluster, connects it to Azure Arc, and configures the Secret Store Extension to sync secrets from Azure Key Vault to Kubernetes. It handles installing dependencies and deploying a sample app that uses the synced secret.", + "Cover": "https://github.com/Azure/arc_jumpstart_drops/script_automation/arc_k8s_secret_store_extension/artifacts/media/sseArcExtensionArch.png", + "Authors": [ + { + "Name": "Zaid Mohammad", + "Link": "https://www.linkedin.com/in/zaidmohd/" + } + ], + "Source": "https://github.com/Azure/arc_jumpstart_drops/tree/main/script_automation/arc_k8s_secret_store_extension", + "Type": "script_automation", + "Difficulty": "Medium", + "ProgrammingLanguage": [ + "Bicep", + "Shell" + ], + "Products": [ + "Azure Arc", + "Arc-enabled Kubernetes", + "K3s", + "Kubernetes", + "Azure Key Vault" + ], + "LastModified": "2025-02-03T00:00:00.000Z", + "CreatedDate": "2025-02-03T00:00:00.000Z", + "Topics": [ + "Kubernetes", + "K3s", + "Security" + ] +} \ No newline at end of file diff --git a/script_automation/arc_k8s_secret_store_extension/README.md b/script_automation/arc_k8s_secret_store_extension/README.md new file mode 100644 index 0000000..817a920 --- /dev/null +++ b/script_automation/arc_k8s_secret_store_extension/README.md @@ -0,0 +1,129 @@ +## Overview + +#### Using Secret Store extension to fetch secrets in Arc-enabled Kubernetes cluster +This Jumpstart drop provides comprehensive automation to set up a lightweight Kubernetes (K3s) cluster, connect it to Azure Arc and configure the Azure Key Vault Secret Store Extension. Secret Store extension synchronizes secrets from Key Vault to your Kubernetes cluster. The automation script handles the installation of all necessary dependencies and deploys a sample application that demonstrates the use of the synchronized secrets within the Kubernetes environment. This setup ensures that your Kubernetes applications can securely access secrets stored in Key Vault, even when operating offline. + +> **Note:** This Jumpstart guide demonstrates how to set up and use the Secret Store extension. For enhanced security, it is recommended to enable encryption of the Kubernetes secret store using [KMS](https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/) plugin. + +> ⚠️ **Disclaimer:** Secret Store Extension is currently in public preview. For further details and updates on availability, please refer to the [Secret Store extension Documentation](https://learn.microsoft.com/en-us/azure/azure-arc/kubernetes/secret-store-extension). + +## Architecture +![Secret Store Extension Architecture.](./artifacts/media/sseArcExtensionArch.png) + +## Prerequisites +- Clone the Azure Arc Drops repository + + ```shell + git clone https://github.com/Azure/arc_jumpstart_drops.git + ``` + +- [Install or update Azure CLI to version 2.53.0 and above](https://learn.microsoft.com/cli/azure/install-azure-cli?view=azure-cli-latest). Use the below command to check your current installed version. + + ```shell + az --version + ``` + +- Register necessary Azure resource providers by running the following commands. + + ```shell + az provider register --namespace Microsoft.Kubernetes --wait + az provider register --namespace Microsoft.KubernetesConfiguration --wait + az provider register --namespace Microsoft.ExtendedLocation --wait + ``` + +- [Generate a new SSH key pair](https://learn.microsoft.com/azure/virtual-machines/linux/create-ssh-keys-detailed) or use an existing one (Windows 10 and above now comes with a built-in ssh client). The SSH key is used to configure secure access to the Linux virtual machines that are used to run the Kubernetes clusters. + + ```shell + ssh-keygen -t rsa -b 4096 + ``` + + To retrieve the SSH public key after it's been created, depending on your environment, use one of the below methods: + - In Linux, use the `cat ~/.ssh/id_rsa.pub` command. + - In Windows (CMD/PowerShell), use the SSH public key file that by default, is located in the _`C:\Users\WINUSER/.ssh/id_rsa.pub`_ folder. + + SSH public key example output: + + ```shell + ssh-rsa o1djFhyNe5NxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxaDU6LwM/BTO1c= user@pc + ``` + +- Edit the [main.bicepparam](https://github.com/microsoft/azure_arc/blob/main/azure_jumpstart_arcbox/bicep/main.bicepparam) template parameters file and supply values for your environment. + - _`sshRSAPublicKey`_ - Your SSH public key + - _`bastion`_ - Set to _`true`_ if you want to use Azure Bastion to connect to _js-k3s_ + +![Screenshot showing Bicep parameters.](./artifacts/media/bicepParameters.png) + +## Getting Started + +The automation performs the following steps: + +- Deploy the infrastructure and create an Azure Key Vault with a secret. +- Install the K3s cluster and onboard it as an Azure Arc-enabled Kubernetes cluster. +- Create a managed identity with access to the secret. +- Enable workload identity federation in the cluster. +- Federate a local service account with the managed identity that has access to the secret. +- Deploy the Azure Key Vault Secret Store Extension (SSE). +- Create two custom resources to define the Azure Key Vault secret to pull and how to store the secret in the cluster. +- Deploy an application pod that references the secret and prints the secret value in the logs. + +### Run the automation + +Navigate to the [deployment folder](https://raw.githubusercontent.com/Azure/arc_jumpstart_drops/sse/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/) and run the below command: + +```shell +az login +az group create --name "" --location "" +az deployment group create -g "" -f "main.bicep" -p "main.bicepparam" +``` + +> **Note:** Secret Store extension is currently available in the following regions: East US, East US2, West US, West US2, West US3, West Europe, and North Europe. For the most up-to-date list of available regions, refer to the [prerequisites](https://learn.microsoft.com/en-gb/azure/azure-arc/kubernetes/secret-store-extension?tabs=arc-k8s#prerequisites) section. + +### Verify the deployment + +- Once your deployment is complete, you can open the Azure portal and see the resources inside your resource group. You will be using the _js-k3s-*_ Azure virtual machine to review the secret store extension automation. You will need to remotely access _js-k3s-*_. + + ![Screenshot showing all deployed resources in the resource group](./artifacts/media/deployed_resources.png) + + > **Note:** For enhanced security posture, SSH (22) ports aren't open by default. You will need to create a network security group (NSG) rule to allow network access to port 22, or use [Azure Bastion](https://learn.microsoft.com/azure/bastion/bastion-overview) access to connect to the VM. + +- SSH to the js-k3s virtual machine. + ```shell + ssh jumpstartuser@js-k3s-* + ``` + ![Screenshot showing ssh to the vm](./artifacts/media/ssh.png) + +- SSE deployment contains a pod with two containers: the controller, which manages storing secrets in the cluster, and the provider, which manages access to, and pulling secrets from, the Azure Key Vault. + ```shell + kubectl --namespace azure-secret-store get pods + ``` + ![Screenshot sync controller crds](./artifacts/media/sseController.png) + +- View the secret synchronized to the k3s cluster. + ```shell + kubectl get secrets --namespace js-namespace + ``` + ![Screenshot showing k8s secrets](./artifacts/media/syncK8sSecrets.png) + +- Run below command to validate the synchronized secret values, stored in the Kubernetes secret store. You can also validate the value from the Key Vault deployed in the resource group. + ```shell + kubectl get secret js-secret-sync --namespace js-namespace -o jsonpath="{.data.js-secret}" | base64 -d + ``` + ![Screenshot showing secret value](./artifacts/media/k8sSecrets.png) + +- We have deployed a sample application running a BusyBox container that continuously prints the value of the synchronized secret via the Secret Store Extension to the logs every 30 seconds. Check the deployed application logs to see the synced secret value. + ```shell + kubectl --namespace js-namespace logs js-app-secrets-sync + ``` + ![Screenshot showing app logs](./artifacts/media/appLogs.png) + +- Run the describe command to get detailed status messages for each synchronization event. This can be used to diagnose connection or configuration errors, and to observe when the secret value changes. + ```shell + kubectl describe secretsync js-secret-sync --namespace js-namespace + ``` + ![Screenshot showing synced secret status](./artifacts/media/syncK8sSecretsStatus.png) + +### Resources + +See [Secret Store extension (preview)](https://learn.microsoft.com/en-gb/azure/azure-arc/kubernetes/secret-store-extension) for the full instructions to set this up yourself. + +To troubleshoot Secret Store extension issues, visit [Secret Store extension troubleshooting](https://learn.microsoft.com/en-us/azure/azure-arc/kubernetes/secret-store-extension#troubleshooting) diff --git a/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/main.bicep b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/main.bicep new file mode 100644 index 0000000..36bd169 --- /dev/null +++ b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/main.bicep @@ -0,0 +1,53 @@ +@description('Location for all resources') +param location string = resourceGroup().location + +@maxLength(5) +@description('Random GUID') +param namingGuid string = toLower(substring(newGuid(), 0, 5)) + +@description('Target GitHub account') +param githubAccount string = 'azure' + +@description('Target GitHub branch') +param githubBranch string = 'main' + +@description('Choice to deploy Bastion to connect to the client VM') +param deployBastion bool = false + +@description('Name of the Cloud VNet') +param virtualNetworkNameCloud string = 'js-vnet-prod' + +@description('Name of the Staging AKS subnet in the cloud virtual network') +param subnetNameCloudK3s string = 'js-subnet-k3s' + +@description('Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example \'ssh-rsa AAAAB...snip...UcyupgH azureuser@linuxvm\'') +param sshRSAPublicKey string + +@description('The name of the Azure Arc K3s cluster') +param k3sArcDataClusterName string = 'js-k3s-${namingGuid}' + +var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/arc_jumpstart_drops/${githubBranch}/script_automation/arc_k8s_secret_store_extension/' + +module mgmtArtifacts 'modules/mgmtArtifacts.bicep' = { + name: 'mgmtArtifacts' + params: { + virtualNetworkNameCloud: virtualNetworkNameCloud + subnetNameCloudK3s: subnetNameCloudK3s + deployBastion: deployBastion + location: location + namingGuid: namingGuid + } +} +module k3sSecretStoreDeployment 'modules/k3s.bicep' = { + name: 'ubuntuRancherK3s2Deployment' + params: { + sshRSAPublicKey: sshRSAPublicKey + templateBaseUrl: templateBaseUrl + subnetId: mgmtArtifacts.outputs.k3sSubnetId + azureLocation: location + vmName : k3sArcDataClusterName + namingGuid: namingGuid + keyVaultName: mgmtArtifacts.outputs.keyVaultName + userAssignedIdentityName: mgmtArtifacts.outputs.userAssignedIdentityName + } +} diff --git a/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/main.bicepparam b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/main.bicepparam new file mode 100644 index 0000000..8a862dc --- /dev/null +++ b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/main.bicepparam @@ -0,0 +1,7 @@ +using 'main.bicep' + +param deployBastion = false +param sshRSAPublicKey = '' + + + diff --git a/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/modules/k3s.bicep b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/modules/k3s.bicep new file mode 100644 index 0000000..f3ffec5 --- /dev/null +++ b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/modules/k3s.bicep @@ -0,0 +1,193 @@ +@description('The name of you Virtual Machine') +param vmName string = 'js-k3s-${namingGuid}' + +@description('Username for the Virtual Machine') +param adminUsername string = 'jumpstartuser' + +@description('RSA public key used for securing SSH access to ArcBox resources. This parameter is only needed when deploying the DataOps or DevOps flavors.') +@secure() +param sshRSAPublicKey string = '' + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version') +@allowed([ + '22_04-lts-gen2' +]) +param ubuntuOSVersion string = '22_04-lts-gen2' + +@description('Location for all resources.') +param azureLocation string = resourceGroup().location + +@description('The size of the VM') +param vmSize string = 'Standard_B4ms' + +@description('Resource Id of the subnet in the virtual network') +param subnetId string + +@description('The base URL used for accessing artifacts and automation artifacts') +param templateBaseUrl string + +@maxLength(5) +@description('Random GUID') +param namingGuid string + +@description('The name of the Key Vault') +param keyVaultName string + +@description('The name of the Key Vault secret') +param keyVaultSecretName string = 'js-secret' + +@description('Azure Key Vault tenant ID') +param tenantId string = subscription().tenantId + +@description('The name of the user assigned identity') +param userAssignedIdentityName string + +@description('The name of the Kubernetes namespace') +param kubernetesNamespace string = 'js-namespace' + +@description('The name of the service account') +param serviceAccountName string = 'js-sa' + +@description('The name of the federated credential identity') +param federatedCredentialIdentityName string = 'js-fci' + +@description('Cert Manager version') +param certManagerVersion string = 'v1.16.2' + +var publicIpAddressName = '${vmName}-pip' +var networkInterfaceName = '${vmName}-nic' +var osDiskType = 'Premium_LRS' +var diskSize = 32 +var numberOfIPAddresses = 1 // The number of IP addresses to create + +// Create multiple public IP addresses +resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { + name: '${publicIpAddressName}${i}' + location: azureLocation + properties: { + publicIPAllocationMethod: 'Static' + publicIPAddressVersion: 'IPv4' + idleTimeoutInMinutes: 4 + } + sku: { + name: 'Basic' + } +}] + +// Create multiple NIC IP configurations and assign the public IP to the IP configuration +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [for i in range(1, numberOfIPAddresses): { + name: 'ipconfig${i}' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + publicIPAddress: { + id: publicIpAddresses[i-1].id + } + primary: i == 1 ? true : false + } + }] + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { + name: vmName + location: azureLocation + identity: { + type: 'SystemAssigned' + } + properties: { + hardwareProfile: { + vmSize: vmSize + } + storageProfile: { + osDisk: { + name: '${vmName}-osdisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + diskSizeGB: diskSize + } + imageReference: { + publisher: 'canonical' + offer: '0001-com-ubuntu-server-jammy' + sku: ubuntuOSVersion + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: networkInterface.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + linuxConfiguration: { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: sshRSAPublicKey + } + ] + } + } + } + } +} + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + principalType: 'ServicePrincipal' + } +} + +// Add role assignment for the VM: Key Vault Secrets Officer +resource vmRoleAssignment_KVSecretsOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Key Vault Secrets Officer') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7') + principalType: 'ServicePrincipal' + } +} + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3sWithSSE' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash k3sWithSSE.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${templateBaseUrl} ${resourceGroup().name} ${keyVaultName} ${keyVaultSecretName} ${tenantId} ${userAssignedIdentityName} ${kubernetesNamespace} ${serviceAccountName} ${federatedCredentialIdentityName} ${certManagerVersion}' + fileUris: [ + '${templateBaseUrl}scripts/k3sWithSSE.sh' + ] + } + } + dependsOn: [ + vmRoleAssignment_Owner + vmRoleAssignment_KVSecretsOfficer + ] +} diff --git a/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/modules/mgmtArtifacts.bicep b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/modules/mgmtArtifacts.bicep new file mode 100644 index 0000000..4b1ae45 --- /dev/null +++ b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/modules/mgmtArtifacts.bicep @@ -0,0 +1,357 @@ +@description('Name of the Cloud VNet') +param virtualNetworkNameCloud string + +@description('Name of the K3s subnet in the cloud virtual network') +param subnetNameCloudK3s string + +@description('Azure Region to deploy the Azure resources') +param location string = resourceGroup().location + +@description('Resource tag for Jumpstart Drops') +param resourceTags object = { + Project: 'Jumpstart_Drops' +} + +@description('Choice to deploy Bastion to connect to the client VM') +param deployBastion bool = false + +@description('Name of the prod Network Security Group') +param networkSecurityGroupNameCloud string = 'js-nsg-prod' + +@description('Name of the Bastion Network Security Group') +param bastionNetworkSecurityGroupName string = 'js-nsg-bastion' + +@description('Azure Key Vault tenant ID') +param tenantId string = subscription().tenantId + +@description('Azure Key Vault SKU') +param akvSku string = 'standard' + +@maxLength(5) +@description('Random GUID') +param namingGuid string + +@description('The name of the user assigned identity') +param userAssignedIdentityName string = 'js-uai-sse' + +var addressPrefixCloud = '10.16.0.0/16' +var subnetAddressPrefixK3s = '10.16.80.0/21' +var bastionSubnetIpPrefix = '10.16.3.64/26' +var bastionSubnetName = 'AzureBastionSubnet' +var bastionSubnetRef = '${cloudVirtualNetwork.id}/subnets/${bastionSubnetName}' +var bastionName = 'js-bastion' +var bastionPublicIpAddressName = '${bastionName}-pip' +var keyVaultName = 'js-kv-${namingGuid}' + +var bastionSubnet = [ + { + name: 'AzureBastionSubnet' + properties: { + addressPrefix: bastionSubnetIpPrefix + networkSecurityGroup: { + id: bastionNetworkSecurityGroup.id + } + } + } +] +var cloudK3sSubnet = [ + { + name: subnetNameCloudK3s + properties: { + addressPrefix: subnetAddressPrefixK3s + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + networkSecurityGroup: { + id: networkSecurityGroupCloud.id + } + } + } +] + +resource cloudVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-07-01' = { + name: virtualNetworkNameCloud + location: location + tags: resourceTags + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefixCloud + ] + } + subnets: (deployBastion == false) + ? cloudK3sSubnet + : union(cloudK3sSubnet, bastionSubnet) + } +} + +resource publicIpAddress 'Microsoft.Network/publicIPAddresses@2023-02-01' = if (deployBastion == true) { + name: bastionPublicIpAddressName + location: location + tags: resourceTags + properties: { + publicIPAllocationMethod: 'Static' + publicIPAddressVersion: 'IPv4' + idleTimeoutInMinutes: 4 + } + sku: { + name: 'Standard' + } +} + +resource networkSecurityGroupCloud 'Microsoft.Network/networkSecurityGroups@2023-02-01' = { + name: networkSecurityGroupNameCloud + location: location + tags: resourceTags + properties: { + securityRules: [ + { + name: 'allow_k8s_80' + properties: { + priority: 1000 + protocol: 'TCP' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '80' + } + } + { + name: 'allow_k8s_8080' + properties: { + priority: 1010 + protocol: 'TCP' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '8080' + } + } + { + name: 'allow_k8s_443' + properties: { + priority: 1020 + protocol: 'TCP' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '443' + } + } + { + name: 'allow_k8s_kubelet' + properties: { + priority: 1060 + protocol: 'Tcp' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '10250' + } + } + ] + } +} + +resource bastionNetworkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-02-01' = if (deployBastion == true) { + name: bastionNetworkSecurityGroupName + location: location + tags: resourceTags + properties: { + securityRules: [ + { + name: 'bastion_allow_https_inbound' + properties: { + priority: 1010 + protocol: 'TCP' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: 'Internet' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '443' + } + } + { + name: 'bastion_allow_gateway_manager_inbound' + properties: { + priority: 1011 + protocol: 'TCP' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: 'GatewayManager' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '443' + } + } + { + name: 'bastion_allow_load_balancer_inbound' + properties: { + priority: 1012 + protocol: 'TCP' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: 'AzureLoadBalancer' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '443' + } + } + { + name: 'bastion_allow_host_comms' + properties: { + priority: 1013 + protocol: '*' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: 'VirtualNetwork' + sourcePortRange: '*' + destinationAddressPrefix: 'VirtualNetwork' + destinationPortRanges: [ + '8080' + '5701' + ] + } + } + { + name: 'bastion_allow_ssh_rdp_outbound' + properties: { + priority: 1014 + protocol: '*' + access: 'Allow' + direction: 'Outbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: 'VirtualNetwork' + destinationPortRanges: [ + '22' + '3389' + ] + } + } + { + name: 'bastion_allow_azure_cloud_outbound' + properties: { + priority: 1015 + protocol: 'TCP' + access: 'Allow' + direction: 'Outbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: 'AzureCloud' + destinationPortRange: '443' + } + } + { + name: 'bastion_allow_bastion_comms' + properties: { + priority: 1016 + protocol: '*' + access: 'Allow' + direction: 'Outbound' + sourceAddressPrefix: 'VirtualNetwork' + sourcePortRange: '*' + destinationAddressPrefix: 'VirtualNetwork' + destinationPortRanges: [ + '8080' + '5701' + ] + } + } + { + name: 'bastion_allow_get_session_info' + properties: { + priority: 1017 + protocol: '*' + access: 'Allow' + direction: 'Outbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: 'Internet' + destinationPortRanges: [ + '80' + '443' + ] + } + } + ] + } +} + +resource bastionHost 'Microsoft.Network/bastionHosts@2023-02-01' = if (deployBastion == true) { + name: bastionName + location: location + tags: resourceTags + properties: { + ipConfigurations: [ + { + name: 'IpConf' + properties: { + publicIPAddress: { + id: publicIpAddress.id + } + subnet: { + id: bastionSubnetRef + } + } + } + ] + } +} + +resource akv 'Microsoft.KeyVault/vaults@2023-02-01' = { + name: keyVaultName + location: location + tags: resourceTags + properties: { + sku: { + name: akvSku + family: 'A' + } + enableRbacAuthorization: true + enableSoftDelete: false + tenantId: tenantId + } +} + +// Create User Assigned Identity +resource userAssignedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-07-31-preview' = { + location: location + name: userAssignedIdentityName +} + +// Add role assignment for the UAI: Key Vault Reader +resource userAssignedIdentity_KVReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(userAssignedIdentity.id, 'Microsoft.Authorization/roleAssignments', 'Key Vault Reader') + scope: resourceGroup() + properties: { + principalId: userAssignedIdentity.properties.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '21090545-7ca7-4776-b22c-e363652d74d2') + principalType: 'ServicePrincipal' + } +} + +// Add role assignment for the UAI: Key Vault Secrets User +resource userAssignedIdentity_KVSecretsUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(userAssignedIdentity.id, 'Microsoft.Authorization/roleAssignments', 'Key Vault Secrets User') + scope: resourceGroup() + properties: { + principalId: userAssignedIdentity.properties.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '4633458b-17de-408a-b874-0445c86b69e6') + principalType: 'ServicePrincipal' + } +} + +output vnetId string = cloudVirtualNetwork.id +output k3sSubnetId string = cloudVirtualNetwork.properties.subnets[0].id +output virtualNetworkNameCloud string = cloudVirtualNetwork.name +output keyVaultName string = keyVaultName +output userAssignedIdentityName string = userAssignedIdentityName diff --git a/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/scripts/k3sWithSSE.sh b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/scripts/k3sWithSSE.sh new file mode 100644 index 0000000..eaaaad9 --- /dev/null +++ b/script_automation/arc_k8s_secret_store_extension/artifacts/Bicep/scripts/k3sWithSSE.sh @@ -0,0 +1,381 @@ +#!/bin/bash +sudo apt-get update + +sudo sed -i "s/PasswordAuthentication no/PasswordAuthentication yes/" /etc/ssh/sshd_config +sudo adduser staginguser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password +sudo echo "staginguser:ArcPassw0rd" | sudo chpasswd + +# Injecting environment variables +echo '#!/bin/bash' >> vars.sh +echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$2 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$3 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $templateBaseUrl:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $resourceGroup:$6| awk '{print substr($1,2); }' >> vars.sh +echo $keyVaultName:$7| awk '{print substr($1,2); }' >> vars.sh +echo $keyVaultSecretName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $azureTenantId:$9 | awk '{print substr($1,2); }' >> vars.sh +echo $userAssignedIdentityName:${10} | awk '{print substr($1,2); }' >> vars.sh +echo $kubernetesNamespace:${11} | awk '{print substr($1,2); }' >> vars.sh +echo $serviceAccountName:${12} | awk '{print substr($1,2); }' >> vars.sh +echo $federatedCredentialIdentityName:${13} | awk '{print substr($1,2); }' >> vars.sh +echo $certManagerVersion:${14} | awk '{print substr($1,2); }' >> vars.sh + +sed -i '2s/^/export adminUsername=/' vars.sh +sed -i '3s/^/export subscriptionId=/' vars.sh +sed -i '4s/^/export vmName=/' vars.sh +sed -i '5s/^/export location=/' vars.sh +sed -i '6s/^/export templateBaseUrl=/' vars.sh +sed -i '7s/^/export resourceGroup=/' vars.sh +sed -i '8s/^/export keyVaultName=/' vars.sh +sed -i '9s/^/export keyVaultSecretName=/' vars.sh +sed -i '10s/^/export azureTenantId=/' vars.sh +sed -i '11s/^/export userAssignedIdentityName=/' vars.sh +sed -i '12s/^/export kubernetesNamespace=/' vars.sh +sed -i '13s/^/export serviceAccountName=/' vars.sh +sed -i '14s/^/export federatedCredentialIdentityName=/' vars.sh +sed -i '15s/^/export certManagerVersion=/' vars.sh + +export vmName=$3 + +# Save the original stdout and stderr +exec 3>&1 4>&2 + +exec >k3sWithSSE-${vmName}.log +exec 2>&1 + +# Set k3 deployment variables +export K3S_VERSION="1.32.0+k3s1" # Do not change! + +chmod +x vars.sh +. ./vars.sh + +# Creating login message of the day (motd) +curl -v -o /etc/profile.d/welcomeK3s.sh ${templateBaseUrl}scripts/welcomeK3s.sh + +# Syncing this script log to 'jumpstart_logs' directory for ease of troubleshooting +sudo -u $adminUsername mkdir -p /home/${adminUsername}/jumpstart_logs +while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/k3sWithSSE-$vmName.log /home/${adminUsername}/jumpstart_logs/k3sWithSSE-$vmName.log; done & + +# Function to check if dpkg lock is in place +check_dpkg_lock() { + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do + echo "Waiting for other package management processes to complete..." + sleep 5 + done +} +# Run the lock check before attempting the installation +check_dpkg_lock + +# Installing Azure CLI & Azure Arc extensions +max_retries=5 +retry_count=0 +success=false + +while [ $retry_count -lt $max_retries ]; do + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + if [ $? -eq 0 ]; then + success=true + break + else + echo "Failed to install Az CLI. Retrying (Attempt $((retry_count+1)))..." + retry_count=$((retry_count+1)) + sleep 10 + fi +done + +echo "" +echo "Log in to Azure" +echo "" +for i in {1..5}; do + sudo -u $adminUsername az login --identity + if [[ $? -eq 0 ]]; then + break + fi + sleep 15 + if [[ $i -eq 5 ]]; then + echo "Error: Failed to login to Azure after 5 retries" + exit 1 + fi +done + +sudo -u $adminUsername az account set --subscription $subscriptionId +az -v + +check_dpkg_lock + +# Installing Azure Arc extensions +echo "" +echo "Installing Azure Arc extensions" +echo "" +sudo -u $adminUsername az extension add --name connectedk8s +sudo -u $adminUsername az extension add --name k8s-configuration +sudo -u $adminUsername az extension add --name k8s-extension + +# Installing Rancher K3s cluster (single control plane) +echo "" +echo "Installing Rancher K3s cluster" +echo "" +publicIp=$(hostname -i) +sudo mkdir ~/.kube +sudo -u $adminUsername mkdir /home/${adminUsername}/.kube +curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --node-ip ${publicIp} --node-external-ip ${publicIp} --bind-address ${publicIp} --tls-san ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} K3S_KUBECONFIG_MODE="644" sh - +if [[ $? -ne 0 ]]; then + echo "ERROR: K3s installation failed" + exit 1 +fi +# Renaming default context to k3s cluster name +context=$(echo $vmName | sed 's/-[^-]*$//') +sudo kubectl config rename-context default $context --kubeconfig /etc/rancher/k3s/k3s.yaml +sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config +sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config +sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging +sudo chown -R $adminUsername /home/${adminUsername}/.kube/ +sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging + +# Installing Helm 3 +echo "" +echo "Installing Helm" +echo "" +sudo snap install helm --classic +if [[ $? -ne 0 ]]; then + echo "ERROR: Helm installation failed" + exit 1 +fi + +echo "" +echo "Making sure Rancher K3s cluster is ready..." +echo "" +sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null +sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' + +# Onboard the cluster to Azure Arc +echo "" +echo "Onboarding the cluster to Azure Arc" +echo "" + +max_retries=5 +retry_count=0 +success=false + +while [ $retry_count -lt $max_retries ]; do + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --enable-oidc-issuer + if [ $? -eq 0 ]; then + success=true + break + else + echo "Failed to onboard cluster to Azure Arc. Retrying (Attempt $((retry_count+1)))..." + retry_count=$((retry_count+1)) + sleep 10 + fi +done + +if [ "$success" = false ]; then + echo "Error: Failed to onboard the cluster to Azure Arc after $max_retries attempts." + exit 1 +fi + +echo "" +echo "Onboarding the k3s cluster to Azure Arc completed" +echo "" + +# Verify if cluster is connected to Azure Arc successfully +connectedClusterInfo=$(sudo -u $adminUsername az connectedk8s show --name $vmName --resource-group $resourceGroup) +echo "Connected cluster info: $connectedClusterInfo" + +# Function to check if an extension is already installed +is_extension_installed() { + extension_name=$1 + extension_count=$(sudo -u $adminUsername az k8s-extension list --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --query "[?name=='$extension_name'] | length(@)") + + if [ "$extension_count" -gt 0 ]; then + return 0 # Extension is installed + else + return 1 # Extension is not installed + fi +} +serviceAccountIssuer=$(sudo -u $adminUsername az connectedk8s show --name $vmName --resource-group $resourceGroup --query "oidcIssuerProfile.issuerUrl" --output tsv) +echo "" +echo "OIDC issuer URL: $serviceAccountIssuer" +echo "" + +# Ensure the last line is empty and delete it if it is +sudo sed -i '${/^$/d}' /etc/systemd/system/k3s.service + +# Append the required flags to the k3s.service file +sudo sed -i '$ a\ '\''--kube-apiserver-arg=--enable-admission-plugins=OwnerReferencesPermissionEnforcement'\'' \\' /etc/systemd/system/k3s.service +if [ $? -ne 0 ]; then + echo "ERROR: Failed to append enable-admission-plugins to k3s.service" + exit 1 +fi + +sudo sed -i "$ a\ '--kube-apiserver-arg=--service-account-issuer=${serviceAccountIssuer}'" /etc/systemd/system/k3s.service +if [ $? -ne 0 ]; then + echo "ERROR: Failed to append service-account-issuer to k3s.service" + exit 1 +fi + +# Reload systemd daemon and restart k3s service +sudo systemctl daemon-reload +if [ $? -ne 0 ]; then + echo "ERROR: Failed to reload systemd daemon" + exit 1 +fi + +sudo systemctl restart k3s +if [ $? -ne 0 ]; then + echo "ERROR: Failed to restart k3s service" + exit 1 +fi + +# Set up the Azure Key Vault and create a secret +max_retries=5 +retry_count=0 +success=false + +while [ $retry_count -lt $max_retries ]; do + sudo -u $adminUsername az keyvault secret set --vault-name $keyVaultName --name $keyVaultSecretName --value $(openssl rand -base64 10) + if [ $? -eq 0 ]; then + success=true + break + else + echo "Failed to set secret in Key Vault. Retrying (Attempt $((retry_count+1)))..." + retry_count=$((retry_count+1)) + sleep 10 + fi +done + +if [ "$success" = false ]; then + echo "Error: Failed to set secret in Key Vault after $max_retries attempts." + exit 1 +fi + +# Create a federated identity credential +cat <