diff --git a/src/templates/finops-hub/modules/azuretimezones.bicep b/src/templates/finops-hub/modules/azuretimezones.bicep index 316208660..04540d375 100644 --- a/src/templates/finops-hub/modules/azuretimezones.bicep +++ b/src/templates/finops-hub/modules/azuretimezones.bicep @@ -22,7 +22,7 @@ param timezoneobject object = { koreacentral: 'Korea Standard Time' koreasouth: 'Korea Standard Time' northcentralus: 'Central Standard Time' - northeurope: 'GMT Standard Time' + northeurope: 'W. Europe Standard Time' norwayeast: 'W. Europe Standard Time' norwaywest: 'W. Europe Standard Time' southcentralus: 'Central Standard Time' @@ -30,8 +30,8 @@ param timezoneobject object = { southeastasia: 'Singapore Standard Time' switzerlandnorth: 'W. Europe Standard Time' switzerlandwest: 'W. Europe Standard Time' - uksouth: 'GMT Standard Time' - ukwest: 'GMT Standard Time' + uksouth: 'Greenwich Mean Time' + ukwest: 'Greenwich Mean Time' westcentralus: 'Central Standard Time' westeurope: 'W. Europe Standard Time' westindia: 'India Standard Time' diff --git a/src/templates/finops-hub/modules/dataFactory.bicep b/src/templates/finops-hub/modules/dataFactory.bicep index 6c9dcecae..39b10e57b 100644 --- a/src/templates/finops-hub/modules/dataFactory.bicep +++ b/src/templates/finops-hub/modules/dataFactory.bicep @@ -59,6 +59,9 @@ param tagsByResource object = {} @description('Optional. Enable public access.') param enablePublicAccess bool +@description('Optional. Sets schedule-based triggers base time.') +param triggersBaseTime string = utcNow('u') + //------------------------------------------------------------------------------ // Variables //------------------------------------------------------------------------------ @@ -69,12 +72,22 @@ var exportApiVersion = '2023-07-01-preview' var hubDataExplorerName = 'hubDataExplorer' // Function to generate the body for a Cost Management export -func getExportBody(exportContainerName string, datasetType string, schemaVersion string, isMonthly bool, exportFormat string, compressionMode string, partitionData string, dataOverwriteBehavior string) string => '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${schemaVersion}", "filters": [] }, "granularity": "Daily" }, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{variables(\'exportName\')}", "name": "@{variables(\'exportName\')}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' +func getExportBody( + exportContainerName string, + datasetType string, + schemaVersion string, + isMonthly bool, + exportFormat string, + compressionMode string, + partitionData string, + dataOverwriteBehavior string +) string => + '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${schemaVersion}", "filters": [] }, "granularity": "Daily" }, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{variables(\'exportName\')}", "name": "@{variables(\'exportName\')}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' var deployDataExplorer = !empty(dataExplorerId) var datasetPropsDefault = { - location: { + location: { type: 'AzureBlobFSLocation' fileName: { value: '@{dataset().fileName}' @@ -100,6 +113,7 @@ var exportManifestAddedTriggerName = '${safeExportContainerName}_ManifestAdded' var ingestionManifestAddedTriggerName = '${safeIngestionContainerName}_ManifestAdded' var updateConfigTriggerName = '${safeConfigContainerName}_SettingsUpdated' var dailyTriggerName = '${safeConfigContainerName}_DailySchedule' +var dailyTriggerDateTime = dateTimeAdd(triggersBaseTime, 'PT1H') var monthlyTriggerName = '${safeConfigContainerName}_MonthlySchedule' var allHubTriggers = [ exportManifestAddedTriggerName @@ -169,20 +183,20 @@ resource managedIntegrationRuntime 'Microsoft.DataFactory/factories/integrationR computeProperties: { location: location dataFlowProperties: { - computeType: 'General' - coreCount: 8 - timeToLive: 10 - cleanup: false - customProperties: [] + computeType: 'General' + coreCount: 8 + timeToLive: 10 + cleanup: false + customProperties: [] } copyComputeScaleProperties: { - dataIntegrationUnit: 16 - timeToLive: 30 + dataIntegrationUnit: 16 + timeToLive: 30 } pipelineExternalComputeScaleProperties: { - timeToLive: 30 - numberOfPipelineNodes: 1 - numberOfExternalNodes: 1 + timeToLive: 30 + numberOfPipelineNodes: 1 + numberOfExternalNodes: 1 } } } @@ -302,29 +316,38 @@ module approveDataExplorerPrivateEndpointConnections 'dataExplorerEndpoints.bice resource triggerManagerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { name: '${dataFactory.name}_triggerManager' location: location - tags: union(tags, contains(tagsByResource, 'Microsoft.ManagedIdentity/userAssignedIdentities') ? tagsByResource['Microsoft.ManagedIdentity/userAssignedIdentities'] : {}) + tags: union( + tags, + contains(tagsByResource, 'Microsoft.ManagedIdentity/userAssignedIdentities') + ? tagsByResource['Microsoft.ManagedIdentity/userAssignedIdentities'] + : {} + ) } -resource triggerManagerRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for role in autoStartRbacRoles: { - name: guid(dataFactory.id, role, triggerManagerIdentity.id) - scope: dataFactory - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role) - principalId: triggerManagerIdentity.properties.principalId - principalType: 'ServicePrincipal' +resource triggerManagerRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ + for role in autoStartRbacRoles: { + name: guid(dataFactory.id, role, triggerManagerIdentity.id) + scope: dataFactory + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role) + principalId: triggerManagerIdentity.properties.principalId + principalType: 'ServicePrincipal' + } } -}] +] // Grant ADF identity access to manage data in storage -resource factoryIdentityStorageRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for role in storageRbacRoles: { - name: guid(storageAccount.id, role, dataFactory.id) - scope: storageAccount - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role) - principalId: dataFactory.identity.principalId - principalType: 'ServicePrincipal' +resource factoryIdentityStorageRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ + for role in storageRbacRoles: { + name: guid(storageAccount.id, role, dataFactory.id) + scope: storageAccount + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role) + principalId: dataFactory.identity.principalId + principalType: 'ServicePrincipal' + } } -}] +] //------------------------------------------------------------------------------ // Delete old triggers and pipelines @@ -344,7 +367,12 @@ resource deleteOldResources 'Microsoft.Resources/deploymentScripts@2020-10-01' = dependsOn: [ triggerManagerRoleAssignments ] - tags: union(tags, contains(tagsByResource, 'Microsoft.Resources/deploymentScripts') ? tagsByResource['Microsoft.Resources/deploymentScripts'] : {}) + tags: union( + tags, + contains(tagsByResource, 'Microsoft.Resources/deploymentScripts') + ? tagsByResource['Microsoft.Resources/deploymentScripts'] + : {} + ) properties: { azPowerShellVersion: '8.0' retentionInterval: 'PT1H' @@ -427,10 +455,12 @@ resource linkedService_keyVault 'Microsoft.DataFactory/factories/linkedservices@ typeProperties: { baseUrl: reference('Microsoft.KeyVault/vaults/${keyVault.name}', '2023-02-01').vaultUri } - connectVia: enablePublicAccess ? null : { - referenceName: managedIntegrationRuntime.name - type: 'IntegrationRuntimeReference' - } + connectVia: enablePublicAccess + ? null + : { + referenceName: managedIntegrationRuntime.name + type: 'IntegrationRuntimeReference' + } } } @@ -445,10 +475,12 @@ resource linkedService_storageAccount 'Microsoft.DataFactory/factories/linkedser typeProperties: { url: reference('Microsoft.Storage/storageAccounts/${storageAccount.name}', '2021-08-01').primaryEndpoints.dfs } - connectVia: enablePublicAccess ? null : { - referenceName: managedIntegrationRuntime.name - type: 'IntegrationRuntimeReference' - } + connectVia: enablePublicAccess + ? null + : { + referenceName: managedIntegrationRuntime.name + type: 'IntegrationRuntimeReference' + } } } @@ -470,10 +502,34 @@ resource linkedService_dataExplorer 'Microsoft.DataFactory/factories/linkedservi tenant: dataFactory.identity.tenantId servicePrincipalId: dataFactory.identity.principalId } - connectVia: enablePublicAccess ? null : { - referenceName: managedIntegrationRuntime.name - type: 'IntegrationRuntimeReference' - } + connectVia: enablePublicAccess + ? null + : { + referenceName: managedIntegrationRuntime.name + type: 'IntegrationRuntimeReference' + } + } +} + +var armEndpointPropertyName = 'aadResourceId' // This is a workaround to avoid the warning about "ResourceId" in the property name +resource linkedService_arm 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = { + name: 'azurerm' + parent: dataFactory + properties: { + annotations: [] + parameters: {} + type: 'RestService' + typeProperties: union( + { + url: environment().resourceManager + authenticationType: 'ManagedServiceIdentity' + enableServerCertificateValidation: true + }, + { + // When bicep sees "ResourceId" in the following property name, it raises a warning. The union and variable work around this to avoid the warning. + '${armEndpointPropertyName}': environment().resourceManager + } + ) } } @@ -496,10 +552,12 @@ resource linkedService_remoteHubStorage 'Microsoft.DataFactory/factories/linkeds secretName: '${toLower(hubName)}-storage-key' } } - connectVia: enablePublicAccess ? null : { - referenceName: managedIntegrationRuntime.name - type: 'IntegrationRuntimeReference' - } + connectVia: enablePublicAccess + ? null + : { + referenceName: managedIntegrationRuntime.name + type: 'IntegrationRuntimeReference' + } } } @@ -520,10 +578,12 @@ resource linkedService_ftkRepo 'Microsoft.DataFactory/factories/linkedservices@2 enableServerCertificateValidation: true authenticationType: 'Anonymous' } - connectVia: enablePublicAccess ? null : { - referenceName: managedIntegrationRuntime.name - type: 'IntegrationRuntimeReference' - } + connectVia: enablePublicAccess + ? null + : { + referenceName: managedIntegrationRuntime.name + type: 'IntegrationRuntimeReference' + } } } @@ -564,7 +624,7 @@ resource dataset_manifest 'Microsoft.DataFactory/factories/datasets@2018-06-01' parameters: { fileName: { type: 'String' - defaultValue: 'manifest.json' + defaultValue: 'manifest.json' } folderPath: { type: 'String' @@ -700,7 +760,9 @@ resource dataset_ingestion 'Microsoft.DataFactory/factories/datasets@2018-06-01' } linkedServiceName: { parameters: {} - referenceName: empty(remoteHubStorageUri) ? linkedService_storageAccount.name : linkedService_remoteHubStorage.name + referenceName: empty(remoteHubStorageUri) + ? linkedService_storageAccount.name + : linkedService_remoteHubStorage.name type: 'LinkedServiceReference' } } @@ -729,7 +791,9 @@ resource dataset_ingestion_files 'Microsoft.DataFactory/factories/datasets@2018- } linkedServiceName: { parameters: {} - referenceName: empty(remoteHubStorageUri) ? linkedService_storageAccount.name : linkedService_remoteHubStorage.name + referenceName: empty(remoteHubStorageUri) + ? linkedService_storageAccount.name + : linkedService_remoteHubStorage.name type: 'LinkedServiceReference' } } @@ -799,6 +863,24 @@ resource dataset_ftkReleaseFile 'Microsoft.DataFactory/factories/datasets@2018-0 } } +resource dataset_resourcegraph 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: 'resourcegraph' + parent: dataFactory + properties: { + annotations: [] + parameters: {} + type: 'RestResource' + typeProperties: { + relativeUrl: '/providers/Microsoft.ResourceGraph/resources?api-version=2022-10-01' + } + linkedServiceName: { + parameters: {} + referenceName: linkedService_arm.name + type: 'LinkedServiceReference' + } + } +} + //------------------------------------------------------------------------------ // Triggers //------------------------------------------------------------------------------ @@ -914,13 +996,22 @@ resource trigger_DailySchedule 'Microsoft.DataFactory/factories/triggers@2018-06 Recurrence: 'Daily' } } + { + pipelineReference: { + referenceName: pipeline_ExecuteQueries.name + type: 'PipelineReference' + } + parameters: { + Recurrence: 'Daily' + } + } ] type: 'ScheduleTrigger' typeProperties: { recurrence: { - frequency: 'Hour' - interval: 24 - startTime: '2023-01-01T01:01:00' + frequency: 'Day' + interval: 1 + startTime: dailyTriggerDateTime timeZone: azuretimezones.outputs.Timezone } } @@ -976,7 +1067,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- parent: dataFactory properties: { activities: [ - { // Get Config + { + // Get Config name: 'Get Config' type: 'Lookup' dependsOn: [] @@ -1006,7 +1098,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Set Version + { + // Set Version name: 'Set Version' type: 'SetVariable' dependsOn: [ @@ -1026,7 +1119,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Set Scopes + { + // Set Scopes name: 'Set Scopes' type: 'SetVariable' dependsOn: [ @@ -1046,7 +1140,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Set Retention + { + // Set Retention name: 'Set Retention' type: 'SetVariable' dependsOn: [ @@ -1066,7 +1161,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Until Capacity Is Available + { + // Until Capacity Is Available name: 'Until Capacity Is Available' type: 'Until' dependsOn: [ @@ -1096,7 +1192,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- type: 'Expression' } activities: [ - { // Confirm Ingestion Capacity + { + // Confirm Ingestion Capacity name: 'Confirm Ingestion Capacity' type: 'AzureDataExplorerCommand' dependsOn: [] @@ -1120,7 +1217,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // If Has Capacity + { + // If Has Capacity name: 'If Has Capacity' type: 'IfCondition' dependsOn: [ @@ -1138,7 +1236,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- type: 'Expression' } ifFalseActivities: [ - { // Wait for Ingestion + { + // Wait for Ingestion name: 'Wait for Ingestion' type: 'Wait' dependsOn: [] @@ -1147,7 +1246,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- waitTimeInSeconds: 15 } } - { // Try Again + { + // Try Again name: 'Try Again' type: 'SetVariable' dependsOn: [ @@ -1170,7 +1270,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } ] ifTrueActivities: [ - { // Save ingestion policy in ADX + { + // Save ingestion policy in ADX name: 'Set ingestion policy in ADX' type: 'AzureDataExplorerCommand' dependsOn: [] @@ -1197,7 +1298,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Save Hub Settings in ADX + { + // Save Hub Settings in ADX name: 'Save Hub Settings in ADX' type: 'AzureDataExplorerCommand' dependsOn: [ @@ -1231,7 +1333,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Update PricingUnits in ADX + { + // Update PricingUnits in ADX name: 'Update PricingUnits in ADX' type: 'AzureDataExplorerCommand' dependsOn: [ @@ -1262,7 +1365,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Update Regions in ADX + { + // Update Regions in ADX name: 'Update Regions in ADX' type: 'AzureDataExplorerCommand' dependsOn: [ @@ -1293,7 +1397,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Update ResourceTypes in ADX + { + // Update ResourceTypes in ADX name: 'Update ResourceTypes in ADX' type: 'AzureDataExplorerCommand' dependsOn: [ @@ -1324,7 +1429,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Update Services in ADX + { + // Update Services in ADX name: 'Update Services in ADX' type: 'AzureDataExplorerCommand' dependsOn: [ @@ -1355,7 +1461,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } - { // Ingestion Complete + { + // Ingestion Complete name: 'Ingestion Complete' type: 'SetVariable' dependsOn: [ @@ -1379,7 +1486,8 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- ] } } - { // Abort On Error + { + // Abort On Error name: 'Abort On Error' type: 'SetVariable' dependsOn: [ @@ -1404,14 +1512,15 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- timeout: '0.02:00:00' } } - { // Timeout Error + { + // Timeout Error name: 'Timeout Error' type: 'Fail' dependsOn: [ { activity: 'Until Capacity Is Available' dependencyConditions: [ - 'Failed' + 'Failed' ] } ] @@ -1450,7 +1559,8 @@ resource pipeline_StartBackfillProcess 'Microsoft.DataFactory/factories/pipeline parent: dataFactory properties: { activities: [ - { // Get Config + { + // Get Config name: 'Get Config' type: 'Lookup' dependsOn: [] @@ -1490,7 +1600,8 @@ resource pipeline_StartBackfillProcess 'Microsoft.DataFactory/factories/pipeline } } } - { // Set backfill end date + { + // Set backfill end date name: 'Set backfill end date' type: 'SetVariable' dependsOn: [ @@ -1510,7 +1621,8 @@ resource pipeline_StartBackfillProcess 'Microsoft.DataFactory/factories/pipeline } } } - { // Set backfill start date + { + // Set backfill start date name: 'Set backfill start date' type: 'SetVariable' dependsOn: [ @@ -1530,7 +1642,8 @@ resource pipeline_StartBackfillProcess 'Microsoft.DataFactory/factories/pipeline } } } - { // Set export start date + { + // Set export start date name: 'Set export start date' type: 'SetVariable' dependsOn: [ @@ -1550,7 +1663,8 @@ resource pipeline_StartBackfillProcess 'Microsoft.DataFactory/factories/pipeline } } } - { // Set export end date + { + // Set export end date name: 'Set export end date' type: 'SetVariable' dependsOn: [ @@ -1570,7 +1684,8 @@ resource pipeline_StartBackfillProcess 'Microsoft.DataFactory/factories/pipeline } } } - { // Every Month + { + // Every Month name: 'Every Month' type: 'Until' dependsOn: [ @@ -1713,7 +1828,8 @@ resource pipeline_RunBackfillJob 'Microsoft.DataFactory/factories/pipelines@2018 parent: dataFactory properties: { activities: [ - { // Get Config + { + // Get Config name: 'Get Config' type: 'Lookup' dependsOn: [] @@ -1753,7 +1869,8 @@ resource pipeline_RunBackfillJob 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // Set Scopes + { + // Set Scopes name: 'Set Scopes' description: 'Save scopes to test if it is an array' type: 'SetVariable' @@ -1778,7 +1895,8 @@ resource pipeline_RunBackfillJob 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // Set Scopes as Array + { + // Set Scopes as Array name: 'Set Scopes as Array' description: 'Wraps a single scope object into an array to work around the PowerShell bug where single-item arrays are sometimes written as a single object instead of an array.' type: 'SetVariable' @@ -1803,7 +1921,8 @@ resource pipeline_RunBackfillJob 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // Filter Invalid Scopes + { + // Filter Invalid Scopes name: 'Filter Invalid Scopes' description: 'Remove any invalid scopes to avoid errors.' type: 'Filter' @@ -1834,7 +1953,8 @@ resource pipeline_RunBackfillJob 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // ForEach Export Scope + { + // ForEach Export Scope name: 'ForEach Export Scope' type: 'ForEach' dependsOn: [ @@ -1892,7 +2012,7 @@ resource pipeline_RunBackfillJob 'Microsoft.DataFactory/factories/pipelines@2018 } method: 'POST' headers: { - 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunBackfill@${ftkVersion}' + 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunBackfill@${ftkVersion}' 'Content-Type': 'application/json' ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}' } @@ -1960,7 +2080,8 @@ resource pipeline_StartExportProcess 'Microsoft.DataFactory/factories/pipelines@ parent: dataFactory properties: { activities: [ - { // Get Config + { + // Get Config name: 'Get Config' type: 'Lookup' dependsOn: [] @@ -2000,7 +2121,8 @@ resource pipeline_StartExportProcess 'Microsoft.DataFactory/factories/pipelines@ } } } - { // Set Scopes + { + // Set Scopes name: 'Set Scopes' description: 'Save scopes to test if it is an array' type: 'SetVariable' @@ -2025,7 +2147,8 @@ resource pipeline_StartExportProcess 'Microsoft.DataFactory/factories/pipelines@ } } } - { // Set Scopes as Array + { + // Set Scopes as Array name: 'Set Scopes as Array' description: 'Wraps a single scope object into an array to work around the PowerShell bug where single-item arrays are sometimes written as a single object instead of an array.' type: 'SetVariable' @@ -2050,7 +2173,8 @@ resource pipeline_StartExportProcess 'Microsoft.DataFactory/factories/pipelines@ } } } - { // Filter Invalid Scopes + { + // Filter Invalid Scopes name: 'Filter Invalid Scopes' description: 'Remove any invalid scopes to avoid errors.' type: 'Filter' @@ -2081,7 +2205,8 @@ resource pipeline_StartExportProcess 'Microsoft.DataFactory/factories/pipelines@ } } } - { // ForEach Export Scope + { + // ForEach Export Scope name: 'ForEach Export Scope' type: 'ForEach' dependsOn: [ @@ -2281,7 +2406,7 @@ resource pipeline_RunExportJobs 'Microsoft.DataFactory/factories/pipelines@2018- type: 'String' defaultValue: environment().resourceManager } - hubName: { + hubName: { type: 'String' defaultValue: hubName } @@ -2299,7 +2424,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 parent: dataFactory properties: { activities: [ - { // Get Config + { + // Get Config name: 'Get Config' type: 'Lookup' dependsOn: [] @@ -2339,7 +2465,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } } } - { // Save Scopes + { + // Save Scopes name: 'Save Scopes' type: 'SetVariable' dependsOn: [ @@ -2363,7 +2490,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } } } - { // Save Scopes as Array + { + // Save Scopes as Array name: 'Save Scopes as Array' type: 'SetVariable' dependsOn: [ @@ -2387,7 +2515,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } } } - { // Filter Invalid Scopes + { + // Filter Invalid Scopes name: 'Filter Invalid Scopes' type: 'Filter' dependsOn: [ @@ -2417,7 +2546,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } } } - { // ForEach Export Scope + { + // ForEach Export Scope name: 'ForEach Export Scope' type: 'ForEach' dependsOn: [ @@ -2436,7 +2566,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } isSequential: true activities: [ - { // 'Create or update open month focus export' + { + // 'Create or update open month focus export' name: 'Create or update open month focus export' type: 'WebActivity' dependsOn: [ @@ -2462,7 +2593,16 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } method: 'PUT' body: { - value: getExportBody(exportContainerName, 'FocusCost', focusSchemaVersion, false, 'Parquet', 'Snappy', 'true', 'CreateNewReport') + value: getExportBody( + exportContainerName, + 'FocusCost', + focusSchemaVersion, + false, + 'Parquet', + 'Snappy', + 'true', + 'CreateNewReport' + ) type: 'Expression' } authentication: { @@ -2474,7 +2614,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } } } - { // 'Set open month focus export name' + { + // 'Set open month focus export name' name: 'Set open month focus export name' type: 'SetVariable' dependsOn: [] @@ -2491,7 +2632,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } } } - { // 'Create or update closed month focus export' + { + // 'Create or update closed month focus export' name: 'Create or update closed month focus export' type: 'WebActivity' dependsOn: [ @@ -2517,7 +2659,16 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } method: 'PUT' body: { - value: getExportBody(exportContainerName, 'FocusCost', focusSchemaVersion, true, 'Parquet', 'Snappy', 'true', 'CreateNewReport') + value: getExportBody( + exportContainerName, + 'FocusCost', + focusSchemaVersion, + true, + 'Parquet', + 'Snappy', + 'true', + 'CreateNewReport' + ) type: 'Expression' } authentication: { @@ -2529,7 +2680,8 @@ resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@20 } } } - { // 'Set closed month focus export name' + { + // 'Set closed month focus export name' name: 'Set closed month focus export name' type: 'SetVariable' dependsOn: [ @@ -2602,7 +2754,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 parent: dataFactory properties: { activities: [ - { // Wait + { + // Wait name: 'Wait' description: 'Files may not be available immediately after being created.' type: 'Wait' @@ -2612,7 +2765,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 waitTimeInSeconds: 60 } } - { // Read Manifest + { + // Read Manifest name: 'Read Manifest' description: 'Load the export manifest to determine the scope, dataset, and date range.' type: 'Lookup' @@ -2658,7 +2812,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Set Has No Rows + { + // Set Has No Rows name: 'Set Has No Rows' description: 'Check the row count ' type: 'SetVariable' @@ -2683,7 +2838,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Set Export Dataset Type + { + // Set Export Dataset Type name: 'Set Export Dataset Type' description: 'Save the dataset type from the export manifest.' type: 'SetVariable' @@ -2708,7 +2864,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Set MCA Column + { + // Set MCA Column name: 'Set MCA Column' description: 'Determines if the dataset schema has channel-specific columns and saves the column name that only exists in MCA to determine if it is an MCA dataset.' type: 'SetVariable' @@ -2733,7 +2890,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Set Export Dataset Version + { + // Set Export Dataset Version name: 'Set Export Dataset Version' description: 'Save the dataset version from the export manifest.' type: 'SetVariable' @@ -2758,7 +2916,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Detect Channel + { + // Detect Channel name: 'Detect Channel' description: 'Determines what channel this export is from. Switch statement handles the different file types if the mcaColumnToCheck variable is set.' type: 'Switch' @@ -2789,7 +2948,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 type: 'Expression' } cases: [ - { // csv + { + // csv value: 'csv' activities: [ { @@ -2855,7 +3015,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } ] } - { // gz + { + // gz value: 'gz' activities: [ { @@ -2921,7 +3082,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } ] } - { // parquet + { + // parquet value: 'parquet' activities: [ { @@ -3009,7 +3171,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 ] } } - { // Set Scope + { + // Set Scope name: 'Set Scope' description: 'Save the scope from the export manifest.' type: 'SetVariable' @@ -3034,7 +3197,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Set Date + { + // Set Date name: 'Set Date' description: 'Save the exported month from the export manifest.' type: 'SetVariable' @@ -3059,7 +3223,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Error: ManifestReadFailed + { + // Error: ManifestReadFailed name: 'Failed to Read Manifest' type: 'Fail' dependsOn: [ @@ -3097,7 +3262,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 errorCode: 'ManifestReadFailed' } } - { // Check Schema + { + // Check Schema name: 'Check Schema' description: 'Verify that the schema file exists in storage.' type: 'GetMetadata' @@ -3152,7 +3318,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Error: SchemaNotFound + { + // Error: SchemaNotFound name: 'Schema Not Found' type: 'Fail' dependsOn: [ @@ -3170,7 +3337,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 errorCode: 'SchemaNotFound' } } - { // Set Hub Dataset + { + // Set Hub Dataset name: 'Set Hub Dataset' type: 'SetVariable' dependsOn: [ @@ -3194,7 +3362,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // Set Destination Folder + { + // Set Destination Folder name: 'Set Destination Folder' type: 'SetVariable' dependsOn: [ @@ -3224,7 +3393,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 } } } - { // For Each Blob + { + // For Each Blob name: 'For Each Blob' description: 'Loop thru each exported file listed in the manifest.' type: 'ForEach' @@ -3245,7 +3415,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 batchCount: enablePublicAccess ? 30 : 4 // so we don't overload the managed runtime isSequential: false activities: [ - { // Execute + { + // Execute name: 'Execute' description: 'Run the ingestion ETL pipeline.' type: 'ExecutePipeline' @@ -3295,7 +3466,8 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 ] } } - { // Copy Manifest + { + // Copy Manifest name: 'Copy Manifest' description: 'Copy the manifest to the ingestion container to trigger ADX ingestion' type: 'Copy' @@ -3419,7 +3591,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 parent: dataFactory properties: { activities: [ - { // Get Existing Parquet Files + { + // Get Existing Parquet Files name: 'Get Existing Parquet Files' description: 'Get the previously ingested files so we can remove any older data. This is necessary to avoid data duplication in reports.' type: 'GetMetadata' @@ -3452,7 +3625,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } } } - { // Filter Out Current Exports + { + // Filter Out Current Exports name: 'Filter Out Current Exports' description: 'Remove existing files from the current export so those files do not get deleted.' type: 'Filter' @@ -3476,7 +3650,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } } } - { // Load Schema Mappings + { + // Load Schema Mappings name: 'Load Schema Mappings' description: 'Get schema mapping file to use for the CSV to parquet conversion.' type: 'Lookup' @@ -3514,7 +3689,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } } } - { // Error: SchemaLoadFailed + { + // Error: SchemaLoadFailed name: 'Failed to Load Schema' type: 'Fail' dependsOn: [ @@ -3534,7 +3710,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 errorCode: 'SchemaLoadFailed' } } - { // Set Additional Columns + { + // Set Additional Columns name: 'Set Additional Columns' type: 'SetVariable' dependsOn: [ @@ -3558,7 +3735,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } } } - { // For Each Old File + { + // For Each Old File name: 'For Each Old File' description: 'Loop thru each of the existing files from previous exports.' type: 'ForEach' @@ -3583,7 +3761,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 type: 'Expression' } activities: [ - { // Delete Old Ingested File + { + // Delete Old Ingested File name: 'Delete Old Ingested File' description: 'Delete the previously ingested files from older exports.' type: 'Delete' @@ -3618,7 +3797,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 ] } } - { // Set Destination Path + { + // Set Destination Path name: 'Set Destination Path' type: 'SetVariable' dependsOn: [] @@ -3635,7 +3815,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } } } - { // Convert to Parquet + { + // Convert to Parquet name: 'Convert to Parquet' description: 'Convert CSV to parquet and move the file to the ${ingestionContainerName} container.' type: 'Switch' @@ -3666,10 +3847,12 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 type: 'Expression' } cases: [ - { // CSV + { + // CSV value: 'csv' activities: [ - { // Convert CSV File + { + // Convert CSV File name: 'Convert CSV File' type: 'Copy' dependsOn: [] @@ -3742,10 +3925,12 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } ] } - { // GZ + { + // GZ value: 'gz' activities: [ - { // Convert GZip CSV File + { + // Convert GZip CSV File name: 'Convert GZip CSV File' type: 'Copy' dependsOn: [] @@ -3818,10 +4003,12 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } ] } - { // Parquet + { + // Parquet value: 'parquet' activities: [ - { // Move Parquet File + { + // Move Parquet File name: 'Move Parquet File' type: 'Copy' dependsOn: [] @@ -3892,7 +4079,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } ] defaultActivities: [ - { // Error: UnsupportedFileType + { + // Error: UnsupportedFileType name: 'Unsupported File Type' type: 'Fail' dependsOn: [] @@ -3908,7 +4096,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 ] } } - { // Read Hub Config + { + // Read Hub Config name: 'Read Hub Config' description: 'Read the hub config to determine if the export should be retained.' type: 'Lookup' @@ -3943,7 +4132,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 } } } - { // If Not Retaining Exports + { + // If Not Retaining Exports name: 'If Not Retaining Exports' description: 'If the msexports retention period <= 0, delete the source file. The main reason to keep the source file is to allow for troubleshooting and reprocessing in the future.' type: 'IfCondition' @@ -3968,7 +4158,8 @@ resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06 type: 'Expression' } ifTrueActivities: [ - { // Delete Source File + { + // Delete Source File name: 'Delete Source File' description: 'Delete the exported data file to keep storage costs down. This file is not referenced by any reporting systems.' type: 'Delete' @@ -4049,12 +4240,12 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 parent: dataFactory properties: { activities: [ - { // Read Hub Config + { + // Read Hub Config name: 'Read Hub Config' description: 'Read the hub config to determine how long data should be retained.' type: 'Lookup' - dependsOn: [ - ] + dependsOn: [] policy: { timeout: '0.12:00:00' retry: 0 @@ -4085,7 +4276,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // Set Final Retention Months + { + // Set Final Retention Months name: 'Set Final Retention Months' type: 'SetVariable' dependsOn: [ @@ -4109,7 +4301,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // Until Capacity Is Available + { + // Until Capacity Is Available name: 'Until Capacity Is Available' type: 'Until' dependsOn: [ @@ -4128,7 +4321,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 type: 'Expression' } activities: [ - { // Confirm Ingestion Capacity + { + // Confirm Ingestion Capacity name: 'Confirm Ingestion Capacity' type: 'AzureDataExplorerCommand' dependsOn: [] @@ -4149,7 +4343,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 type: 'LinkedServiceReference' } } - { // If Has Capacity + { + // If Has Capacity name: 'If Has Capacity' type: 'IfCondition' dependsOn: [ @@ -4167,7 +4362,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 type: 'Expression' } ifFalseActivities: [ - { // Wait for Ingestion + { + // Wait for Ingestion name: 'Wait for Ingestion' type: 'Wait' dependsOn: [] @@ -4176,7 +4372,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 waitTimeInSeconds: 15 } } - { // Try Again + { + // Try Again name: 'Try Again' type: 'SetVariable' dependsOn: [ @@ -4199,7 +4396,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 } ] ifTrueActivities: [ - { // Pre-Ingest Cleanup + { + // Pre-Ingest Cleanup name: 'Pre-Ingest Cleanup' description: 'Cost Management exports include all month-to-date data from the previous export run. To ensure data is not double-reported, it must be dropped from the raw table before ingestion completes. Remove previous ingestions into the raw table for the month and any previous runs of the current ingestion month file in any table.' type: 'AzureDataExplorerCommand' @@ -4226,7 +4424,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // Ingest Data + { + // Ingest Data name: 'Ingest Data' type: 'AzureDataExplorerCommand' dependsOn: [ @@ -4260,7 +4459,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // Post-Ingest Cleanup + { + // Post-Ingest Cleanup name: 'Post-Ingest Cleanup' description: 'Cost Management exports include all month-to-date data from the previous export run. To ensure data is not double-reported, it must be dropped after ingestion completes. Remove the current ingestion month file from raw and any old ingestions for the month from the final table.' type: 'AzureDataExplorerCommand' @@ -4294,7 +4494,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 } } } - { // Ingestion Complete + { + // Ingestion Complete name: 'Ingestion Complete' type: 'SetVariable' dependsOn: [ @@ -4315,7 +4516,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 value: false } } - { // Abort On Ingestion Error + { + // Abort On Ingestion Error name: 'Abort On Ingestion Error' type: 'SetVariable' dependsOn: [ @@ -4336,7 +4538,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 value: false } } - { // Error: DataExplorerIngestionFailed + { + // Error: DataExplorerIngestionFailed name: 'Ingestion Failed Error' type: 'Fail' dependsOn: [ @@ -4356,7 +4559,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 errorCode: 'DataExplorerIngestionFailed' } } - { // Abort On Pre-Ingest Drop Error + { + // Abort On Pre-Ingest Drop Error name: 'Abort On Pre-Ingest Drop Error' type: 'SetVariable' dependsOn: [ @@ -4377,7 +4581,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 value: false } } - { // Error: DataExplorerPreIngestionDropFailed + { + // Error: DataExplorerPreIngestionDropFailed name: 'Pre-Ingest Drop Failed Error' type: 'Fail' dependsOn: [ @@ -4397,7 +4602,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 errorCode: 'DataExplorerPreIngestionDropFailed' } } - { // Abort On Post-Ingest Drop Error + { + // Abort On Post-Ingest Drop Error name: 'Abort On Post-Ingest Drop Error' type: 'SetVariable' dependsOn: [ @@ -4418,7 +4624,8 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 value: false } } - { // Error: DataExplorerPostIngestionDropFailed + { + // Error: DataExplorerPostIngestionDropFailed name: 'Post-Ingest Drop Failed Error' type: 'Fail' dependsOn: [ @@ -4492,26 +4699,28 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines properties: { concurrency: 1 activities: [ - { // Wait - name: 'Wait' - description: 'Files may not be available immediately after being created.' - type: 'Wait' - dependsOn: [] - userProperties: [] - typeProperties: { - waitTimeInSeconds: 60 - } + { + // Wait + name: 'Wait' + description: 'Files may not be available immediately after being created.' + type: 'Wait' + dependsOn: [] + userProperties: [] + typeProperties: { + waitTimeInSeconds: 60 + } } - { // Set Container Folder Path + { + // Set Container Folder Path name: 'Set Container Folder Path' type: 'SetVariable' dependsOn: [ - { - activity: 'Wait' - dependencyConditions: [ - 'Succeeded' - ] - } + { + activity: 'Wait' + dependencyConditions: [ + 'Succeeded' + ] + } ] policy: { secureOutput: false @@ -4526,7 +4735,8 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines } } } - { // Get Existing Parquet Files + { + // Get Existing Parquet Files name: 'Get Existing Parquet Files' description: 'Get the previously ingested files so we can get file paths.' type: 'GetMetadata' @@ -4566,7 +4776,8 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines } } } - { // Filter Out Folders and manifest files + { + // Filter Out Folders and manifest files name: 'Filter Out Folders' description: 'Remove any folders or manifest files.' type: 'Filter' @@ -4590,16 +4801,17 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines } } } - { // Set Ingestion Timestamp + { + // Set Ingestion Timestamp name: 'Set Ingestion Timestamp' type: 'SetVariable' dependsOn: [ - { - activity: 'Wait' - dependencyConditions: [ - 'Succeeded' - ] - } + { + activity: 'Wait' + dependencyConditions: [ + 'Succeeded' + ] + } ] policy: { secureOutput: false @@ -4614,7 +4826,8 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines } } } - { // For Each Old File + { + // For Each Old File name: 'For Each Old File' description: 'Loop thru each of the existing files.' type: 'ForEach' @@ -4640,7 +4853,8 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines type: 'Expression' } activities: [ - { // Execute + { + // Execute name: 'Execute' description: 'Run the ADX ETL pipeline.' type: 'ExecutePipeline' @@ -4682,7 +4896,8 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines ] } } - { // If No Files + { + // If No Files name: 'If No Files' description: 'If there are no files found, fail the pipeline.' type: 'IfCondition' @@ -4701,7 +4916,8 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines type: 'Expression' } ifTrueActivities: [ - { // Error: IngestionFilesNotFound + { + // Error: IngestionFilesNotFound name: 'Files Not Found' type: 'Fail' dependsOn: [] @@ -4737,6 +4953,503 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines } } +//------------------------------------------------------------------------------ +// queries export pipeline +// Triggered by daily trigger +//------------------------------------------------------------------------------ +@description('Queues the queries_ETL_ingestion pipeline to extract query results from multiple sources (e.g., Resource Graph)') +resource pipeline_ExecuteQueries 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: 'queries_ExecuteETL' + parent: dataFactory + properties: { + activities: [ + { + // Load Queries + name: 'Load Queries' + type: 'Lookup' + dependsOn: [] + policy: { + timeout: '0.00:10:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'JsonSource' + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: true + wildcardFileName: '*.json' + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + } + dataset: { + referenceName: dataset_config.name + type: 'DatasetReference' + parameters: { + fileName: 'settings.json' + folderPath: '${configContainerName}/queries' + } + } + firstRowOnly: false + } + } + { + // Iterate Files + name: 'Iterate Files' + type: 'ForEach' + dependsOn: [ + { + activity: 'Load Queries' + dependencyConditions: [ + 'Succeeded' + ] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@activity(\'Load Queries\').output.value' + type: 'Expression' + } + batchCount: 2 + isSequential: false + activities: [ + { + // Execute File Queries + name: 'Execute File Queries' + description: 'Execute the queries declared in the queries file.' + type: 'ExecutePipeline' + dependsOn: [] + policy: { + secureInput: false + } + userProperties: [] + typeProperties: { + pipeline: { + referenceName: pipeline_ExecuteQueries_query.name + type: 'PipelineReference' + } + waitOnCompletion: true + parameters: { + inputDataset: { + value: '@item().inputDataset' + type: 'Expression' + } + outputDataset: { + value: '@item().outputDataset' + type: 'Expression' + } + schemaFile: { + value: '@item().schemaFile' + type: 'Expression' + } + queryScope: { + value: '@item().scope' + type: 'Expression' + } + query: { + value: '@item().query' + type: 'Expression' + } + queryVersion: { + value: '@item().version' + type: 'Expression' + } + querySource: { + value: '@item().source' + type: 'Expression' + } + queryProvider: { + value: '@item().provider' + type: 'Expression' + } + queryType: { + value: '@item().type' + type: 'Expression' + } + } + } + } + ] + } + } + ] + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + +//------------------------------------------------------------------------------ +// specific query export pipeline +// Triggered by queries_ExecuteETL pipeline +//------------------------------------------------------------------------------ +@description('Extracts query results from a specific source (e.g., Resource Graph)') +resource pipeline_ExecuteQueries_query 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: 'queries_ETL_ingestion' + parent: dataFactory + properties: { + activities: [ + { + // Set blob timestamp + name: 'Set Blob Timestamp' + type: 'SetVariable' + dependsOn: [] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'blobExportTimestamp' + value: { + value: '@concat(utcNow(\'yyyy\'),\'/\',utcNow(\'MM\'),\'/\',utcNow(\'dd\'),\'/\')' + type: 'Expression' + } + } + } + { + // Set instance id + name: 'Set Instance Id' + type: 'SetVariable' + dependsOn: [] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'instanceId' + value: { + value: '@guid()' + type: 'Expression' + } + } + } + { + // Set initial query error value + name: 'Set Query Error Value' + type: 'SetVariable' + dependsOn: [] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'queryError' + value: { + value: '@string(\'\')' + type: 'Expression' + } + } + } + { + // Set blob base path + name: 'Set Blob Base Path' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Set Blob Timestamp' + dependencyConditions: ['Succeeded'] + } + { + activity: 'Set Instance Id' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'blobBasePath' + value: { + value: '@concat(pipeline().parameters.outputDataset, \'/\', variables(\'blobExportTimestamp\'), pipeline().parameters.queryScope, \'/\', variables(\'instanceId\'), \'${ingestionIdFileNameSeparator}\')' + type: 'Expression' + } + } + } + { + // Load Schema Mappings + name: 'Load Schema Mappings' + type: 'Lookup' + dependsOn: [] + policy: { + timeout: '0.12:00:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'JsonSource' + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: true + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + } + dataset: { + referenceName: dataset_config.name + type: 'DatasetReference' + parameters: { + fileName: { + value: '@pipeline().parameters.schemaFile' + type: 'Expression' + } + folderPath: '${configContainerName}/schemas' + } + } + } + } + { + // Error: SchemaLoadFailed + name: 'Failed to Load Schema' + type: 'Fail' + dependsOn: [ + { + activity: 'Load Schema Mappings' + dependencyConditions: ['Failed'] + } + ] + userProperties: [] + typeProperties: { + message: { + value: '@concat(\'Unable to load the \', pipeline().parameters.schemaFile, \' queries schema file. Please confirm the schema and version are supported for FinOps hubs ingestion. Unsupported files will remain in the ingestion container.\')' + type: 'Expression' + } + errorCode: 'SchemaLoadFailed' + } + } + { + type: 'Switch' + name: 'Switch Query Type' + dependsOn: [ + { + activity: 'Set Blob Base Path' + dependencyConditions: ['Succeeded'] + } + { + activity: 'Load Schema Mappings' + dependencyConditions: ['Succeeded'] + } + { + activity: 'Set Query Error Value' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + on: { + value: '@pipeline().parameters.inputDataset' + type: 'Expression' + } + cases: [ + { + value: dataset_resourcegraph.name + activities: [ + { + name: 'Execute ARG Query' + type: 'Copy' + dependsOn: [] + policy: { + timeout: '0.00:10:00' + retry: 0 + retryIntervalInSeconds: 60 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'RestSource' + httpRequestTimeout: '00:02:00' + requestInterval: '00.00:00:00.050' + requestMethod: 'POST' + requestBody: { + value: '@concat(\'{ "query": "\', pipeline().parameters.query, \' | extend x_SourceName=\\"\', pipeline().parameters.querySource, \'\\", x_SourceType=\\"\', pipeline().parameters.queryType, \'\\", x_SourceProvider=\\"\', pipeline().parameters.queryProvider, \'\\", x_SourceVersion=\\"\', pipeline().parameters.queryVersion, \'\\"" }\')' + type: 'Expression' + } + additionalHeaders: { + 'Content-Type': 'application/json' + } + } + sink: { + type: 'ParquetSink' + storeSettings: { + type: 'AzureBlobFSWriteSettings' + } + formatSettings: { + type: 'ParquetWriteSettings' + fileExtension: '.parquet' + } + } + enableStaging: false + translator: { + value: '@activity(\'Load Schema Mappings\').output.firstRow.translator' + type: 'Expression' + } + } + inputs: [ + { + referenceName: dataset_resourcegraph.name + type: 'DatasetReference' + parameters: {} + } + ] + outputs: [ + { + referenceName: dataset_ingestion.name + type: 'DatasetReference' + parameters: { + blobPath: { + value: '@concat(variables(\'blobBasePath\'), pipeline().parameters.queryType, \'.parquet\')' + type: 'Expression' + } + } + } + ] + } + { + name: 'Set ARG Query Error' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Execute ARG Query' + dependencyConditions: ['Failed'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'queryError' + value: { + value: '@activity(\'Execute ARG Query\').output.errors[0].Message' + type: 'Expression' + } + } + } + ] + } + ] + defaultActivities: [ + { + type: 'Fail' + name: 'Unsupported input dataset' + userProperties: [] + typeProperties: { + message: { + value: '@concat(\'Unable to execute the specified query because the input data set is not supported. Dataset: \', pipeline().parameters.inputDataset)' + type: 'Expression' + } + errorCode: 'UnsupportedInputDataset' + } + } + ] + } + } + { + // Catch Query Failure + name: 'Catch Query Failure' + type: 'IfCondition' + dependsOn: [ + { + activity: 'Switch Query Type' + dependencyConditions: ['Completed'] + } + ] + userProperties: [] + typeProperties: { + expression: { + value: '@and(not(empty(variables(\'queryError\'))), not(contains(variables(\'queryError\'), \'Sequence contains no elements\')))' + type: 'Expression' + } + ifTrueActivities: [ + { + name: 'Fail Pipeline' + type: 'Fail' + dependsOn: [] + userProperties: [] + typeProperties: { + message: { + value: '@concat(\'Pipeline failed due to a \', pipeline().parameters.inputDataset, \' query error. Error: \', variables(\'queryError\'))' + type: 'Expression' + } + errorCode: 'QueryFailed' + } + } + ] + } + } + ] + policy: { + elapsedTimeMetric: {} + } + variables: { + instanceId: { + type: 'String' + } + blobExportTimestamp: { + type: 'String' + } + blobBasePath: { + type: 'String' + } + queryError: { + type: 'String' + } + } + parameters: { + inputDataset: { + type: 'string' + } + outputDataset: { + type: 'string' + } + schemaFile: { + type: 'string' + } + queryScope: { + type: 'string' + } + query: { + type: 'string' + } + queryVersion: { + type: 'string' + } + querySource: { + type: 'string' + } + queryProvider: { + type: 'string' + } + queryType: { + type: 'string' + } + } + annotations: [] + } +} + //------------------------------------------------------------------------------ // Start all triggers //------------------------------------------------------------------------------ @@ -4745,7 +5458,12 @@ resource startTriggers 'Microsoft.Resources/deploymentScripts@2020-10-01' = { name: '${dataFactory.name}_startTriggers' // chinaeast2 is the only region in China that supports deployment scripts location: startsWith(location, 'china') ? 'chinaeast2' : location - tags: union(tags, contains(tagsByResource, 'Microsoft.Resources/deploymentScripts') ? tagsByResource['Microsoft.Resources/deploymentScripts'] : {}) + tags: union( + tags, + contains(tagsByResource, 'Microsoft.Resources/deploymentScripts') + ? tagsByResource['Microsoft.Resources/deploymentScripts'] + : {} + ) identity: { type: 'UserAssigned' userAssignedIdentities: { @@ -4785,7 +5503,7 @@ resource startTriggers 'Microsoft.Resources/deploymentScripts@2020-10-01' = { } { name: 'Pipelines' - value: join([ pipeline_InitializeHub.name ], '|') + value: join([pipeline_InitializeHub.name], '|') } ] } diff --git a/src/templates/finops-hub/modules/scripts/Copy-FileToAzureBlob.ps1 b/src/templates/finops-hub/modules/scripts/Copy-FileToAzureBlob.ps1 index cd9e151cf..8d5d18492 100644 --- a/src/templates/finops-hub/modules/scripts/Copy-FileToAzureBlob.ps1 +++ b/src/templates/finops-hub/modules/scripts/Copy-FileToAzureBlob.ps1 @@ -182,3 +182,14 @@ $schemaFiles.PSObject.Properties | ForEach-Object { $_.Value | Out-File $tempPath Set-AzStorageBlobContent @storageContext -File $tempPath -Blob "schemas/$fileName" -Force | Out-Null } + +# Save query files to storage +$queryFiles = $env:queryFiles | ConvertFrom-Json -Depth 10 +Write-Output "Uploading ${$queryFiles.PSObject.Properties.Count} query files..." +$queryFiles.PSObject.Properties | ForEach-Object { + $fileName = "$($_.Name).json" + $tempPath = "./$fileName" + Write-Output " Uploading $($_.Name).json..." + $_.Value | Out-File $tempPath + Set-AzStorageBlobContent @storageContext -File $tempPath -Blob "queries/$fileName" -Force | Out-Null +} diff --git a/src/templates/finops-hub/modules/scripts/HubSetup.kql b/src/templates/finops-hub/modules/scripts/HubSetup.kql index 55a294b03..3b5a6f647 100644 --- a/src/templates/finops-hub/modules/scripts/HubSetup.kql +++ b/src/templates/finops-hub/modules/scripts/HubSetup.kql @@ -57,6 +57,14 @@ Recommendations_v1_0() database('Ingestion').Recommendations_final_v1_0 } +// Recommendations_final_v1_0r2 +.create-or-alter function +with (docstring = 'Gets all recommendations aligned to FOCUS 1.0.', folder = 'Recommendations') +Recommendations_v1_0r2() +{ + database('Ingestion').Recommendations_final_v1_0r2 +} + // Transactions_final_v1_0 .create-or-alter function @@ -99,7 +107,7 @@ Prices() with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') Recommendations() { - Recommendations_v1_0() + Recommendations_v1_0r2() } diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-AdvisorCost.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-AdvisorCost.json new file mode 100644 index 000000000..9f059c897 --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-AdvisorCost.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "advisorresources | where type == 'microsoft.advisor/recommendations' | where properties.category == 'Cost' | project x_RecommendationId=id, x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory=tostring(properties.category), x_RecommendationImpact=tostring(properties.impact), x_RecommendationTypeId= tostring(properties.recommendationTypeId), x_RecommendationControl=tostring(properties.extendedProperties.recommendationControl), x_RecommendationMaturityLevel=tostring(properties.extendedProperties.maturityLevel), x_RecommendationDescription=tostring(properties.shortDescription.problem), x_RecommendationSolution=tostring(properties.shortDescription.solution), ResourceId=tolower(properties.resourceMetadata.resourceId), x_ResourceType=tolower(properties.impactedField), ResourceName=tolower(properties.impactedValue), x_RecommendationDetails=tostring(properties.extendedProperties), x_RecommendationDate=tostring(properties.lastUpdated) | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "Advisor", + "type": "HubsRecommendations-AdvisorCost", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-BackendlessAppGateways.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-BackendlessAppGateways.json new file mode 100644 index 000000000..a2d57b467 --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-BackendlessAppGateways.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'Microsoft.Network/applicationGateways' | extend backendPoolsCount = array_length(properties.backendAddressPools),SKUName= tostring(properties.sku.name), SKUTier= tostring(properties.sku.tier),SKUCapacity=properties.sku.capacity,backendPools=properties.backendAddressPools,resourceGroup=strcat('/subscriptions/',subscriptionId,'/resourceGroups/',resourceGroup)| project id, name, SKUName, SKUTier, SKUCapacity,resourceGroup,subscriptionId, AppGWName=name, type, Location=location| join ( resources | where type =~ 'Microsoft.Network/applicationGateways' | mvexpand backendPools = properties.backendAddressPools | extend backendIPCount = array_length(backendPools.properties.backendIPConfigurations) | extend backendAddressesCount = array_length(backendPools.properties.backendAddresses) | extend backendPoolName = backendPools.properties.backendAddressPools.name | summarize backendIPCount = sum(backendIPCount) ,backendAddressesCount=sum(backendAddressesCount) by id) on id| project-away id1| where (backendIPCount == 0 or isempty(backendIPCount)) and (backendAddressesCount==0 or isempty(backendAddressesCount))| project x_RecommendationId=strcat(tolower(id),'-idle'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='High', x_RecommendationTypeId='4f69df93-5972-44e0-97cf-4343c2bcf4b8', x_RecommendationControl='UsageOptimization/OrphanedResources', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='Application Gateway without any backend pool', x_RecommendationSolution='Review and remove this resource if not needed', ResourceId = tolower(id), x_ResourceType=type, ResourceName=tolower(AppGWName), x_RecommendationDetails= strcat('{\\\"backendIPCount\\\": ', backendIPCount, ',\\\"Location\\\": \\\"', Location, '\\\" }'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-BackendlessAppGateways", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-BackendlessLoadBalancers.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-BackendlessLoadBalancers.json new file mode 100644 index 000000000..04af36b66 --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-BackendlessLoadBalancers.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/loadbalancers' and array_length(properties.backendAddressPools) == 0 and sku.name!='Basic' | extend SKUName=tostring(sku.name) | extend SKUTier=tostring(sku.tier), Location=location | extend backendAddressPools = properties.backendAddressPools | extend id,name, SKUName,SKUTier,backendAddressPools, location,resourceGroup, subscriptionId, type| project x_RecommendationId=strcat(tolower(id),'-idle'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='High', x_RecommendationTypeId='ab703887-fa23-4915-abdc-3defbea89f7a', x_RecommendationControl='UsageOptimization/OrphanedResources', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='Load balancer without a backend pool', x_RecommendationSolution='Review and remove this resource if not needed', ResourceId = tolower(id), x_ResourceType=type, ResourceName=tolower(name), x_RecommendationDetails= strcat('{\\\"SKUName\\\": ', SKUName, ',\\\"SKUTier\\\": \\\"', SKUTier, ',\\\"Location\\\": \\\"', Location, '\\\" }'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-BackendlessLoadBalancers", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-EmptySQLElasticPools.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-EmptySQLElasticPools.json new file mode 100644 index 000000000..e72047f03 --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-EmptySQLElasticPools.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type == 'microsoft.sql/servers/elasticpools'| extend elasticPoolId = tolower(tostring(id)), elasticPoolName = name, elasticPoolRG = resourceGroup,skuName=tostring(sku.name),skuTier=tostring(sku.tier),skuCapacity=tostring(sku.capacity), Location=location, type| join kind=leftouter ( resources | where type == 'microsoft.sql/servers/databases'| extend elasticPoolId = tolower(tostring(properties.elasticPoolId)) ) on elasticPoolId| summarize databaseCount = countif(isnotempty(elasticPoolId1)) by elasticPoolId, elasticPoolName,serverResourceGroup=resourceGroup,name,skuName,skuTier,skuCapacity,elasticPoolRG,Location, type, subscriptionId| where databaseCount == 0 | project elasticPoolId, elasticPoolName, databaseCount, elasticPoolRG ,skuName,skuTier ,skuCapacity, Location, type, subscriptionId| project x_RecommendationId=strcat(tolower(elasticPoolId),'-idle'), x_ResourceGroupName=tolower(elasticPoolRG), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='High', x_RecommendationTypeId='50987aae-a46d-49ae-bd41-a670a4dd18bd', x_RecommendationControl='UsageOptimization/OrphanedResources', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='SQL Database elastic pool has no associated databases', x_RecommendationSolution='Review and remove this resource if not needed', ResourceId = tolower(elasticPoolId), x_ResourceType=type, ResourceName=tolower(elasticPoolName), x_RecommendationDetails= strcat('{\\\"skuName\\\": ', skuName, ',\\\"skuTier\\\": \\\"', skuTier,',\\\"skuCapacity\\\": \\\"', skuCapacity,',\\\"Location\\\": \\\"', Location, '\\\" }'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-EmptySQLElasticPools", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-NonSpotAKSClusters.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-NonSpotAKSClusters.json new file mode 100644 index 000000000..b7a933c62 --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-NonSpotAKSClusters.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type == 'microsoft.containerservice/managedclusters' | mvexpand AgentPoolProfiles = properties.agentPoolProfiles| project id, type, ProfileName = tostring(AgentPoolProfiles.name), Sku = tostring(sku.name), Tier = tostring(sku.tier), mode = AgentPoolProfiles.mode, AutoScaleEnabled = AgentPoolProfiles.enableAutoScaling, SpotVM = AgentPoolProfiles.scaleSetPriority, VMSize = tostring(AgentPoolProfiles.vmSize), NodeCount = tostring(AgentPoolProfiles.['count']), minCount = tostring(AgentPoolProfiles.minCount), maxCount = tostring(AgentPoolProfiles.maxCount), Location=location, resourceGroup, subscriptionId, AKSname = name| where AutoScaleEnabled == 'true' and isnull(SpotVM)| project x_RecommendationId=strcat(tolower(id),'-notSpot'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='Medium', x_RecommendationTypeId='c26abcc2-d5e6-4654-be4a-7d338e5c1e5f', x_RecommendationControl='UsageOptimization/OptimizeResources', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='The AKS cluster agent pool scale set is not utilizing Spot VMs', x_RecommendationSolution='Consider enabling Spot VMs for this AKS cluster to optimize costs, as Spot VMs offer significantly lower pricing compared to regular VMs', ResourceId = tolower(id), x_ResourceType=type, ResourceName=tolower(AKSname), x_RecommendationDetails= strcat('{\\\"AutoScaleEnabled\\\": ', AutoScaleEnabled, ', \\\"SpotVM\\\": \\\"', SpotVM, '\\\", \\\"VMSize\\\": \\\"', VMSize, '\\\", \\\"Location\\\": \\\"', Location, '\\\", \\\"NodeCount\\\": \\\"', NodeCount, '\\\", \\\"minCount\\\": \\\"', minCount, '\\\", \\\"maxCount\\\": \\\"', maxCount, '\\\" }'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-NonSpotAKSClusters", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-SQLVMsWithoutAHB.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-SQLVMsWithoutAHB.json new file mode 100644 index 000000000..f6e32f6eb --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-SQLVMsWithoutAHB.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resourcecontainers | where type =~ 'Microsoft.Resources/subscriptions' | where tostring (properties.subscriptionPolicies.quotaId) !has 'MSDNDevTest_2014-09-01' | extend SubscriptionName=name | join ( resources | where type =~ 'Microsoft.SqlVirtualMachine/SqlVirtualMachines' and tostring(properties.['sqlServerLicenseType']) != 'AHUB' | extend SQLID=id, VMName = name, resourceGroup, Location = location, LicenseType = tostring(properties.['sqlServerLicenseType']), OSType=tostring(properties.storageProfile.imageReference.offer), SQLAgentType = tostring(properties.['sqlManagement']), SQLVersion = tostring(properties.['sqlImageOffer']), SQLSKU=tostring(properties.['sqlImageSku'])) on subscriptionId | join ( resources | where type =~ 'Microsoft.Compute/virtualmachines' | extend ActualCores = toint(extract('.[A-Z]([0-9]+)', 1, tostring(properties.hardwareProfile.vmSize))) | project VMName = tolower(name), VMSize = tostring(properties.hardwareProfile.vmSize),ActualCores, subscriptionId, vmType=type, vmResourceGroup=resourceGroup) on VMName| order by id asc | where SQLSKU != 'Developer' and SQLSKU != 'Express'| extend CheckAHBSQLVM= case( type == 'Microsoft.SqlVirtualMachine/SqlVirtualMachines', iif((properties.['sqlServerLicenseType']) != 'AHUB', 'AHB-disabled', 'AHB-enabled'), 'Not Windows')| project SQLID,VMName,resourceGroup, Location, VMSize, SQLVersion, SQLSKU, SQLAgentType, LicenseType, SubscriptionName,type,CheckAHBSQLVM, subscriptionId,ActualCores, vmType, vmResourceGroup| project x_RecommendationId=strcat(tolower(SQLID),'-CheckAHBSQLVM'), x_ResourceGroupName=tolower(vmResourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='High', x_RecommendationTypeId='01decd62-f91b-4434-abe5-9a09e13e018f', x_RecommendationControl='RateOptimization/Licensing', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='SQL virtual machine is not leveraging Azure Hybrid Benefit', x_RecommendationSolution='Review the SQL licensing option', ResourceId = tolower(SQLID), x_ResourceType=vmType, ResourceName=tolower(VMName), x_RecommendationDetails= strcat('{\\\"VMSize\\\": \\\"', VMSize, '\\\", \\\"CheckAHBSQLVM\\\": \\\"', CheckAHBSQLVM, '\\\", \\\"ActualCores\\\": \\\"', ActualCores, '\\\", \\\"SQLVersion\\\": \\\"', SQLVersion, '\\\", \\\"SQLSKU\\\": \\\"', SQLSKU, '\\\", \\\"SQLAgentType\\\": \\\"', SQLAgentType, '\\\", \\\"LicenseType\\\": \\\"', LicenseType, '\\\", \\\"Location\\\": \\\"', Location, '\\\"}'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-SQLVMsWithoutAHB", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-StoppedVMs.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-StoppedVMs.json new file mode 100644 index 000000000..eaf6ccbf5 --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-StoppedVMs.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.compute/virtualmachines' and tostring(properties.extended.instanceView.powerState.displayStatus) != 'VM deallocated' and tostring(properties.extended.instanceView.powerState.displayStatus) != 'VM running' | extend PowerState=tostring(properties.extended.instanceView.powerState.displayStatus) | extend Location=location, type| project id, PowerState, Location, resourceGroup, subscriptionId, VMName=name, type| project x_RecommendationId=strcat(tolower(id),'-notDeallocated'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='Medium', x_RecommendationTypeId='ab2ff882-e927-4093-9d11-631be0219975', x_RecommendationControl='UsageOptimization/OptimizeResources', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='Virtual machine is powered off but not deallocated', x_RecommendationSolution='Deallocate the virtual machine to ensure it does not incur in compute costs', ResourceId = tolower(id), x_ResourceType=type, ResourceName=tolower(VMName), x_RecommendationDetails= strcat('{\\\"PowerState\\\": ', PowerState, ',\\\"Location\\\": \\\"', Location, '\\\" }'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-StoppedVMs", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-UnattachedDisks.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-UnattachedDisks.json new file mode 100644 index 000000000..c9aaeb84c --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-UnattachedDisks.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.compute/disks' and isempty(managedBy) | extend diskState = tostring(properties.diskState) | where diskState != 'ActiveSAS' and tags !contains 'ASR-ReplicaDisk' and tags !contains 'asrseeddisk' | extend DiskId=id, DiskIDfull=id, DiskName=name, SKUName=sku.name, SKUTier=sku.tier, DiskSizeGB=tostring(properties.diskSizeGB), Location=location, TimeCreated=tostring(properties.timeCreated), SubId=subscriptionId | order by DiskId asc | project DiskId, DiskIDfull, DiskName, DiskSizeGB, SKUName, SKUTier, resourceGroup, Location, TimeCreated, subscriptionId, type| project x_RecommendationId=strcat(tolower(DiskId),'-unattached'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='High', x_RecommendationTypeId='e0c02939-ce02-4f9d-881f-8067ae7ec90f', x_RecommendationControl='UsageOptimization/OrphanedResources', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='Unattached (orphaned) disk is incurring in storage costs', x_RecommendationSolution='Remove or downgrade the unattached disk', ResourceId = tolower(DiskId), x_ResourceType=type, ResourceName=tolower(DiskName), x_RecommendationDetails= strcat('{\\\"DiskSizeGB\\\": ', DiskSizeGB, ', \\\"SKUName\\\": \\\"', SKUName, '\\\", \\\"SKUTier\\\": \\\"', SKUTier, '\\\", \\\"Location\\\": \\\"', Location, '\\\", \\\"TimeCreated\\\": \\\"', TimeCreated, '\\\" }'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-UnattachedDisks", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-UnattachedPublicIPs.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-UnattachedPublicIPs.json new file mode 100644 index 000000000..5043c0c12 --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-UnattachedPublicIPs.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'Microsoft.Network/publicIPAddresses' and isempty(properties.ipConfiguration) and isempty(properties.natGateway) and properties.publicIPAllocationMethod =~ 'Static' | extend PublicIpId=id, IPName=name, AllocationMethod=tostring(properties.publicIPAllocationMethod), SKUName=sku.name, Location=location | project PublicIpId, IPName, SKUName, resourceGroup, Location, AllocationMethod, subscriptionId, type, name | union ( resources | where type =~ 'microsoft.network/networkinterfaces' and isempty(properties.virtualMachine) and isnull(properties.privateEndpoint) and isnotempty(properties.ipConfigurations) | extend IPconfig = properties.ipConfigurations | mv-expand IPconfig | extend PublicIpId= tostring(IPconfig.properties.publicIPAddress.id) | project PublicIpId, name | join ( resources | where type =~ 'Microsoft.Network/publicIPAddresses'| extend PublicIpId=id, IPName=name, AllocationMethod=tostring(properties.publicIPAllocationMethod), SKUName=sku.name, resourceGroup, Location=location, name, id ) on PublicIpId | extend PublicIpId,IPName, SKUName, resourceGroup, Location, AllocationMethod,name, subscriptionId )| project x_RecommendationId=strcat(tolower(PublicIpId),'-idle'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='Low', x_RecommendationTypeId='3ecbf770-9404-4504-a450-cc198e8b2a7d', x_RecommendationControl='UsageOptimization/OrphanedResources', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='Unattached (orphaned) public IP is incurring in networking costs', x_RecommendationSolution='Review and remove this resource if not needed', ResourceId = tolower(PublicIpId), x_ResourceType=type, ResourceName=tolower(name), x_RecommendationDetails= strcat('{\\\"SKUName\\\": ', SKUName, ',\\\"AllocationMethod\\\": \\\"', AllocationMethod,',\\\"Location\\\": \\\"', Location, '\\\" }'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-UnattachedPublicIPs", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/HubsRecommendations-VMsWithoutAHB.json b/src/templates/finops-hub/modules/scripts/HubsRecommendations-VMsWithoutAHB.json new file mode 100644 index 000000000..768912bd6 --- /dev/null +++ b/src/templates/finops-hub/modules/scripts/HubsRecommendations-VMsWithoutAHB.json @@ -0,0 +1,11 @@ +{ + "inputDataset": "resourcegraph", + "outputDataset": "Recommendations", + "provider": "Microsoft", + "query": "resourcecontainers | where type =~ 'Microsoft.Resources/subscriptions' | where tostring (properties.subscriptionPolicies.quotaId) !has 'MSDNDevTest_2014-09-01' | extend SubscriptionName=name | join ( resources | where type =~ 'microsoft.compute/virtualmachines' or type =~ 'microsoft.compute/virtualMachineScaleSets' | where tostring(properties.storageProfile.imageReference.publisher ) == 'MicrosoftWindowsServer' or tostring(properties.virtualMachineProfile.storageProfile.osDisk.osType) == 'Windows' or tostring(properties.storageProfile.imageReference.publisher ) == 'microsoftsqlserver' | where tostring(properties.['licenseType']) !has 'Windows' and tostring(properties.virtualMachineProfile.['licenseType']) == 'Windows_Server' | extend WindowsId=id, VMSku=tostring(properties.hardwareProfile.vmSize), vmResourceGroup=resourceGroup, vmType=type, Location=location,LicenseType = tostring(properties.['licenseType']) | extend ActualCores = toint(extract('.[A-Z]([0-9]+)', 1, tostring(properties.hardwareProfile.vmSize))) | extend CheckAHBWindows = case( type == 'microsoft.compute/virtualmachines' or type =~ 'microsoft.compute/virtualMachineScaleSets', iif((properties.['licenseType']) !has 'Windows' and (properties.virtualMachineProfile.['licenseType']) !has 'Windows' , 'AHB-disabled', 'AHB-enabled'), 'Not Windows' )) on subscriptionId | project x_RecommendationId=strcat(tolower(WindowsId),'-CheckAHBWindows'), x_ResourceGroupName=tolower(vmResourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationImpact='Medium', x_RecommendationTypeId='f326c065-b9f7-4a0e-a0f1-5a1c060bc25d', x_RecommendationControl='RateOptimization/Licensing', x_RecommendationMaturityLevel='Preview', x_RecommendationDescription='Windows virtual machine is not leveraging Azure Hybrid Benefit', x_RecommendationSolution='Review the virtual machine licensing option', ResourceId = tolower(WindowsId), x_ResourceType=vmType, ResourceName=tolower(split(WindowsId,'/')[-1]), x_RecommendationDetails= strcat('{\\\"VMSku\\\": ', VMSku, ',\\\"CheckAHBWindows\\\": \\\"', CheckAHBWindows,',\\\"ActualCores\\\": \\\"', ActualCores,',\\\"Location\\\": \\\"', Location, '\\\" }'), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "schemaFile": "recommendations_1.0.json", + "scope": "azurerbac", + "source": "ResourceGraph", + "type": "HubsRecommendations-VMsWithoutAHB", + "version": "1.0" +} \ No newline at end of file diff --git a/src/templates/finops-hub/modules/scripts/IngestionSetup.kql b/src/templates/finops-hub/modules/scripts/IngestionSetup.kql index 8214d8969..85ccffa07 100644 --- a/src/templates/finops-hub/modules/scripts/IngestionSetup.kql +++ b/src/templates/finops-hub/modules/scripts/IngestionSetup.kql @@ -1371,6 +1371,8 @@ CommitmentDiscountUsage_transform_v1_0() RecommendedQuantity2: decimal, // MS CM MCA resv reco 2024-05-01 -- Renamed to remove spaces RecommendedQuantityNormalized: decimal, // MS CM EA resv reco 2024-05-01 RecommendedQuantityNormalized2: decimal, // MS CM MCA resv reco 2024-05-01 -- Renamed to remove spaces + ResourceId: string, // Hubs Recommendations 1.0 + ResourceName: string, // Hubs Recommendations 1.0 ResourceType: string, // MS CM EA+MCA resv reco 2024-05-01 Scope: string, // MS CM EA resv reco 2024-05-01 scope: string, // MS CM MCA resv reco 2024-05-01 @@ -1378,10 +1380,24 @@ CommitmentDiscountUsage_transform_v1_0() SkuName: string, // MS CM MCA resv reco 2024-05-01 SkuProperties: string, // MS CM EA resv reco 2024-05-01 SkuProperties2: string, // MS CM MCA resv reco 2024-05-01 -- Renamed to remove spaces + SubAccountId: string, // Hubs Recommendations 1.0 + SubAccountName: string, // Hubs Recommendations 1.0 SubscriptionId: string, // MS CM EA+MCA resv reco 2024-05-01 Term: string, // MS CM EA+MCA resv reco 2024-05-01 TotalCostWithReservedInstances: decimal, // MS CM EA resv reco 2024-05-01 TotalCostWithReservedInstances2: decimal, // MS CM MCA resv reco 2024-05-01 -- Renamed to remove spaces + x_RecommendationCategory: string, // Hubs Recommendations 1.0 + x_RecommendationControl: string, // Hubs Recommendations 1.0 + x_RecommendationDate: datetime, // Hubs Recommendations 1.0 + x_RecommendationDescription: string, // Hubs Recommendations 1.0 + x_RecommendationDetails: string, // Hubs Recommendations 1.0 + x_RecommendationId: string, // Hubs Recommendations 1.0 + x_RecommendationImpact: string, // Hubs Recommendations 1.0 + x_RecommendationMaturityLevel: string, // Hubs Recommendations 1.0 + x_RecommendationSolution: string, // Hubs Recommendations 1.0 + x_RecommendationTypeId: string, // Hubs Recommendations 1.0 + x_ResourceGroupName: string, // Hubs Recommendations 1.0 + x_ResourceType: string, // Hubs Recommendations 1.0 x_SourceName: string, // Hubs add-on x_SourceProvider: string, // Hubs add-on x_SourceType: string, // Hubs add-on @@ -1412,6 +1428,8 @@ CommitmentDiscountUsage_transform_v1_0() { "Column": "RecommendedQuantity2", "Properties": { "Field": "Recommended Quantity" } }, { "Column": "RecommendedQuantityNormalized", "Properties": { "Field": "RecommendedQuantityNormalized" } }, { "Column": "RecommendedQuantityNormalized2", "Properties": { "Field": "Recommended Quantity Normalized" } }, + { "Column": "ResourceId", "Properties": { "Field": "ResourceId" } }, + { "Column": "ResourceName", "Properties": { "Field": "ResourceName" } }, { "Column": "ResourceType", "Properties": { "Field": "ResourceType" } }, { "Column": "Scope", "Properties": { "Field": "Scope" } }, { "Column": "scope", "Properties": { "Field": "scope" } }, @@ -1419,10 +1437,24 @@ CommitmentDiscountUsage_transform_v1_0() { "Column": "SkuName", "Properties": { "Field": "SkuName" } }, { "Column": "SkuProperties", "Properties": { "Field": "SkuProperties" } }, { "Column": "SkuProperties2", "Properties": { "Field": "Sku Properties" } }, + { "Column": "SubAccountId", "Properties": { "Field": "SubAccountId" } }, + { "Column": "SubAccountName", "Properties": { "Field": "SubAccountName" } }, { "Column": "SubscriptionId", "Properties": { "Field": "SubscriptionId" } }, { "Column": "Term", "Properties": { "Field": "Term" } }, { "Column": "TotalCostWithReservedInstances", "Properties": { "Field": "TotalCostWithReservedInstances" } }, { "Column": "TotalCostWithReservedInstances2", "Properties": { "Field": "Total Cost With ReservedInstances" } }, + { "Column": "x_RecommendationCategory", "Properties": { "Field": "x_RecommendationCategory" } }, + { "Column": "x_RecommendationControl", "Properties": { "Field": "x_RecommendationControl" } }, + { "Column": "x_RecommendationDate", "Properties": { "Field": "x_RecommendationDate" } }, + { "Column": "x_RecommendationDescription", "Properties": { "Field": "x_RecommendationDescription" } }, + { "Column": "x_RecommendationDetails", "Properties": { "Field": "x_RecommendationDetails" } }, + { "Column": "x_RecommendationId", "Properties": { "Field": "x_RecommendationId" } }, + { "Column": "x_RecommendationImpact", "Properties": { "Field": "x_RecommendationImpact" } }, + { "Column": "x_RecommendationMaturityLevel", "Properties": { "Field": "x_RecommendationMaturityLevel" } }, + { "Column": "x_RecommendationSolution", "Properties": { "Field": "x_RecommendationSolution" } }, + { "Column": "x_RecommendationTypeId", "Properties": { "Field": "x_RecommendationTypeId" } }, + { "Column": "x_ResourceGroupName", "Properties": { "Field": "x_ResourceGroupName" } }, + { "Column": "x_ResourceType", "Properties": { "Field": "x_ResourceType" } }, { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, @@ -1439,6 +1471,74 @@ CommitmentDiscountUsage_transform_v1_0() .create-or-alter function with (docstring='All recommendations transformed to FOCUS 1.0.', folder='Recommendations') Recommendations_transform_v1_0() +{ + // TODO: Move ProviderName upstream + // TODO: Why is SubscriptionId not included? + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', toint(''), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Recommendations_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationRecommendations', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Build recommendation details + | extend x_RecommendationDetails = case( + x_SourceType == 'ReservationRecommendations', bag_pack( + 'CommitmentDiscountNormalizedGroup', InstanceFlexibilityGroup, + 'CommitmentDiscountNormalizedRatio', InstanceFlexibilityRatio, + 'CommitmentDiscountNormalizedSize', NormalizedSize, + 'CommitmentDiscountResourceType', ResourceType, + 'CommitmentDiscountScope', coalesce(Scope, scope), + 'RecommendedQuantity', RecommendedQuantity, + 'RecommendedQuantityNormalized', RecommendedQuantityNormalized, + 'RegionId', Location, + 'SkuMeterId', coalesce(MeterId, MeterID), + 'SkuPriceDetails', SkuProperties, + 'SkuSize', coalesce(SKU, SkuName), + 'SkuTerm', isoMonths(Term) + ), + dynamic({}) + ) + // + // Sort columns and apply final transforms + | project + // TODO: Should we include ChargePeriodEnd? + ChargePeriodStart = FirstUsageDate, + ProviderName, + SubAccountId = iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), ''), + // TODO: SubAccountName, + x_IngestionTime = ingestion_time(), // Hubs add-on + x_LookbackPeriod = case( + LookBackPeriod matches regex @'^Last([0-9]+)Days$', replace_regex(LookBackPeriod, @'^Last([0-9]+)Days$', @'P\1D'), + '' + ), + x_RecommendationCostAfter = TotalCostWithReservedInstances, + x_RecommendationCostBefore = CostWithNoReservedInstances, + x_RecommendationCostSavings = NetSavings, + x_RecommendationDetails, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + +// Recommendations_transform_v1_0r2 function +.create-or-alter function +with (docstring='All recommendations transformed to FOCUS 1.0.', folder='Recommendations') +Recommendations_transform_v1_0r2() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 let isoMonths = (duration: string) { @@ -1496,19 +1596,35 @@ Recommendations_transform_v1_0() 'SkuSize', coalesce(SKU, SkuName), 'SkuTerm', isoMonths(Term) ), + x_SourceType startswith 'HubsRecommendations-', bag_pack( + 'RecommendationCategory', x_RecommendationCategory, + 'RecommendationControl', x_RecommendationControl, + 'RecommendationDetails', x_RecommendationDetails, + 'RecommendationId', x_RecommendationId, + 'RecommendationImpact', x_RecommendationImpact, + 'RecommendationMaturityLevel', x_RecommendationMaturityLevel, + 'RecommendationSolution', x_RecommendationSolution, + 'RecommendationTypeId', x_RecommendationTypeId + ), dynamic({}) ) // // Sort columns and apply final transforms | project ProviderName, - SubAccountId = iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), ''), - x_IngestionTime = ingestion_time(), + ResourceId, + ResourceName, + ResourceType = coalesce(ResourceType, x_ResourceType), + SubAccountId = coalesce(SubAccountId, iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), '')), + SubAccountName, x_EffectiveCostAfter = TotalCostWithReservedInstances, x_EffectiveCostBefore = CostWithNoReservedInstances, x_EffectiveCostSavings = NetSavings, - x_RecommendationDate = FirstUsageDate + (toint(extract(@'^Last([0-9]+)Days$', 1, LookBackPeriod)) * 1d), + x_IngestionTime = ingestion_time(), + x_RecommendationDate = coalesce(x_RecommendationDate, FirstUsageDate + (toint(extract(@'^Last([0-9]+)Days$', 1, LookBackPeriod)) * 1d)), + x_RecommendationDescription, x_RecommendationDetails, + x_ResourceGroupName, x_SourceName, x_SourceProvider, x_SourceType, @@ -1519,18 +1635,41 @@ Recommendations_transform_v1_0() // Recommendations_final_v1_0 table .create-merge table Recommendations_final_v1_0 ( - ProviderName: string, - SubAccountId: string, - x_IngestionTime: datetime, - x_EffectiveCostAfter: decimal, - x_EffectiveCostBefore: decimal, - x_EffectiveCostSavings: decimal, - x_RecommendationDate: datetime, - x_RecommendationDetails: dynamic, - x_SourceName: string, - x_SourceProvider: string, - x_SourceType: string, - x_SourceVersion: string + ChargePeriodStart: datetime, + ProviderName: string, + SubAccountId: string, + x_IngestionTime: datetime, + x_LookbackPeriod: string, + x_RecommendationCostAfter: decimal, + x_RecommendationCostBefore: decimal, + x_RecommendationCostSavings: decimal, + x_RecommendationDetails: dynamic, + x_SourceName: string, + x_SourceProvider: string, + x_SourceType: string, + x_SourceVersion: string +) + +// Recommendations_final_v1_0r2 table +.create-merge table Recommendations_final_v1_0r2 ( + ProviderName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + SubAccountId: string, + SubAccountName: string, + x_EffectiveCostAfter: decimal, + x_EffectiveCostBefore: decimal, + x_EffectiveCostSavings: decimal, + x_IngestionTime: datetime, + x_RecommendationDate: datetime, + x_RecommendationDescription: string, + x_RecommendationDetails: dynamic, + x_ResourceGroupName: string, + x_SourceName: string, + x_SourceProvider: string, + x_SourceType: string, + x_SourceVersion: string ) //---------------------------------------------------------------------------------------------------------------------- @@ -1548,6 +1687,18 @@ Recommendations_transform_v1_0() }] ``` +// Update policy for Recommendations_raw -> Recommendations_final_v1_0r2 table +// NOTE: Must be after transform function is defined +.alter table Recommendations_final_v1_0r2 policy update +``` +[{ + "IsEnabled": true, + "Source": "Recommendations_raw", + "Query": "Recommendations_transform_v1_0r2()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` //===| Transactions |=================================================================================================== // Supported versions: diff --git a/src/templates/finops-hub/modules/storage.bicep b/src/templates/finops-hub/modules/storage.bicep index 3d06c3a40..023299fd2 100644 --- a/src/templates/finops-hub/modules/storage.bicep +++ b/src/templates/finops-hub/modules/storage.bicep @@ -72,12 +72,25 @@ var schemaFiles = { 'focuscost_1.0-preview(v1)': loadTextContent('../schemas/focuscost_1.0-preview(v1).json') 'pricesheet_2023-05-01_ea': loadTextContent('../schemas/pricesheet_2023-05-01_ea.json') 'pricesheet_2023-05-01_mca': loadTextContent('../schemas/pricesheet_2023-05-01_mca.json') + 'recommendations_1.0': loadTextContent('../schemas/recommendations_1.0.json') 'reservationdetails_2023-03-01': loadTextContent('../schemas/reservationdetails_2023-03-01.json') 'reservationrecommendations_2023-05-01_ea': loadTextContent('../schemas/reservationrecommendations_2023-05-01_ea.json') 'reservationrecommendations_2023-05-01_mca': loadTextContent('../schemas/reservationrecommendations_2023-05-01_mca.json') 'reservationtransactions_2023-05-01_ea': loadTextContent('../schemas/reservationtransactions_2023-05-01_ea.json') 'reservationtransactions_2023-05-01_mca': loadTextContent('../schemas/reservationtransactions_2023-05-01_mca.json') } +var queryFiles = { + 'HubsRecommendations-AdvisorCost': loadTextContent('./scripts/HubsRecommendations-AdvisorCost.json') + 'HubsRecommendations-BackendlessAppGateways': loadTextContent('./scripts/HubsRecommendations-BackendlessAppGateways.json') + 'HubsRecommendations-BackendlessLoadBalancers': loadTextContent('./scripts/HubsRecommendations-BackendlessLoadBalancers.json') + 'HubsRecommendations-EmptySQLElasticPools': loadTextContent('./scripts/HubsRecommendations-EmptySQLElasticPools.json') + 'HubsRecommendations-NonSpotAKSClusters': loadTextContent('./scripts/HubsRecommendations-NonSpotAKSClusters.json') + 'HubsRecommendations-SQLVMsWithoutAHB': loadTextContent('./scripts/HubsRecommendations-SQLVMsWithoutAHB.json') + 'HubsRecommendations-StoppedVMs': loadTextContent('./scripts/HubsRecommendations-StoppedVMs.json') + 'HubsRecommendations-UnattachedDisks': loadTextContent('./scripts/HubsRecommendations-UnattachedDisks.json') + 'HubsRecommendations-UnattachedPublicIPs': loadTextContent('./scripts/HubsRecommendations-UnattachedPublicIPs.json') + 'HubsRecommendations-VMsWithoutAHB': loadTextContent('./scripts/HubsRecommendations-VMsWithoutAHB.json') +} // Roles needed to auto-start triggers // Storage Blob Data Contributor - used by deployment scripts to write data to blob storage @@ -463,6 +476,10 @@ resource uploadSettings 'Microsoft.Resources/deploymentScripts@2023-08-01' = { name: 'schemaFiles' value: string(schemaFiles) } + { + name: 'queryFiles' + value: string(queryFiles) + } ] scriptContent: loadTextContent('./scripts/Copy-FileToAzureBlob.ps1') }) diff --git a/src/templates/finops-hub/schemas/recommendations_1.0.json b/src/templates/finops-hub/schemas/recommendations_1.0.json new file mode 100644 index 000000000..7d8c907a8 --- /dev/null +++ b/src/templates/finops-hub/schemas/recommendations_1.0.json @@ -0,0 +1,145 @@ +{ + "additionalColumns": [], + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "path": "[['ResourceId']" + }, + "sink": { + "path": "ResourceId" + } + }, + { + "source": { + "path": "[['ResourceName']" + }, + "sink": { + "path": "ResourceName" + } + }, + { + "source": { + "path": "[['SubAccountId']" + }, + "sink": { + "path": "SubAccountId" + } + }, + { + "source": { + "path": "[['SubAccountName']" + }, + "sink": { + "path": "SubAccountName" + } + }, + { + "source": { + "path": "[['x_RecommendationCategory']" + }, + "sink": { + "path": "x_RecommendationCategory" + } + }, + { + "source": { + "path": "[['x_RecommendationControl']" + }, + "sink": { + "path": "x_RecommendationControl" + } + }, + { + "source": { + "path": "[['x_RecommendationDate']" + }, + "sink": { + "path": "x_RecommendationDate" + } + }, + { + "source": { + "path": "[['x_RecommendationDescription']" + }, + "sink": { + "path": "x_RecommendationDescription" + } + }, + { + "source": { + "path": "[['x_RecommendationDetails']" + }, + "sink": { + "path": "x_RecommendationDetails" + } + }, + { + "source": { + "path": "[['x_RecommendationId']" + }, + "sink": { + "path": "x_RecommendationId" + } + }, + { + "source": { + "path": "[['x_RecommendationImpact']" + }, + "sink": { + "path": "x_RecommendationImpact" + } + }, + { + "source": { + "path": "[['x_RecommendationMaturityLevel']" + }, + "sink": { + "path": "x_RecommendationMaturityLevel" + } + }, + { + "source": { + "path": "[['x_RecommendationProvider']" + }, + "sink": { + "path": "x_RecommendationProvider" + } + }, + { + "source": { + "path": "[['x_RecommendationSolution']" + }, + "sink": { + "path": "x_RecommendationSolution" + } + }, + { + "source": { + "path": "[['x_RecommendationTypeId']" + }, + "sink": { + "path": "x_RecommendationTypeId" + } + }, + { + "source": { + "path": "[['x_ResourceGroupName']" + }, + "sink": { + "path": "x_ResourceGroupName" + } + }, + { + "source": { + "path": "[['x_ResourceType']" + }, + "sink": { + "path": "x_ResourceType" + } + } + ], + "collectionReference": "$['data']" + } +} \ No newline at end of file