diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 2f87fb78d..bb213e4a6 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -36,9 +36,10 @@ The following section lists features and enhancements that are currently in deve - The **hub-app** module tracks telemetry when an app is deployed. - The **hub-storage** module creates containers in the hub storage account. - The **hub-event-trigger** module creates a trigger in the hub Data Factory instance. + - The **hub-database** module runs KQL scripts in the Data Explorer database. - The **hub-vault** module adds secrets to the hub vault. -**Fixed** +- **Fixed** - Workaround subnets reordering and bicep limitation ### [Optimization engine](optimization-engine/overview.md) diff --git a/src/templates/finops-hub/modules/dataExplorer.bicep b/src/templates/finops-hub/modules/dataExplorer.bicep index 67cdcf9c3..c23ccb91b 100644 --- a/src/templates/finops-hub/modules/dataExplorer.bicep +++ b/src/templates/finops-hub/modules/dataExplorer.bicep @@ -129,9 +129,6 @@ param enablePublicAccess bool //------------------------------------------------------------------------------ // cSpell:ignore ftkver, privatelink -var ftkver = any(loadTextContent('ftkver.txt')) // any() is used to suppress a warning the array size (only happens when version does not contain a dash) -var ftkVersion = contains(ftkver, '-') ? split(ftkver, '-')[0] : ftkver -var ftkBranch = contains(ftkver, '-') ? split(ftkver, '-')[1] : '' var dataExplorerPrivateDnsZoneName = replace('privatelink.${location}.${replace(environment().suffixes.storage, 'core', 'kusto')}', '..', '.') // Actual = Minimum(ClusterMaximumConcurrentOperations, Number of nodes in cluster * Maximum(1, Core count per node * CoreUtilizationCoefficient)) @@ -272,90 +269,81 @@ resource cluster 'Microsoft.Kusto/clusters@2023-08-15' = { name: 'Ingestion' location: location kind: 'ReadWrite' - - // Open data functions are split to keep size under the 131KB limit for loadTextContent() - resource OpenDataFunctions_resource_type_1 'scripts' = { name: 'OpenDataFunctions_resource_type_1', properties: { scriptContent: loadTextContent('scripts/OpenDataFunctions_resource_type_1.kql'), continueOnErrors: continueOnErrors, forceUpdateTag: forceUpdateTag }} - resource OpenDataFunctions_resource_type_2 'scripts' = { name: 'OpenDataFunctions_resource_type_2', properties: { scriptContent: loadTextContent('scripts/OpenDataFunctions_resource_type_2.kql'), continueOnErrors: continueOnErrors, forceUpdateTag: forceUpdateTag }} - resource OpenDataFunctions_resource_type_3 'scripts' = { name: 'OpenDataFunctions_resource_type_3', properties: { scriptContent: loadTextContent('scripts/OpenDataFunctions_resource_type_3.kql'), continueOnErrors: continueOnErrors, forceUpdateTag: forceUpdateTag }} - resource OpenDataFunctions_resource_type_4 'scripts' = { name: 'OpenDataFunctions_resource_type_4', properties: { scriptContent: loadTextContent('scripts/OpenDataFunctions_resource_type_4.kql'), continueOnErrors: continueOnErrors, forceUpdateTag: forceUpdateTag }} - - resource openDataScript 'scripts' = { - name: 'OpenDataFunctions' - dependsOn: [ - ingestionDb::OpenDataFunctions_resource_type_1 - ingestionDb::OpenDataFunctions_resource_type_2 - ingestionDb::OpenDataFunctions_resource_type_3 - ingestionDb::OpenDataFunctions_resource_type_4 - ] - properties: { - scriptContent: loadTextContent('scripts/OpenDataFunctions.kql') - continueOnErrors: continueOnErrors - forceUpdateTag: forceUpdateTag - } - } - - resource commonScript 'scripts' = { - name: 'CommonFunctions' - dependsOn: [ - ingestionDb::openDataScript - ] - properties: { - scriptContent: loadTextContent('scripts/Common.kql') - continueOnErrors: continueOnErrors - forceUpdateTag: forceUpdateTag - } - } - - resource setupScript 'scripts' = { - name: 'SetupScript' - dependsOn: [ - ingestionDb::commonScript - ] - properties: { - scriptContent: replace(replace(replace(replace(loadTextContent('scripts/IngestionSetup.kql'), - '$$adfPrincipalId$$', dataFactory.identity.principalId), - '$$adfTenantId$$', dataFactory.identity.tenantId), - '$$ftkOpenDataFolder$$', empty(ftkBranch) ? 'https://github.com/microsoft/finops-toolkit/releases/download/v${ftkVersion}' : 'https://raw.githubusercontent.com/microsoft/finops-toolkit/${ftkBranch}/src/open-data'), - '$$rawRetentionInDays$$', string(rawRetentionInDays)) - continueOnErrors: continueOnErrors - forceUpdateTag: forceUpdateTag - } - } } resource hubDb 'databases' = { name: 'Hub' location: location kind: 'ReadWrite' - dependsOn: [ - ingestionDb::setupScript - ] + } +} - resource commonScript 'scripts' = { - name: 'CommonFunctions' - properties: { - scriptContent: loadTextContent('scripts/Common.kql') - continueOnErrors: continueOnErrors - forceUpdateTag: forceUpdateTag - } +module ingestion_OpenDataInternalScripts 'hub-database.bicep' = { + name: 'ingestion_OpenDataInternalScripts' + params: { + clusterName: cluster.name + databaseName: cluster::ingestionDb.name + scripts: { + OpenDataFunctions_resource_type_1: loadTextContent('scripts/OpenDataFunctions_resource_type_1.kql') + OpenDataFunctions_resource_type_2: loadTextContent('scripts/OpenDataFunctions_resource_type_2.kql') + OpenDataFunctions_resource_type_3: loadTextContent('scripts/OpenDataFunctions_resource_type_3.kql') + OpenDataFunctions_resource_type_4: loadTextContent('scripts/OpenDataFunctions_resource_type_4.kql') } + continueOnErrors: continueOnErrors + forceUpdateTag: forceUpdateTag + } +} - resource setupScript 'scripts' = { - name: 'SetupScript' - dependsOn: [ - hubDb::commonScript - ] - properties: { - scriptContent: replace(replace(loadTextContent('scripts/HubSetup.kql'), - '$$adfPrincipalId$$', dataFactory.identity.principalId), - '$$adfTenantId$$', dataFactory.identity.tenantId) - continueOnErrors: continueOnErrors - forceUpdateTag: forceUpdateTag - } +module ingestion_CommonScripts 'hub-database.bicep' = { + name: 'ingestion_CommonScripts' + dependsOn: [ + ingestion_OpenDataInternalScripts + ] + params: { + clusterName: cluster.name + databaseName: cluster::ingestionDb.name + scripts: { + openDataScript: loadTextContent('scripts/OpenDataFunctions.kql') + commonScript: loadTextContent('scripts/Common.kql') + } + continueOnErrors: continueOnErrors + forceUpdateTag: forceUpdateTag + } +} + +module ingestion_SetupScript 'hub-database.bicep' = { + name: 'ingestion_SetupScript' + dependsOn: [ + ingestion_CommonScripts + ] + params: { + clusterName: cluster.name + databaseName: cluster::ingestionDb.name + scripts: { + setupScript: replace(loadTextContent('scripts/IngestionSetup.kql'), '$$rawRetentionInDays$$', string(rawRetentionInDays)) } + continueOnErrors: continueOnErrors + forceUpdateTag: forceUpdateTag } } +module hub_SetupScript 'hub-database.bicep' = { + name: 'hub_SetupScript' + dependsOn: [ + ingestion_SetupScript + ] + params: { + clusterName: cluster.name + databaseName: cluster::hubDb.name + scripts: { + commonScript: loadTextContent('scripts/Common.kql') + setupScript: replace(loadTextContent('scripts/HubSetup.kql'), '$$rawRetentionInDays$$', string(rawRetentionInDays)) + } + continueOnErrors: continueOnErrors + forceUpdateTag: forceUpdateTag + } +} + // Authorize Kusto Cluster to read storage resource clusterStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { name: guid(cluster.name, subscription().id, 'Storage Blob Data Contributor') diff --git a/src/templates/finops-hub/modules/hub-database.bicep b/src/templates/finops-hub/modules/hub-database.bicep new file mode 100644 index 000000000..085e53b0f --- /dev/null +++ b/src/templates/finops-hub/modules/hub-database.bicep @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//============================================================================== +// Parameters +//============================================================================== + +@description('Required. Name of the FinOps hub Data Explorer instance.') +param clusterName string + +@description('Required. Name of the FinOps hub Data Explorer database to create or update.') +param databaseName string + +@description('Required. List of database scripts to run. The key is the name of the database script and the value is the KQL script content.') +param scripts object + +@description('Optional. If true, ingestion will continue even if some rows fail to ingest. Default: false.') +param continueOnErrors bool = false + +@description('Optional. Forces the table to be updated if different from the last time it was deployed.') +param forceUpdateTag string = utcNow() + + +//============================================================================== +// Resources +//============================================================================== + +resource cluster 'Microsoft.Kusto/clusters@2023-08-15' existing = { + name: clusterName + + resource database 'databases' existing = { + name: databaseName + + resource script 'scripts' = [for scr in items(scripts) : { + name: scr.key + properties: { + scriptContent: scr.value + continueOnErrors: continueOnErrors + forceUpdateTag: forceUpdateTag + } + }] + } +} + + +//============================================================================== +// Outputs +//============================================================================== + +// TODO