[Hub apps] Create the hub-database module for KQL script execution

docs-mslearn/toolkit/changelog.md

@@ -36,9 +36,10 @@ The following section lists features and enhancements that are currently in deve
     - The **hub-app** module tracks telemetry when an app is deployed.
     - The **hub-storage** module creates containers in the hub storage account.
     - The **hub-event-trigger** module creates a trigger in the hub Data Factory instance.
+    - The **hub-database** module runs KQL scripts in the Data Explorer database.
     - The **hub-vault** module adds secrets to the hub vault.
 
-**Fixed**
+- **Fixed**
   - Workaround subnets reordering and bicep limitation
 
 ### [Optimization engine](optimization-engine/overview.md)

src/templates/finops-hub/modules/dataExplorer.bicep

@@ -129,9 +129,6 @@ param enablePublicAccess bool
 //------------------------------------------------------------------------------
 
 // cSpell:ignore ftkver, privatelink
-var ftkver = any(loadTextContent('ftkver.txt')) // any() is used to suppress a warning the array size (only happens when version does not contain a dash)
-var ftkVersion = contains(ftkver, '-') ? split(ftkver, '-')[0] : ftkver
-var ftkBranch = contains(ftkver, '-') ? split(ftkver, '-')[1] : ''
 var dataExplorerPrivateDnsZoneName = replace('privatelink.${location}.${replace(environment().suffixes.storage, 'core', 'kusto')}', '..', '.')
 
 // Actual = Minimum(ClusterMaximumConcurrentOperations, Number of nodes in cluster * Maximum(1, Core count per node * CoreUtilizationCoefficient))
@@ -272,90 +269,81 @@ resource cluster 'Microsoft.Kusto/clusters@2023-08-15' = {
     name: 'Ingestion'
     location: location
     kind: 'ReadWrite'
-
-    // Open data functions are split to keep size under the 131KB limit for loadTextContent()
-    resource OpenDataFunctions_resource_type_1 'scripts' = { name: 'OpenDataFunctions_resource_type_1', properties: { scriptContent: loadTextContent('scripts/OpenDataFunctions_resource_type_1.kql'), continueOnErrors: continueOnErrors, forceUpdateTag: forceUpdateTag }}
-    resource OpenDataFunctions_resource_type_2 'scripts' = { name: 'OpenDataFunctions_resource_type_2', properties: { scriptContent: loadTextContent('scripts/OpenDataFunctions_resource_type_2.kql'), continueOnErrors: continueOnErrors, forceUpdateTag: forceUpdateTag }}
-    resource OpenDataFunctions_resource_type_3 'scripts' = { name: 'OpenDataFunctions_resource_type_3', properties: { scriptContent: loadTextContent('scripts/OpenDataFunctions_resource_type_3.kql'), continueOnErrors: continueOnErrors, forceUpdateTag: forceUpdateTag }}
-    resource OpenDataFunctions_resource_type_4 'scripts' = { name: 'OpenDataFunctions_resource_type_4', properties: { scriptContent: loadTextContent('scripts/OpenDataFunctions_resource_type_4.kql'), continueOnErrors: continueOnErrors, forceUpdateTag: forceUpdateTag }}
-
-    resource openDataScript 'scripts' = {
-      name: 'OpenDataFunctions'
-      dependsOn: [
-        ingestionDb::OpenDataFunctions_resource_type_1
-        ingestionDb::OpenDataFunctions_resource_type_2
-        ingestionDb::OpenDataFunctions_resource_type_3
-        ingestionDb::OpenDataFunctions_resource_type_4
-      ]
-      properties: {
-        scriptContent: loadTextContent('scripts/OpenDataFunctions.kql')
-        continueOnErrors: continueOnErrors
-        forceUpdateTag: forceUpdateTag
-      }
-    }
-
-    resource commonScript 'scripts' = {
-      name: 'CommonFunctions'
-      dependsOn: [
-        ingestionDb::openDataScript
-      ]
-      properties: {
-        scriptContent: loadTextContent('scripts/Common.kql')
-        continueOnErrors: continueOnErrors
-        forceUpdateTag: forceUpdateTag
-      }
-    }
-
-    resource setupScript 'scripts' = {
-      name: 'SetupScript'
-      dependsOn: [
-        ingestionDb::commonScript
-      ]
-      properties: {
-        scriptContent: replace(replace(replace(replace(loadTextContent('scripts/IngestionSetup.kql'),
-          '$$adfPrincipalId$$', dataFactory.identity.principalId),
-          '$$adfTenantId$$', dataFactory.identity.tenantId),
-          '$$ftkOpenDataFolder$$', empty(ftkBranch) ? 'https://github.com/microsoft/finops-toolkit/releases/download/v${ftkVersion}' : 'https://raw.githubusercontent.com/microsoft/finops-toolkit/${ftkBranch}/src/open-data'),
-          '$$rawRetentionInDays$$', string(rawRetentionInDays))
-        continueOnErrors: continueOnErrors
-        forceUpdateTag: forceUpdateTag
-      }
-    }
   }
 
   resource hubDb 'databases' = {
     name: 'Hub'
     location: location
     kind: 'ReadWrite'
-    dependsOn: [
-      ingestionDb::setupScript
-    ]
+  }
+}
 
-    resource commonScript 'scripts' = {
-      name: 'CommonFunctions'
-      properties: {
-        scriptContent: loadTextContent('scripts/Common.kql')
-        continueOnErrors: continueOnErrors
-        forceUpdateTag: forceUpdateTag
-      }
+module ingestion_OpenDataInternalScripts 'hub-database.bicep' = {
+  name: 'ingestion_OpenDataInternalScripts'
+  params: {
+    clusterName: cluster.name
+    databaseName: cluster::ingestionDb.name
+    scripts: {
+      OpenDataFunctions_resource_type_1: loadTextContent('scripts/OpenDataFunctions_resource_type_1.kql')
+      OpenDataFunctions_resource_type_2: loadTextContent('scripts/OpenDataFunctions_resource_type_2.kql')
+      OpenDataFunctions_resource_type_3: loadTextContent('scripts/OpenDataFunctions_resource_type_3.kql')
+      OpenDataFunctions_resource_type_4: loadTextContent('scripts/OpenDataFunctions_resource_type_4.kql')
     }
+    continueOnErrors: continueOnErrors
+    forceUpdateTag: forceUpdateTag
+  }
+}
 
-    resource setupScript 'scripts' = {
-      name: 'SetupScript'
-      dependsOn: [
-        hubDb::commonScript
-      ]
-      properties: {
-        scriptContent: replace(replace(loadTextContent('scripts/HubSetup.kql'),
-          '$$adfPrincipalId$$', dataFactory.identity.principalId),
-          '$$adfTenantId$$', dataFactory.identity.tenantId)
-        continueOnErrors: continueOnErrors
-        forceUpdateTag: forceUpdateTag
-      }
+module ingestion_CommonScripts 'hub-database.bicep' = {
+  name: 'ingestion_CommonScripts'
+  dependsOn: [
+    ingestion_OpenDataInternalScripts
+  ]
+  params: {
+    clusterName: cluster.name
+    databaseName: cluster::ingestionDb.name
+    scripts: {
+      openDataScript: loadTextContent('scripts/OpenDataFunctions.kql')
+      commonScript: loadTextContent('scripts/Common.kql')
+    }
+    continueOnErrors: continueOnErrors
+    forceUpdateTag: forceUpdateTag
+  }
+}
+
+module ingestion_SetupScript 'hub-database.bicep' = {
+  name: 'ingestion_SetupScript'
+  dependsOn: [
+    ingestion_CommonScripts
+  ]
+  params: {
+    clusterName: cluster.name
+    databaseName: cluster::ingestionDb.name
+    scripts: {
+      setupScript: replace(loadTextContent('scripts/IngestionSetup.kql'), '$$rawRetentionInDays$$', string(rawRetentionInDays))
     }
+    continueOnErrors: continueOnErrors
+    forceUpdateTag: forceUpdateTag
   }
 }
 
+module hub_SetupScript 'hub-database.bicep' = {
+  name: 'hub_SetupScript'
+  dependsOn: [
+    ingestion_SetupScript
+  ]
+  params: {
+    clusterName: cluster.name
+    databaseName: cluster::hubDb.name
+    scripts: {
+      commonScript: loadTextContent('scripts/Common.kql')
+      setupScript: replace(loadTextContent('scripts/HubSetup.kql'), '$$rawRetentionInDays$$', string(rawRetentionInDays))
+    }
+    continueOnErrors: continueOnErrors
+    forceUpdateTag: forceUpdateTag
+  }
+}
+
 // Authorize Kusto Cluster to read storage
 resource clusterStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
   name: guid(cluster.name, subscription().id, 'Storage Blob Data Contributor')

src/templates/finops-hub/modules/hub-database.bicep

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//==============================================================================
+// Parameters
+//==============================================================================
+
+@description('Required. Name of the FinOps hub Data Explorer instance.')
+param clusterName string
+
+@description('Required. Name of the FinOps hub Data Explorer database to create or update.')
+param databaseName string
+
+@description('Required. List of database scripts to run. The key is the name of the database script and the value is the KQL script content.')
+param scripts object
+
+@description('Optional. If true, ingestion will continue even if some rows fail to ingest. Default: false.')
+param continueOnErrors bool = false
+
+@description('Optional. Forces the table to be updated if different from the last time it was deployed.')
+param forceUpdateTag string = utcNow()
+
+
+//==============================================================================
+// Resources
+//==============================================================================
+
+resource cluster 'Microsoft.Kusto/clusters@2023-08-15' existing = {
+  name: clusterName
+
+  resource database 'databases' existing = {
+    name: databaseName
+
+    resource script 'scripts' = [for scr in items(scripts) : {
+      name: scr.key
+      properties: {
+        scriptContent: scr.value
+        continueOnErrors: continueOnErrors
+        forceUpdateTag: forceUpdateTag
+      }
+    }]
+  }
+}
+
+
+//==============================================================================
+// Outputs
+//==============================================================================
+
+// TODO