From 515f0f04c9d4f5310c398c7e8dcf0bd82d2dbbb7 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 9 Apr 2025 03:01:39 -0700 Subject: [PATCH] Create a hub-event-trigger bicep module as a placeholder for ADF triggers --- docs-mslearn/toolkit/changelog.md | 2 + .../finops-hub/modules/dataFactory.bicep | 121 +++++++----------- .../modules/hub-event-trigger.bicep | 95 ++++++++++++++ 3 files changed, 144 insertions(+), 74 deletions(-) create mode 100644 src/templates/finops-hub/modules/hub-event-trigger.bicep diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index a74212f41..41774e94f 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -32,6 +32,8 @@ The following section lists features and enhancements that are currently in deve - **Added** - Added mslearn docs for Add-FinOpsServicePrincipal powershell command. + - Created a new bicep modules to support extensibility: + - The **hub-event-trigger** module creates a trigger in the hub Data Factory instance.
diff --git a/src/templates/finops-hub/modules/dataFactory.bicep b/src/templates/finops-hub/modules/dataFactory.bicep index f391fff76..320f564ab 100644 --- a/src/templates/finops-hub/modules/dataFactory.bicep +++ b/src/templates/finops-hub/modules/dataFactory.bicep @@ -425,6 +425,7 @@ resource stopTriggers 'Microsoft.Resources/deploymentScripts@2020-10-01' = { //------------------------------------------------------------------------------ // cSpell:ignore linkedservices +// TODO: Move to the hub-app module resource linkedService_keyVault 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = if (!empty(remoteHubStorageUri)) { name: keyVault.name parent: dataFactory @@ -443,6 +444,7 @@ resource linkedService_keyVault 'Microsoft.DataFactory/factories/linkedservices@ } } +// TODO: Move to the hub-app module resource linkedService_storageAccount 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = { name: storageAccount.name parent: dataFactory @@ -812,97 +814,68 @@ resource dataset_ftkReleaseFile 'Microsoft.DataFactory/factories/datasets@2018-0 // Triggers //------------------------------------------------------------------------------ -resource trigger_ExportManifestAdded 'Microsoft.DataFactory/factories/triggers@2018-06-01' = { - name: exportManifestAddedTriggerName - parent: dataFactory +// TODO: Create apps_PublishEvent pipeline { event, properties } + +module trigger_ExportManifestAdded 'hub-event-trigger.bicep' = { + name: 'trigger_ExportManifestAdded' dependsOn: [ stopTriggers ] - properties: { - annotations: [] - pipelines: [ - { - pipelineReference: { - referenceName: pipeline_ExecuteExportsETL.name - type: 'PipelineReference' - } - parameters: { - folderPath: '@triggerBody().folderPath' - fileName: '@triggerBody().fileName' - } - } - ] - type: 'BlobEventsTrigger' - typeProperties: { - blobPathBeginsWith: '/${exportContainerName}/blobs/' - blobPathEndsWith: 'manifest.json' - ignoreEmptyBlobs: true - scope: storageAccount.id - events: [ - 'Microsoft.Storage.BlobCreated' - ] + params: { + dataFactoryName: dataFactory.name + triggerName: exportManifestAddedTriggerName + + // TODO: Replace pipeline with event: 'Microsoft.FinOpsToolkit.CostManagement.ExportManifestAdded' + pipelineName: pipeline_ExecuteExportsETL.name + pipelineParameters: { + folderPath: '@triggerBody().folderPath' + fileName: '@triggerBody().fileName' } + + storageAccountName: storageAccount.name + storageContainer: exportContainerName + storagePathEndsWith: 'manifest.json' } } -resource trigger_IngestionManifestAdded 'Microsoft.DataFactory/factories/triggers@2018-06-01' = if (deployDataExplorer) { - name: ingestionManifestAddedTriggerName - parent: dataFactory +module trigger_IngestionManifestAdded 'hub-event-trigger.bicep' = { + name: 'trigger_IngestionManifestAdded' dependsOn: [ stopTriggers ] - properties: { - annotations: [] - pipelines: [ - { - pipelineReference: { - referenceName: pipeline_ExecuteIngestionETL.name - type: 'PipelineReference' - } - parameters: { - folderPath: '@triggerBody().folderPath' - } - } - ] - type: 'BlobEventsTrigger' - typeProperties: { - blobPathBeginsWith: '/${ingestionContainerName}/blobs/' - blobPathEndsWith: 'manifest.json' - ignoreEmptyBlobs: true - scope: storageAccount.id - events: [ - 'Microsoft.Storage.BlobCreated' - ] + params: { + dataFactoryName: dataFactory.name + triggerName: ingestionManifestAddedTriggerName + + // TODO: Replace pipeline with event: 'Microsoft.FinOpsToolkit.Hubs.IngestionManifestAdded' + pipelineName: pipeline_ExecuteIngestionETL.name + pipelineParameters: { + folderPath: '@triggerBody().folderPath' } + + storageAccountName: storageAccount.name + storageContainer: ingestionContainerName + storagePathEndsWith: 'manifest.json' } } -resource trigger_SettingsUpdated 'Microsoft.DataFactory/factories/triggers@2018-06-01' = { - name: updateConfigTriggerName - parent: dataFactory +module trigger_SettingsUpdated 'hub-event-trigger.bicep' = { + name: 'trigger_SettingsUpdated' dependsOn: [ stopTriggers ] - properties: { - annotations: [] - pipelines: [ - { - pipelineReference: { - referenceName: pipeline_ConfigureExports.name - type: 'PipelineReference' - } - } - ] - type: 'BlobEventsTrigger' - typeProperties: { - blobPathBeginsWith: '/${configContainerName}/blobs/' - blobPathEndsWith: 'settings.json' - ignoreEmptyBlobs: true - scope: storageAccount.id - events: [ - 'Microsoft.Storage.BlobCreated' - ] - } + params: { + dataFactoryName: dataFactory.name + triggerName: updateConfigTriggerName + + // TODO: Replace pipeline with event: 'Microsoft.FinOpsToolkit.Hubs.SettingsUpdated' + pipelineName: pipeline_ConfigureExports.name + pipelineParameters: {} + + storageAccountName: storageAccount.name + storageContainer: configContainerName + // TODO: Change this to startswith + storagePathEndsWith: 'settings.json' } } diff --git a/src/templates/finops-hub/modules/hub-event-trigger.bicep b/src/templates/finops-hub/modules/hub-event-trigger.bicep new file mode 100644 index 000000000..b1d7c8793 --- /dev/null +++ b/src/templates/finops-hub/modules/hub-event-trigger.bicep @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//============================================================================== +// Parameters +//============================================================================== + +@description('Required. Name of the publisher-specific Data Factory instance.') +param dataFactoryName string + +@description('Required. Name of the Data Factory trigger to create or update.') +param triggerName string + +// Storage details +@description('Optional. Azure storage container to monitor for updates and trigger events for.') +param storageAccountName string = '' +@description('Optional. Azure storage container to monitor for updates and trigger events for.') +param storageContainer string = '' +@description('Optional. Beginning of the storage path within the specified storageContainer to monitor for updates and trigger events for.') +param storagePathStartsWith string = '' +@description('Optional. End of the storage path to monitor for updates and trigger events for.') +param storagePathEndsWith string = '' + +// Target pipeline details +@description('Required. Name of the Data Factory pipeline to execute when the trigger is executed.') +param pipelineName string +@description('Required. Parameters to pass to the pipeline when the trigger is executed.') +param pipelineParameters object +// @description('Required. Fully-qualified identifier of the event to publish.') +// param event string + + +//============================================================================== +// Resources +//============================================================================== + +//------------------------------------------------------------------------------ +// Get references to existing resources +//------------------------------------------------------------------------------ + +resource storageAccount 'Microsoft.Storage/storageAccounts@2024-01-01' existing = { + name: storageAccountName +} + +//------------------------------------------------------------------------------ +// Create trigger +//------------------------------------------------------------------------------ + +resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = { + name: dataFactoryName + + resource storageTrigger 'triggers' = if (!empty(storageAccountName)) { + name: triggerName + properties: { + annotations: [] + pipelines: [ + { + // TODO: Replace with apps_PublishEvent pipeline when event publishing is enabled + pipelineReference: { + referenceName: pipelineName + type: 'PipelineReference' + } + parameters: pipelineParameters + + // pipelineReference: { + // referenceName: 'apps_PublishEvent' + // type: 'PipelineReference' + // } + // parameters: { + // event: event + // properties: '@triggerBody()' // pass all trigger properties to the pipeline + // } + } + ] + type: 'BlobEventsTrigger' + typeProperties: { + blobPathBeginsWith: '/${storageContainer}/blobs/${storagePathStartsWith}' + blobPathEndsWith: storagePathEndsWith + ignoreEmptyBlobs: true + scope: storageAccount.id + events: [ + 'Microsoft.Storage.BlobCreated' + ] + } + } + } +} + + +//============================================================================== +// Outputs +//============================================================================== + +// @description('Fully-qualified event that is triggered when the configured event occurs.') +// output event string = event