From b7f2777255ea6d6a1c15228f5728633e15e1086b Mon Sep 17 00:00:00 2001 From: Uday Kumar Pasumarthy Date: Wed, 9 Jul 2025 21:09:43 -0700 Subject: [PATCH 1/2] Adding Fabric template for sharepoint capacity scenario --- .../Unlock SharePoint Capacity Insights.json | 1789 +++++++++++++++++ .../manifest.json | 30 + 2 files changed, 1819 insertions(+) create mode 100644 templates/Unlock SharePoint Capacity Insights/Unlock SharePoint Capacity Insights.json create mode 100644 templates/Unlock SharePoint Capacity Insights/manifest.json diff --git a/templates/Unlock SharePoint Capacity Insights/Unlock SharePoint Capacity Insights.json b/templates/Unlock SharePoint Capacity Insights/Unlock SharePoint Capacity Insights.json new file mode 100644 index 00000000..bf40be21 --- /dev/null +++ b/templates/Unlock SharePoint Capacity Insights/Unlock SharePoint Capacity Insights.json @@ -0,0 +1,1789 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "LS_Lakehouse-SQLEndPoint": { + "type": "string" + }, + "LS_Office365_MGDC": { + "type": "string" + } + }, + "resources": [ + { + "apiVersion": "2018-06-01", + "dependsOn": [], + "name": "Unlock SharePoint Capacity Insights", + "properties": { + "activities": [ + { + "dependsOn": [ + { + "activity": "Extract Sites From MGDC", + "dependencyConditions": [ + "Completed" + ] + }, + { + "activity": "Extract Files From MGDC", + "dependencyConditions": [ + "Completed" + ] + } + ], + "name": "Merge Sites and Files To Final Table", + "policy": { + "retry": 0, + "retryIntervalInSeconds": 30, + "secureInput": false, + "secureOutput": false, + "timeout": "0.12:00:00" + }, + "type": "TridentNotebook", + "typeProperties": { + "notebookId": "9e1e4bfb-8b70-4a68-9212-97fa7a183a8e", + "parameters": { + "filesFinalTableName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesFinalTableName" + } + }, + "filesStagingTableName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesStagingTableName" + } + }, + "lakehouseName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@pipeline().parameters.LakehouseName" + } + }, + "runId": { + "type": "string", + "value": { + "type": "Expression", + "value": "@pipeline().RunId" + } + }, + "sitesFinalTableName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesFinalTableName" + } + }, + "sitesStagingTableName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesStagingTableName" + } + }, + "workspaceId": { + "type": "string", + "value": { + "type": "Expression", + "value": "@pipeline().DataFactory" + } + } + }, + "workspaceId": { + "type": "Expression", + "value": "@pipeline().DataFactory" + } + } + }, + { + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "description": "", + "name": "Set End Snapshot Date", + "policy": { + "secureInput": false, + "secureOutput": false + }, + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).EndSnapshotDate\n\n" + }, + "variableName": "EndTime" + } + }, + { + "dependsOn": [], + "name": "Read Last Snapshot Dates", + "policy": { + "retry": 0, + "retryIntervalInSeconds": 30, + "secureInput": false, + "secureOutput": false, + "timeout": "0.12:00:00" + }, + "type": "TridentNotebook", + "typeProperties": { + "notebookId": "3b37b5cd-619c-43d0-a68c-2027a8ccd32b", + "parameters": { + "endTime": { + "type": "string", + "value": { + "type": "Expression", + "value": "@coalesce(pipeline().parameters.EndTime,formatDateTime(adddays(startOfDay(pipeline().TriggerTime),-2),'yyyy-MM-ddTHH:mm:ssZ'))\n" + } + }, + "filesFinalTableName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@pipeline().parameters.FilesTableName" + } + }, + "filessStagingTableName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@concat(pipeline().parameters.FilesTableName,pipeline().parameters.StagingSuffix)" + } + }, + "lakehouseName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@pipeline().parameters.LakehouseName" + } + }, + "runId": { + "type": "string", + "value": { + "type": "Expression", + "value": "@pipeline().RunId" + } + }, + "sitesFinalTableName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@pipeline().parameters.SitesTableName" + } + }, + "sitesStagingTableName": { + "type": "string", + "value": { + "type": "Expression", + "value": "@concat(pipeline().parameters.SitesTableName,pipeline().parameters.StagingSuffix)" + } + }, + "workspaceId": { + "type": "string", + "value": { + "type": "Expression", + "value": "@pipeline().DataFactory" + } + } + }, + "workspaceId": { + "type": "Expression", + "value": "@pipeline().DataFactory" + } + } + }, + { + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "Set Sites Snapshot Date", + "policy": { + "secureInput": false, + "secureOutput": false + }, + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@coalesce(pipeline().parameters.StartTime, json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesSnapshotDate)" + }, + "variableName": "StartTimeForSites" + } + }, + { + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "Set Files Snapshot Date", + "policy": { + "secureInput": false, + "secureOutput": false + }, + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@coalesce(pipeline().parameters.StartTime, json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesSnapshotDate)" + }, + "variableName": "StartTimeForFiles" + } + }, + { + "dependsOn": [ + { + "activity": "Wait 1 Min", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "externalReferences": { + "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" + }, + "name": "Sites View - Lakehouse SQLEndpoint", + "policy": { + "retry": 2, + "retryIntervalInSeconds": 300, + "secureInput": false, + "secureOutput": false, + "timeout": "0.12:00:00" + }, + "type": "Script", + "typeProperties": { + "database": { + "type": "Expression", + "value": "@pipeline().parameters.LakehouseName" + }, + "scriptBlockExecutionTimeout": "02:00:00", + "scripts": [ + { + "text": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesView" + }, + "type": "NonQuery" + } + ] + } + }, + { + "dependsOn": [ + { + "activity": "Wait 1 Min", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "externalReferences": { + "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" + }, + "name": "Files View - Lakehouse SQLEndpoint", + "policy": { + "retry": 2, + "retryIntervalInSeconds": 300, + "secureInput": false, + "secureOutput": false, + "timeout": "0.12:00:00" + }, + "type": "Script", + "typeProperties": { + "database": { + "type": "Expression", + "value": "@pipeline().parameters.LakehouseName" + }, + "scriptBlockExecutionTimeout": "02:00:00", + "scripts": [ + { + "text": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesView" + }, + "type": "NonQuery" + } + ] + } + }, + { + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "Does Files Data Exists", + "policy": { + "secureInput": false, + "secureOutput": false + }, + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@bool(json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesDataExists)" + }, + "variableName": "FilesDataExists" + } + }, + { + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "Does Sites Data Exists", + "policy": { + "secureInput": false, + "secureOutput": false + }, + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@bool(json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesDataExists)" + }, + "variableName": "SitesDataExists" + } + }, + { + "dependsOn": [ + { + "activity": "Set Sites Snapshot Date", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set End Snapshot Date", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Does Sites Data Exists", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "description": "", + "name": "Extract Sites From MGDC", + "type": "IfCondition", + "typeProperties": { + "expression": { + "type": "Expression", + "value": "@and(and(variables('SitesDataExists'),\n not(pipeline().parameters.ForceFullWhenDataExists)\t \n\t ), greaterOrEquals( startOfDay(variables('StartTimeForSites')) , startOfDay(variables('EndTime')) ))\n\n " + }, + "ifFalseActivities": [ + { + "dependsOn": [], + "name": "Extract Sites To Staging", + "policy": { + "retry": 0, + "retryIntervalInSeconds": 30, + "secureInput": false, + "secureOutput": false, + "timeout": "0.12:00:00" + }, + "type": "Copy", + "typeProperties": { + "enableStaging": true, + "sink": { + "datasetSettings": { + "annotations": [], + "linkedService": { + "name": "e629b492_d56f_43cc_a93f_3558c83b2f5d", + "properties": { + "annotations": [], + "type": "Lakehouse", + "typeProperties": { + "artifactId": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).LakehouseId", + "rootFolder": "Tables", + "workspaceId": "@pipeline().DataFactory" + } + } + }, + "schema": [], + "type": "LakehouseTable", + "typeProperties": { + "schema": "dbo", + "table": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesStagingTableName" + } + } + }, + "partitionOption": "None", + "tableActionOption": "OverwriteSchema", + "type": "LakehouseTableSink" + }, + "source": { + "DataFilter": { + "type": "Expression", + "value": "@pipeline().parameters.SitesFilter" + }, + "datasetSettings": { + "annotations": [], + "externalReferences": { + "connection": "[parameters('LS_Office365_MGDC')]" + }, + "schema": [], + "type": "Office365Table", + "typeProperties": { + "tableName": "BasicDataSet_v0.SharePointSites_v1" + } + }, + "dateFilterColumn": "SnapshotDate", + "endTime": { + "type": "Expression", + "value": "@variables('EndTime')" + }, + "startTime": { + "type": "Expression", + "value": "@variables('StartTimeForSites')" + }, + "type": "Office365Source" + }, + "translator": { + "columnFlattenSettings": { + "flattenColumnDelimiter": "_", + "treatArrayAsString": false, + "treatStructAsString": false + }, + "mappings": [ + { + "sink": { + "name": "ptenant", + "physicalType": "string" + }, + "source": { + "name": "ptenant", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Id", + "physicalType": "string" + }, + "source": { + "name": "Id", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Url", + "physicalType": "string" + }, + "source": { + "name": "Url", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "ArchiveState", + "physicalType": "string" + }, + "source": { + "name": "ArchiveState", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "RootWeb_Configuration", + "physicalType": "long" + }, + "source": { + "name": "RootWeb_Configuration", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "RootWeb_Id", + "physicalType": "string" + }, + "source": { + "name": "RootWeb_Id", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "RootWeb_Title", + "physicalType": "string" + }, + "source": { + "name": "RootWeb_Title", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "RootWeb_WebTemplate", + "physicalType": "string" + }, + "source": { + "name": "RootWeb_WebTemplate", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "RootWeb_WebTemplateId", + "physicalType": "string" + }, + "source": { + "name": "RootWeb_WebTemplateId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "RootWeb_LastItemModifiedDate", + "physicalType": "timestamp" + }, + "source": { + "name": "RootWeb_LastItemModifiedDate", + "physicalType": "datetime", + "type": "DateTime" + } + }, + { + "sink": { + "name": "WebCount", + "physicalType": "long" + }, + "source": { + "name": "WebCount", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "StorageQuota", + "physicalType": "long" + }, + "source": { + "name": "StorageQuota", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "StorageUsed", + "physicalType": "long" + }, + "source": { + "name": "StorageUsed", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "StorageMetrics_MetadataSize", + "physicalType": "long" + }, + "source": { + "name": "StorageMetrics_MetadataSize", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "StorageMetrics_TotalFileCount", + "physicalType": "long" + }, + "source": { + "name": "StorageMetrics_TotalFileCount", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "StorageMetrics_TotalFileStreamSize", + "physicalType": "long" + }, + "source": { + "name": "StorageMetrics_TotalFileStreamSize", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "StorageMetrics_TotalSize", + "physicalType": "long" + }, + "source": { + "name": "StorageMetrics_TotalSize", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "GroupId", + "physicalType": "string" + }, + "source": { + "name": "GroupId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "GeoLocation", + "physicalType": "string" + }, + "source": { + "name": "GeoLocation", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "IsInRecycleBin", + "physicalType": "boolean" + }, + "source": { + "name": "IsInRecycleBin", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "RecycleBinItemCount", + "physicalType": "long" + }, + "source": { + "name": "RecycleBinItemCount", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "RecycleBinItemSize", + "physicalType": "long" + }, + "source": { + "name": "RecycleBinItemSize", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "SecondStageRecycleBinStorageUsage", + "physicalType": "long" + }, + "source": { + "name": "SecondStageRecycleBinStorageUsage", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "IsTeamsConnectedSite", + "physicalType": "boolean" + }, + "source": { + "name": "IsTeamsConnectedSite", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "IsTeamsChannelSite", + "physicalType": "boolean" + }, + "source": { + "name": "IsTeamsChannelSite", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "TeamsChannelType", + "physicalType": "string" + }, + "source": { + "name": "TeamsChannelType", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "IsCommunicationSite", + "physicalType": "boolean" + }, + "source": { + "name": "IsCommunicationSite", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "IsHubSite", + "physicalType": "boolean" + }, + "source": { + "name": "IsHubSite", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "HubSiteId", + "physicalType": "string" + }, + "source": { + "name": "HubSiteId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "IsOneDrive", + "physicalType": "boolean" + }, + "source": { + "name": "IsOneDrive", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "BlockAccessFromUnmanagedDevices", + "physicalType": "boolean" + }, + "source": { + "name": "BlockAccessFromUnmanagedDevices", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "BlockDownloadOfAllFilesOnUnmanagedDevices", + "physicalType": "boolean" + }, + "source": { + "name": "BlockDownloadOfAllFilesOnUnmanagedDevices", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "BlockDownloadOfViewableFilesOnUnmanagedDevices", + "physicalType": "boolean" + }, + "source": { + "name": "BlockDownloadOfViewableFilesOnUnmanagedDevices", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "IsExternalSharingEnabled", + "physicalType": "boolean" + }, + "source": { + "name": "IsExternalSharingEnabled", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "Privacy", + "physicalType": "string" + }, + "source": { + "name": "Privacy", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "ShareByEmailEnabled", + "physicalType": "boolean" + }, + "source": { + "name": "ShareByEmailEnabled", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "ShareByLinkEnabled", + "physicalType": "boolean" + }, + "source": { + "name": "ShareByLinkEnabled", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "SiteConnectedToPrivateGroup", + "physicalType": "boolean" + }, + "source": { + "name": "SiteConnectedToPrivateGroup", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "SensitivityLabelInfo_DisplayName", + "physicalType": "string" + }, + "source": { + "name": "SensitivityLabelInfo_DisplayName", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SensitivityLabelInfo_Id", + "physicalType": "string" + }, + "source": { + "name": "SensitivityLabelInfo_Id", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Classification", + "physicalType": "string" + }, + "source": { + "name": "Classification", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "IBMode", + "physicalType": "string" + }, + "source": { + "name": "IBMode", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "IBSegments", + "physicalType": "string" + }, + "source": { + "name": "IBSegments", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "RelatedGroupId", + "physicalType": "string" + }, + "source": { + "name": "RelatedGroupId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Owner_AadObjectId", + "physicalType": "string" + }, + "source": { + "name": "Owner_AadObjectId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Owner_Email", + "physicalType": "string" + }, + "source": { + "name": "Owner_Email", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Owner_Name", + "physicalType": "string" + }, + "source": { + "name": "Owner_Name", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Owner_UPN", + "physicalType": "string" + }, + "source": { + "name": "Owner_UPN", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SecondaryContact_AadObjectId", + "physicalType": "string" + }, + "source": { + "name": "SecondaryContact_AadObjectId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SecondaryContact_Email", + "physicalType": "string" + }, + "source": { + "name": "SecondaryContact_Email", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SecondaryContact_Name", + "physicalType": "string" + }, + "source": { + "name": "SecondaryContact_Name", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SecondaryContact_UPN", + "physicalType": "string" + }, + "source": { + "name": "SecondaryContact_UPN", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "ReadLocked", + "physicalType": "boolean" + }, + "source": { + "name": "ReadLocked", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "ReadOnly", + "physicalType": "boolean" + }, + "source": { + "name": "ReadOnly", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "CreatedTime", + "physicalType": "timestamp" + }, + "source": { + "name": "CreatedTime", + "physicalType": "datetime", + "type": "DateTime" + } + }, + { + "sink": { + "name": "LastSecurityModifiedDate", + "physicalType": "timestamp" + }, + "source": { + "name": "LastSecurityModifiedDate", + "physicalType": "datetime", + "type": "DateTime" + } + }, + { + "sink": { + "name": "Operation", + "physicalType": "string" + }, + "source": { + "name": "Operation", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "LastUserAccessDate", + "physicalType": "timestamp" + }, + "source": { + "name": "LastUserAccessDate", + "physicalType": "datetime", + "type": "DateTime" + } + }, + { + "sink": { + "name": "SnapshotDate", + "physicalType": "timestamp" + }, + "source": { + "name": "SnapshotDate", + "physicalType": "datetime", + "type": "DateTime" + } + } + ], + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + } + } + ], + "ifTrueActivities": [ + { + "dependsOn": [], + "description": "Do Not Extract Sites - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", + "name": "Do Not Extract Sites", + "type": "Fail", + "typeProperties": { + "errorCode": "-999", + "message": "Error: -999 - Do Not Extract Sites - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false" + } + } + ] + } + }, + { + "dependsOn": [ + { + "activity": "Set Files Snapshot Date", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set End Snapshot Date", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Does Files Data Exists", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "description": "", + "name": "Extract Files From MGDC", + "type": "IfCondition", + "typeProperties": { + "expression": { + "type": "Expression", + "value": "@and(and(variables('FilesDataExists'),\n not(pipeline().parameters.ForceFullWhenDataExists)\t \n\t ), greaterOrEquals( startOfDay(variables('StartTimeForFiles')) , startOfDay(variables('EndTime')) ))" + }, + "ifFalseActivities": [ + { + "dependsOn": [], + "name": "Extract Files To Staging", + "policy": { + "retry": 0, + "retryIntervalInSeconds": 30, + "secureInput": false, + "secureOutput": false, + "timeout": "0.12:00:00" + }, + "type": "Copy", + "typeProperties": { + "enableStaging": true, + "sink": { + "datasetSettings": { + "annotations": [], + "linkedService": { + "name": "f589a5e3_9e9f_4bbe_8e32_55faadf2c82f", + "properties": { + "annotations": [], + "type": "Lakehouse", + "typeProperties": { + "artifactId": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).LakehouseId", + "rootFolder": "Tables", + "workspaceId": "@pipeline().DataFactory" + } + } + }, + "schema": [], + "type": "LakehouseTable", + "typeProperties": { + "schema": "dbo", + "table": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesStagingTableName" + } + } + }, + "partitionOption": "None", + "tableActionOption": "OverwriteSchema", + "type": "LakehouseTableSink" + }, + "source": { + "datasetSettings": { + "annotations": [], + "DataFilter": { + "type": "Expression", + "value": "@pipeline().parameters.FilesFilter" + }, + "externalReferences": { + "connection": "[parameters('LS_Office365_MGDC')]" + }, + "schema": [], + "type": "Office365Table", + "typeProperties": { + "tableName": "BasicDataSet_v0.SharePointFiles_v1" + } + }, + "dateFilterColumn": "SnapshotDate", + "endTime": { + "type": "Expression", + "value": "@variables('EndTime')" + }, + "startTime": { + "type": "Expression", + "value": "@variables('StartTimeForFiles')" + }, + "type": "Office365Source" + }, + "translator": { + "columnFlattenSettings": { + "flattenColumnDelimiter": "_", + "treatArrayAsString": false, + "treatStructAsString": false + }, + "mappings": [ + { + "sink": { + "name": "ptenant", + "physicalType": "string" + }, + "source": { + "name": "ptenant", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SiteId", + "physicalType": "string" + }, + "source": { + "name": "SiteId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Author_Email", + "physicalType": "string" + }, + "source": { + "name": "Author_Email", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Author_Name", + "physicalType": "string" + }, + "source": { + "name": "Author_Name", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "DirName", + "physicalType": "string" + }, + "source": { + "name": "DirName", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Extension", + "physicalType": "string" + }, + "source": { + "name": "Extension", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "FileName", + "physicalType": "string" + }, + "source": { + "name": "FileName", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "IsLabelEncrypted", + "physicalType": "boolean" + }, + "source": { + "name": "IsLabelEncrypted", + "physicalType": "boolean", + "type": "Boolean" + } + }, + { + "sink": { + "name": "ItemId", + "physicalType": "string" + }, + "source": { + "name": "ItemId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "ListId", + "physicalType": "string" + }, + "source": { + "name": "ListId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "ListServerTemplate", + "physicalType": "string" + }, + "source": { + "name": "ListServerTemplate", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "MajorVersion", + "physicalType": "integer" + }, + "source": { + "name": "MajorVersion", + "physicalType": "int32", + "type": "Int32" + } + }, + { + "sink": { + "name": "MinorVersion", + "physicalType": "integer" + }, + "source": { + "name": "MinorVersion", + "physicalType": "int32", + "type": "Int32" + } + }, + { + "sink": { + "name": "ModifiedBy_Email", + "physicalType": "string" + }, + "source": { + "name": "ModifiedBy_Email", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "ModifiedBy_Name", + "physicalType": "string" + }, + "source": { + "name": "ModifiedBy_Name", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "Operation", + "physicalType": "string" + }, + "source": { + "name": "Operation", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "QuickXorHash", + "physicalType": "string" + }, + "source": { + "name": "QuickXorHash", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "ScopeId", + "physicalType": "string" + }, + "source": { + "name": "ScopeId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SensitivityLabelInfo_DisplayName", + "physicalType": "string" + }, + "source": { + "name": "SensitivityLabelInfo_DisplayName", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SensitivityLabelInfo_Id", + "physicalType": "string" + }, + "source": { + "name": "SensitivityLabelInfo_Id", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SiteUrl", + "physicalType": "string" + }, + "source": { + "name": "SiteUrl", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "SizeInBytes", + "physicalType": "long" + }, + "source": { + "name": "SizeInBytes", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "SizeInBytesWithVersions", + "physicalType": "long" + }, + "source": { + "name": "SizeInBytesWithVersions", + "physicalType": "int64", + "type": "Int64" + } + }, + { + "sink": { + "name": "TimeCreated", + "physicalType": "timestamp" + }, + "source": { + "name": "TimeCreated", + "physicalType": "datetime", + "type": "DateTime" + } + }, + { + "sink": { + "name": "TimeLastModified", + "physicalType": "timestamp" + }, + "source": { + "name": "TimeLastModified", + "physicalType": "datetime", + "type": "DateTime" + } + }, + { + "sink": { + "name": "WebId", + "physicalType": "string" + }, + "source": { + "name": "WebId", + "physicalType": "string", + "type": "String" + } + }, + { + "sink": { + "name": "WebTemplateId", + "physicalType": "integer" + }, + "source": { + "name": "WebTemplateId", + "physicalType": "int32", + "type": "Int32" + } + }, + { + "sink": { + "name": "SnapshotDate", + "physicalType": "timestamp" + }, + "source": { + "name": "SnapshotDate", + "physicalType": "datetime", + "type": "DateTime" + } + } + ], + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + } + } + ], + "ifTrueActivities": [ + { + "dependsOn": [], + "description": "Do Not Extract Files - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", + "name": "Do Not Extract Files", + "type": "Fail", + "typeProperties": { + "errorCode": "-999", + "message": "Error: -999 - Do Not Extract Files - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false" + } + } + ] + } + }, + { + "dependsOn": [ + { + "activity": "Extract Files From MGDC", + "dependencyConditions": [ + "Failed" + ] + } + ], + "name": "Files Validations Failed", + "policy": { + "secureInput": false, + "secureOutput": false + }, + "type": "SetVariable", + "typeProperties": { + "value": "true", + "variableName": "FilesExtractValidationsFailed" + } + }, + { + "dependsOn": [ + { + "activity": "Extract Sites From MGDC", + "dependencyConditions": [ + "Failed" + ] + } + ], + "description": "true", + "name": "Site Validations Failed", + "policy": { + "secureInput": false, + "secureOutput": false + }, + "type": "SetVariable", + "typeProperties": { + "value": "true", + "variableName": "FilesExtractValidationsFailed" + } + }, + { + "dependsOn": [ + { + "activity": "Sites View - Lakehouse SQLEndpoint", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Files View - Lakehouse SQLEndpoint", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Files Aggs View - Lakehouse SQLEndpoint", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "description": "If this activity completed then Pipeline Completed Successfully", + "name": "Pipeline Success", + "policy": { + "secureInput": false, + "secureOutput": false + }, + "type": "SetVariable", + "typeProperties": { + "value": "Pipeline Completed Successfully", + "variableName": "PipelineFinalStatus" + } + }, + { + "dependsOn": [ + { + "activity": "Merge Sites and Files To Final Table", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "Wait 1 Min", + "type": "Wait", + "typeProperties": { + "waitTimeInSeconds": 60 + } + }, + { + "dependsOn": [ + { + "activity": "Wait 1 Min", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "externalReferences": { + "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" + }, + "name": "Files Aggs View - Lakehouse SQLEndpoint", + "policy": { + "retry": 2, + "retryIntervalInSeconds": 300, + "secureInput": false, + "secureOutput": false, + "timeout": "0.12:00:00" + }, + "type": "Script", + "typeProperties": { + "database": { + "type": "Expression", + "value": "@pipeline().parameters.LakehouseName" + }, + "scriptBlockExecutionTimeout": "02:00:00", + "scripts": [ + { + "text": { + "type": "Expression", + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FileAggsView" + }, + "type": "NonQuery" + } + ] + } + } + ], + "lastModifiedByObjectId": "909f00ae-4251-400b-9be6-4af2e0aa2c1c", + "lastPublishTime": "2025-07-10T01:55:59Z", + "parameters": { + "EndTime": { + "type": "string" + }, + "FilesFilter": { + "type": "string" + }, + "FilesTableName": { + "defaultValue": "Files", + "type": "string" + }, + "ForceFullWhenDataExists": { + "defaultValue": false, + "type": "bool" + }, + "LakehouseName": { + "defaultValue": "<>", + "type": "string" + }, + "SitesFilter": { + "type": "string" + }, + "SitesTableName": { + "defaultValue": "Sites", + "type": "string" + }, + "StagingSuffix": { + "defaultValue": "_Staging", + "type": "string" + }, + "StartTime": { + "type": "string" + } + }, + "variables": { + "EndTime": { + "type": "String" + }, + "FilesDataExists": { + "defaultValue": false, + "type": "Boolean" + }, + "FilesExtractValidationsFailed": { + "defaultValue": "false", + "type": "String" + }, + "PipelineFinalStatus": { + "type": "String" + }, + "SiteExtractValidationsFailed": { + "defaultValue": "false", + "type": "String" + }, + "SitesDataExists": { + "defaultValue": false, + "type": "Boolean" + }, + "StartTimeForFiles": { + "type": "String" + }, + "StartTimeForSites": { + "type": "String" + } + } + }, + "type": "pipelines" + } + ], + "variables": {} +} \ No newline at end of file diff --git a/templates/Unlock SharePoint Capacity Insights/manifest.json b/templates/Unlock SharePoint Capacity Insights/manifest.json new file mode 100644 index 00000000..9e108bee --- /dev/null +++ b/templates/Unlock SharePoint Capacity Insights/manifest.json @@ -0,0 +1,30 @@ +{ + "annotations": [], + "author": "Microsoft", + "categories": [], + "icons": [ + "TridentNotebook", + "SetVariable", + "TridentNotebook" + ], + "image": "NotebookMerge Sites andFiles To Final TableSet variableSet End SnapshotDateNotebookRead Last SnapshotDatesSet variableSet Sites SnapshotDateSet variableSet Files SnapshotDateScriptSites View -Lakehouse...ScriptFiles View -Lakehouse...Set variableDoes Files DataExistsSet variableDoes Sites DataExistsIf ConditionExtract Sites FromMGDCTrueDo NotExtract...+FalseExtractSites To...+If ConditionExtract Files FromMGDCTrueDo NotExtract...+FalseExtractFiles To...+Set variableFiles ValidationsFailedSet variableSite ValidationsFailedSet variablePipeline SuccessWaitWait 1 MinScriptFiles Aggs View -Lakehouse...", + "name": "Unlock SharePoint Capacity Insights", + "requires": { + "linkedservices": { + "LS_Lakehouse-SQLEndPoint": { + "supportTypes": [ + "AzureSqlDatabase" + ] + }, + "LS_Office365_MGDC": { + "supportTypes": [ + "Office365" + ] + } + } + }, + "scope": [ + "PBI" + ], + "services": [] +} \ No newline at end of file From 59e16352e32d7c9c61c909585bbefe8e0de5d170 Mon Sep 17 00:00:00 2001 From: Uday Kumar Pasumarthy Date: Thu, 10 Jul 2025 17:39:50 -0700 Subject: [PATCH 2/2] Adding capacity template and updates to oversharing template --- .../Unlock SharePoint Capacity Insights.json | 1461 +++++++++-------- .../manifest.json | 14 +- ...nlock SharePoint Oversharing Insights.json | 38 +- .../manifest.json | 3 +- 4 files changed, 765 insertions(+), 751 deletions(-) diff --git a/templates/Unlock SharePoint Capacity Insights/Unlock SharePoint Capacity Insights.json b/templates/Unlock SharePoint Capacity Insights/Unlock SharePoint Capacity Insights.json index bf40be21..1fec2dc9 100644 --- a/templates/Unlock SharePoint Capacity Insights/Unlock SharePoint Capacity Insights.json +++ b/templates/Unlock SharePoint Capacity Insights/Unlock SharePoint Capacity Insights.json @@ -9,14 +9,17 @@ "type": "string" } }, + "variables": {}, "resources": [ { - "apiVersion": "2018-06-01", - "dependsOn": [], "name": "Unlock SharePoint Capacity Insights", + "type": "pipelines", + "apiVersion": "2018-06-01", "properties": { "activities": [ { + "name": "Merge Sites and Files To Final Table", + "type": "TridentNotebook", "dependsOn": [ { "activity": "Extract Sites From MGDC", @@ -31,75 +34,76 @@ ] } ], - "name": "Merge Sites and Files To Final Table", "policy": { + "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, - "secureInput": false, "secureOutput": false, - "timeout": "0.12:00:00" + "secureInput": false }, - "type": "TridentNotebook", "typeProperties": { "notebookId": "9e1e4bfb-8b70-4a68-9212-97fa7a183a8e", + "workspaceId": { + "value": "@pipeline().DataFactory", + "type": "Expression" + }, "parameters": { - "filesFinalTableName": { - "type": "string", + "sitesStagingTableName": { "value": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesFinalTableName" - } + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesStagingTableName", + "type": "Expression" + }, + "type": "string" }, - "filesStagingTableName": { - "type": "string", + "sitesFinalTableName": { "value": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesStagingTableName" - } + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesFinalTableName", + "type": "Expression" + }, + "type": "string" }, - "lakehouseName": { - "type": "string", + "filesStagingTableName": { "value": { - "type": "Expression", - "value": "@pipeline().parameters.LakehouseName" - } + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesStagingTableName", + "type": "Expression" + }, + "type": "string" }, - "runId": { - "type": "string", + "filesFinalTableName": { "value": { - "type": "Expression", - "value": "@pipeline().RunId" - } + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesFinalTableName", + "type": "Expression" + }, + "type": "string" }, - "sitesFinalTableName": { - "type": "string", + "runId": { "value": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesFinalTableName" - } + "value": "@pipeline().RunId", + "type": "Expression" + }, + "type": "string" }, - "sitesStagingTableName": { - "type": "string", + "lakehouseName": { "value": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesStagingTableName" - } + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + }, + "type": "string" }, "workspaceId": { - "type": "string", "value": { - "type": "Expression", - "value": "@pipeline().DataFactory" - } + "value": "@pipeline().DataFactory", + "type": "Expression" + }, + "type": "string" } - }, - "workspaceId": { - "type": "Expression", - "value": "@pipeline().DataFactory" } } }, { + "name": "Set End Snapshot Date", + "description": "", + "type": "SetVariable", "dependsOn": [ { "activity": "Read Last Snapshot Dates", @@ -108,99 +112,98 @@ ] } ], - "description": "", - "name": "Set End Snapshot Date", "policy": { - "secureInput": false, - "secureOutput": false + "secureOutput": false, + "secureInput": false }, - "type": "SetVariable", "typeProperties": { + "variableName": "EndTime", "value": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).EndSnapshotDate\n\n" - }, - "variableName": "EndTime" + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).EndSnapshotDate\n\n", + "type": "Expression" + } } }, { - "dependsOn": [], "name": "Read Last Snapshot Dates", + "type": "TridentNotebook", + "dependsOn": [], "policy": { + "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, - "secureInput": false, "secureOutput": false, - "timeout": "0.12:00:00" + "secureInput": false }, - "type": "TridentNotebook", "typeProperties": { "notebookId": "3b37b5cd-619c-43d0-a68c-2027a8ccd32b", + "workspaceId": { + "value": "@pipeline().DataFactory", + "type": "Expression" + }, "parameters": { - "endTime": { - "type": "string", + "sitesStagingTableName": { "value": { - "type": "Expression", - "value": "@coalesce(pipeline().parameters.EndTime,formatDateTime(adddays(startOfDay(pipeline().TriggerTime),-2),'yyyy-MM-ddTHH:mm:ssZ'))\n" - } + "value": "@concat(pipeline().parameters.SitesTableName,pipeline().parameters.StagingSuffix)", + "type": "Expression" + }, + "type": "string" }, - "filesFinalTableName": { - "type": "string", + "sitesFinalTableName": { "value": { - "type": "Expression", - "value": "@pipeline().parameters.FilesTableName" - } + "value": "@pipeline().parameters.SitesTableName", + "type": "Expression" + }, + "type": "string" }, "filessStagingTableName": { - "type": "string", "value": { - "type": "Expression", - "value": "@concat(pipeline().parameters.FilesTableName,pipeline().parameters.StagingSuffix)" - } + "value": "@concat(pipeline().parameters.FilesTableName,pipeline().parameters.StagingSuffix)", + "type": "Expression" + }, + "type": "string" }, - "lakehouseName": { - "type": "string", + "filesFinalTableName": { "value": { - "type": "Expression", - "value": "@pipeline().parameters.LakehouseName" - } + "value": "@pipeline().parameters.FilesTableName", + "type": "Expression" + }, + "type": "string" }, "runId": { - "type": "string", "value": { - "type": "Expression", - "value": "@pipeline().RunId" - } + "value": "@pipeline().RunId", + "type": "Expression" + }, + "type": "string" }, - "sitesFinalTableName": { - "type": "string", + "lakehouseName": { "value": { - "type": "Expression", - "value": "@pipeline().parameters.SitesTableName" - } + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + }, + "type": "string" }, - "sitesStagingTableName": { - "type": "string", + "workspaceId": { "value": { - "type": "Expression", - "value": "@concat(pipeline().parameters.SitesTableName,pipeline().parameters.StagingSuffix)" - } + "value": "@pipeline().DataFactory", + "type": "Expression" + }, + "type": "string" }, - "workspaceId": { - "type": "string", + "endTime": { "value": { - "type": "Expression", - "value": "@pipeline().DataFactory" - } + "value": "@coalesce(pipeline().parameters.EndTime,formatDateTime(adddays(startOfDay(pipeline().TriggerTime),-2),'yyyy-MM-ddTHH:mm:ssZ'))\n", + "type": "Expression" + }, + "type": "string" } - }, - "workspaceId": { - "type": "Expression", - "value": "@pipeline().DataFactory" } } }, { + "name": "Set Sites Snapshot Date", + "type": "SetVariable", "dependsOn": [ { "activity": "Read Last Snapshot Dates", @@ -209,21 +212,21 @@ ] } ], - "name": "Set Sites Snapshot Date", "policy": { - "secureInput": false, - "secureOutput": false + "secureOutput": false, + "secureInput": false }, - "type": "SetVariable", "typeProperties": { + "variableName": "StartTimeForSites", "value": { - "type": "Expression", - "value": "@coalesce(pipeline().parameters.StartTime, json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesSnapshotDate)" - }, - "variableName": "StartTimeForSites" + "value": "@coalesce(pipeline().parameters.StartTime, json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesSnapshotDate)", + "type": "Expression" + } } }, { + "name": "Set Files Snapshot Date", + "type": "SetVariable", "dependsOn": [ { "activity": "Read Last Snapshot Dates", @@ -232,21 +235,21 @@ ] } ], - "name": "Set Files Snapshot Date", "policy": { - "secureInput": false, - "secureOutput": false + "secureOutput": false, + "secureInput": false }, - "type": "SetVariable", "typeProperties": { + "variableName": "StartTimeForFiles", "value": { - "type": "Expression", - "value": "@coalesce(pipeline().parameters.StartTime, json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesSnapshotDate)" - }, - "variableName": "StartTimeForFiles" + "value": "@coalesce(pipeline().parameters.StartTime, json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesSnapshotDate)", + "type": "Expression" + } } }, { + "name": "Sites View - Lakehouse SQLEndpoint", + "type": "Script", "dependsOn": [ { "activity": "Wait 1 Min", @@ -255,36 +258,36 @@ ] } ], - "externalReferences": { - "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" - }, - "name": "Sites View - Lakehouse SQLEndpoint", "policy": { + "timeout": "0.12:00:00", "retry": 2, "retryIntervalInSeconds": 300, - "secureInput": false, "secureOutput": false, - "timeout": "0.12:00:00" + "secureInput": false }, - "type": "Script", "typeProperties": { - "database": { - "type": "Expression", - "value": "@pipeline().parameters.LakehouseName" - }, - "scriptBlockExecutionTimeout": "02:00:00", "scripts": [ { + "type": "NonQuery", "text": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesView" - }, - "type": "NonQuery" + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesView", + "type": "Expression" + } } - ] + ], + "scriptBlockExecutionTimeout": "02:00:00", + "database": { + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + } + }, + "externalReferences": { + "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" } }, { + "name": "Files View - Lakehouse SQLEndpoint", + "type": "Script", "dependsOn": [ { "activity": "Wait 1 Min", @@ -293,36 +296,36 @@ ] } ], - "externalReferences": { - "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" - }, - "name": "Files View - Lakehouse SQLEndpoint", "policy": { + "timeout": "0.12:00:00", "retry": 2, "retryIntervalInSeconds": 300, - "secureInput": false, "secureOutput": false, - "timeout": "0.12:00:00" + "secureInput": false }, - "type": "Script", "typeProperties": { - "database": { - "type": "Expression", - "value": "@pipeline().parameters.LakehouseName" - }, - "scriptBlockExecutionTimeout": "02:00:00", "scripts": [ { + "type": "NonQuery", "text": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesView" - }, - "type": "NonQuery" + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesView", + "type": "Expression" + } } - ] + ], + "scriptBlockExecutionTimeout": "02:00:00", + "database": { + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + } + }, + "externalReferences": { + "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" } }, { + "name": "Does Files Data Exists", + "type": "SetVariable", "dependsOn": [ { "activity": "Read Last Snapshot Dates", @@ -331,21 +334,21 @@ ] } ], - "name": "Does Files Data Exists", "policy": { - "secureInput": false, - "secureOutput": false + "secureOutput": false, + "secureInput": false }, - "type": "SetVariable", "typeProperties": { + "variableName": "FilesDataExists", "value": { - "type": "Expression", - "value": "@bool(json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesDataExists)" - }, - "variableName": "FilesDataExists" + "value": "@bool(json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesDataExists)", + "type": "Expression" + } } }, { + "name": "Does Sites Data Exists", + "type": "SetVariable", "dependsOn": [ { "activity": "Read Last Snapshot Dates", @@ -354,21 +357,22 @@ ] } ], - "name": "Does Sites Data Exists", "policy": { - "secureInput": false, - "secureOutput": false + "secureOutput": false, + "secureInput": false }, - "type": "SetVariable", "typeProperties": { + "variableName": "SitesDataExists", "value": { - "type": "Expression", - "value": "@bool(json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesDataExists)" - }, - "variableName": "SitesDataExists" + "value": "@bool(json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesDataExists)", + "type": "Expression" + } } }, { + "name": "Extract Sites From MGDC", + "description": "", + "type": "IfCondition", "dependsOn": [ { "activity": "Set Sites Snapshot Date", @@ -389,29 +393,55 @@ ] } ], - "description": "", - "name": "Extract Sites From MGDC", - "type": "IfCondition", "typeProperties": { "expression": { - "type": "Expression", - "value": "@and(and(variables('SitesDataExists'),\n not(pipeline().parameters.ForceFullWhenDataExists)\t \n\t ), greaterOrEquals( startOfDay(variables('StartTimeForSites')) , startOfDay(variables('EndTime')) ))\n\n " + "value": "@and(and(variables('SitesDataExists'),\n not(pipeline().parameters.ForceFullWhenDataExists)\t \n\t ), greaterOrEquals( startOfDay(variables('StartTimeForSites')) , startOfDay(variables('EndTime')) ))\n\n ", + "type": "Expression" }, "ifFalseActivities": [ { - "dependsOn": [], "name": "Extract Sites To Staging", + "type": "Copy", + "dependsOn": [], "policy": { + "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, - "secureInput": false, "secureOutput": false, - "timeout": "0.12:00:00" + "secureInput": false }, - "type": "Copy", "typeProperties": { - "enableStaging": true, + "source": { + "type": "Office365Source", + "dateFilterColumn": "SnapshotDate", + "startTime": { + "value": "@variables('StartTimeForSites')", + "type": "Expression" + }, + "endTime": { + "value": "@variables('EndTime')", + "type": "Expression" + }, + "datasetSettings": { + "annotations": [], + "type": "Office365Table", + "schema": [], + "typeProperties": { + "tableName": "BasicDataSet_v0.SharePointSites_v1" + }, + "externalReferences": { + "connection": "[parameters('LS_Office365_MGDC')]" + } + }, + "DataFilter": { + "value": "@pipeline().parameters.SitesFilter", + "type": "Expression" + } + }, "sink": { + "type": "LakehouseTableSink", + "tableActionOption": "OverwriteSchema", + "partitionOption": "None", "datasetSettings": { "annotations": [], "linkedService": { @@ -420,715 +450,686 @@ "annotations": [], "type": "Lakehouse", "typeProperties": { + "workspaceId": "@pipeline().DataFactory", "artifactId": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).LakehouseId", - "rootFolder": "Tables", - "workspaceId": "@pipeline().DataFactory" + "rootFolder": "Tables" } } }, - "schema": [], "type": "LakehouseTable", + "schema": [], "typeProperties": { "schema": "dbo", "table": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesStagingTableName" + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesStagingTableName", + "type": "Expression" } } - }, - "partitionOption": "None", - "tableActionOption": "OverwriteSchema", - "type": "LakehouseTableSink" - }, - "source": { - "DataFilter": { - "type": "Expression", - "value": "@pipeline().parameters.SitesFilter" - }, - "datasetSettings": { - "annotations": [], - "externalReferences": { - "connection": "[parameters('LS_Office365_MGDC')]" - }, - "schema": [], - "type": "Office365Table", - "typeProperties": { - "tableName": "BasicDataSet_v0.SharePointSites_v1" - } - }, - "dateFilterColumn": "SnapshotDate", - "endTime": { - "type": "Expression", - "value": "@variables('EndTime')" - }, - "startTime": { - "type": "Expression", - "value": "@variables('StartTimeForSites')" - }, - "type": "Office365Source" + } }, + "enableStaging": true, "translator": { - "columnFlattenSettings": { - "flattenColumnDelimiter": "_", - "treatArrayAsString": false, - "treatStructAsString": false - }, + "type": "TabularTranslator", "mappings": [ { - "sink": { + "source": { "name": "ptenant", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ptenant", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Id", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Id", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Url", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Url", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "ArchiveState", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ArchiveState", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "RootWeb_Configuration", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "RootWeb_Configuration", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "RootWeb_Id", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "RootWeb_Id", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "RootWeb_Title", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "RootWeb_Title", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "RootWeb_WebTemplate", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "RootWeb_WebTemplate", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "RootWeb_WebTemplateId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "RootWeb_WebTemplateId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "RootWeb_LastItemModifiedDate", - "physicalType": "timestamp" + "type": "DateTime", + "physicalType": "datetime" }, - "source": { + "sink": { "name": "RootWeb_LastItemModifiedDate", - "physicalType": "datetime", - "type": "DateTime" + "physicalType": "timestamp" } }, { - "sink": { + "source": { "name": "WebCount", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "WebCount", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "StorageQuota", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "StorageQuota", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "StorageUsed", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "StorageUsed", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "StorageMetrics_MetadataSize", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "StorageMetrics_MetadataSize", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "StorageMetrics_TotalFileCount", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "StorageMetrics_TotalFileCount", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "StorageMetrics_TotalFileStreamSize", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "StorageMetrics_TotalFileStreamSize", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "StorageMetrics_TotalSize", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "StorageMetrics_TotalSize", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "GroupId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "GroupId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "GeoLocation", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "GeoLocation", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "IsInRecycleBin", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "IsInRecycleBin", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "RecycleBinItemCount", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "RecycleBinItemCount", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "RecycleBinItemSize", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "RecycleBinItemSize", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "SecondStageRecycleBinStorageUsage", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "SecondStageRecycleBinStorageUsage", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "IsTeamsConnectedSite", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "IsTeamsConnectedSite", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "IsTeamsChannelSite", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "IsTeamsChannelSite", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "TeamsChannelType", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "TeamsChannelType", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "IsCommunicationSite", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "IsCommunicationSite", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "IsHubSite", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "IsHubSite", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "HubSiteId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "HubSiteId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "IsOneDrive", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "IsOneDrive", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "BlockAccessFromUnmanagedDevices", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "BlockAccessFromUnmanagedDevices", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "BlockDownloadOfAllFilesOnUnmanagedDevices", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "BlockDownloadOfAllFilesOnUnmanagedDevices", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "BlockDownloadOfViewableFilesOnUnmanagedDevices", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "BlockDownloadOfViewableFilesOnUnmanagedDevices", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "IsExternalSharingEnabled", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "IsExternalSharingEnabled", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "Privacy", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Privacy", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "ShareByEmailEnabled", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "ShareByEmailEnabled", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "ShareByLinkEnabled", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "ShareByLinkEnabled", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "SiteConnectedToPrivateGroup", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "SiteConnectedToPrivateGroup", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "SensitivityLabelInfo_DisplayName", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SensitivityLabelInfo_DisplayName", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SensitivityLabelInfo_Id", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SensitivityLabelInfo_Id", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Classification", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Classification", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "IBMode", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "IBMode", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "IBSegments", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "IBSegments", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "RelatedGroupId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "RelatedGroupId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Owner_AadObjectId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Owner_AadObjectId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Owner_Email", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Owner_Email", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Owner_Name", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Owner_Name", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Owner_UPN", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Owner_UPN", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SecondaryContact_AadObjectId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SecondaryContact_AadObjectId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SecondaryContact_Email", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SecondaryContact_Email", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SecondaryContact_Name", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SecondaryContact_Name", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SecondaryContact_UPN", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SecondaryContact_UPN", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "ReadLocked", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "ReadLocked", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "ReadOnly", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "ReadOnly", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "CreatedTime", - "physicalType": "timestamp" + "type": "DateTime", + "physicalType": "datetime" }, - "source": { + "sink": { "name": "CreatedTime", - "physicalType": "datetime", - "type": "DateTime" + "physicalType": "timestamp" } }, { - "sink": { + "source": { "name": "LastSecurityModifiedDate", - "physicalType": "timestamp" + "type": "DateTime", + "physicalType": "datetime" }, - "source": { + "sink": { "name": "LastSecurityModifiedDate", - "physicalType": "datetime", - "type": "DateTime" + "physicalType": "timestamp" } }, { - "sink": { + "source": { "name": "Operation", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Operation", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "LastUserAccessDate", - "physicalType": "timestamp" + "type": "DateTime", + "physicalType": "datetime" }, - "source": { + "sink": { "name": "LastUserAccessDate", - "physicalType": "datetime", - "type": "DateTime" + "physicalType": "timestamp" } }, { - "sink": { + "source": { "name": "SnapshotDate", - "physicalType": "timestamp" + "type": "DateTime", + "physicalType": "datetime" }, - "source": { + "sink": { "name": "SnapshotDate", - "physicalType": "datetime", - "type": "DateTime" + "physicalType": "timestamp" } } ], - "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false + }, + "columnFlattenSettings": { + "treatArrayAsString": false, + "treatStructAsString": false, + "flattenColumnDelimiter": "_" } } } @@ -1136,19 +1137,22 @@ ], "ifTrueActivities": [ { - "dependsOn": [], - "description": "Do Not Extract Sites - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", "name": "Do Not Extract Sites", + "description": "Do Not Extract Sites - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", "type": "Fail", + "dependsOn": [], "typeProperties": { - "errorCode": "-999", - "message": "Error: -999 - Do Not Extract Sites - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false" + "message": "Error: -999 - Do Not Extract Sites - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", + "errorCode": "-999" } } ] } }, { + "name": "Extract Files From MGDC", + "description": "", + "type": "IfCondition", "dependsOn": [ { "activity": "Set Files Snapshot Date", @@ -1169,29 +1173,55 @@ ] } ], - "description": "", - "name": "Extract Files From MGDC", - "type": "IfCondition", "typeProperties": { "expression": { - "type": "Expression", - "value": "@and(and(variables('FilesDataExists'),\n not(pipeline().parameters.ForceFullWhenDataExists)\t \n\t ), greaterOrEquals( startOfDay(variables('StartTimeForFiles')) , startOfDay(variables('EndTime')) ))" + "value": "@and(and(variables('FilesDataExists'),\n not(pipeline().parameters.ForceFullWhenDataExists)\t \n\t ), greaterOrEquals( startOfDay(variables('StartTimeForFiles')) , startOfDay(variables('EndTime')) ))", + "type": "Expression" }, "ifFalseActivities": [ { - "dependsOn": [], "name": "Extract Files To Staging", + "type": "Copy", + "dependsOn": [], "policy": { + "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, - "secureInput": false, "secureOutput": false, - "timeout": "0.12:00:00" + "secureInput": false }, - "type": "Copy", "typeProperties": { - "enableStaging": true, + "source": { + "type": "Office365Source", + "dateFilterColumn": "SnapshotDate", + "startTime": { + "value": "@variables('StartTimeForFiles')", + "type": "Expression" + }, + "endTime": { + "value": "@variables('EndTime')", + "type": "Expression" + }, + "datasetSettings": { + "annotations": [], + "type": "Office365Table", + "schema": [], + "typeProperties": { + "tableName": "BasicDataSet_v0.SharePointFiles_v1" + }, + "DataFilter": { + "value": "@pipeline().parameters.FilesFilter", + "type": "Expression" + }, + "externalReferences": { + "connection": "[parameters('LS_Office365_MGDC')]" + } + } + }, "sink": { + "type": "LakehouseTableSink", + "tableActionOption": "OverwriteSchema", + "partitionOption": "None", "datasetSettings": { "annotations": [], "linkedService": { @@ -1200,374 +1230,345 @@ "annotations": [], "type": "Lakehouse", "typeProperties": { + "workspaceId": "@pipeline().DataFactory", "artifactId": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).LakehouseId", - "rootFolder": "Tables", - "workspaceId": "@pipeline().DataFactory" + "rootFolder": "Tables" } } }, - "schema": [], "type": "LakehouseTable", + "schema": [], "typeProperties": { "schema": "dbo", "table": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesStagingTableName" + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FilesStagingTableName", + "type": "Expression" } } - }, - "partitionOption": "None", - "tableActionOption": "OverwriteSchema", - "type": "LakehouseTableSink" - }, - "source": { - "datasetSettings": { - "annotations": [], - "DataFilter": { - "type": "Expression", - "value": "@pipeline().parameters.FilesFilter" - }, - "externalReferences": { - "connection": "[parameters('LS_Office365_MGDC')]" - }, - "schema": [], - "type": "Office365Table", - "typeProperties": { - "tableName": "BasicDataSet_v0.SharePointFiles_v1" - } - }, - "dateFilterColumn": "SnapshotDate", - "endTime": { - "type": "Expression", - "value": "@variables('EndTime')" - }, - "startTime": { - "type": "Expression", - "value": "@variables('StartTimeForFiles')" - }, - "type": "Office365Source" + } }, + "enableStaging": true, "translator": { - "columnFlattenSettings": { - "flattenColumnDelimiter": "_", - "treatArrayAsString": false, - "treatStructAsString": false - }, + "type": "TabularTranslator", "mappings": [ { - "sink": { + "source": { "name": "ptenant", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ptenant", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SiteId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SiteId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Author_Email", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Author_Email", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Author_Name", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Author_Name", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "DirName", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "DirName", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Extension", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Extension", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "FileName", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "FileName", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "IsLabelEncrypted", + "type": "Boolean", "physicalType": "boolean" }, - "source": { + "sink": { "name": "IsLabelEncrypted", - "physicalType": "boolean", - "type": "Boolean" + "physicalType": "boolean" } }, { - "sink": { + "source": { "name": "ItemId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ItemId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "ListId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ListId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "ListServerTemplate", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ListServerTemplate", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "MajorVersion", - "physicalType": "integer" + "type": "Int32", + "physicalType": "int32" }, - "source": { + "sink": { "name": "MajorVersion", - "physicalType": "int32", - "type": "Int32" + "physicalType": "integer" } }, { - "sink": { + "source": { "name": "MinorVersion", - "physicalType": "integer" + "type": "Int32", + "physicalType": "int32" }, - "source": { + "sink": { "name": "MinorVersion", - "physicalType": "int32", - "type": "Int32" + "physicalType": "integer" } }, { - "sink": { + "source": { "name": "ModifiedBy_Email", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ModifiedBy_Email", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "ModifiedBy_Name", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ModifiedBy_Name", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "Operation", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "Operation", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "QuickXorHash", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "QuickXorHash", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "ScopeId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "ScopeId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SensitivityLabelInfo_DisplayName", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SensitivityLabelInfo_DisplayName", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SensitivityLabelInfo_Id", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SensitivityLabelInfo_Id", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SiteUrl", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "SiteUrl", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "SizeInBytes", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "SizeInBytes", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "SizeInBytesWithVersions", - "physicalType": "long" + "type": "Int64", + "physicalType": "int64" }, - "source": { + "sink": { "name": "SizeInBytesWithVersions", - "physicalType": "int64", - "type": "Int64" + "physicalType": "long" } }, { - "sink": { + "source": { "name": "TimeCreated", - "physicalType": "timestamp" + "type": "DateTime", + "physicalType": "datetime" }, - "source": { + "sink": { "name": "TimeCreated", - "physicalType": "datetime", - "type": "DateTime" + "physicalType": "timestamp" } }, { - "sink": { + "source": { "name": "TimeLastModified", - "physicalType": "timestamp" + "type": "DateTime", + "physicalType": "datetime" }, - "source": { + "sink": { "name": "TimeLastModified", - "physicalType": "datetime", - "type": "DateTime" + "physicalType": "timestamp" } }, { - "sink": { + "source": { "name": "WebId", + "type": "String", "physicalType": "string" }, - "source": { + "sink": { "name": "WebId", - "physicalType": "string", - "type": "String" + "physicalType": "string" } }, { - "sink": { + "source": { "name": "WebTemplateId", - "physicalType": "integer" + "type": "Int32", + "physicalType": "int32" }, - "source": { + "sink": { "name": "WebTemplateId", - "physicalType": "int32", - "type": "Int32" + "physicalType": "integer" } }, { - "sink": { + "source": { "name": "SnapshotDate", - "physicalType": "timestamp" + "type": "DateTime", + "physicalType": "datetime" }, - "source": { + "sink": { "name": "SnapshotDate", - "physicalType": "datetime", - "type": "DateTime" + "physicalType": "timestamp" } } ], - "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false + }, + "columnFlattenSettings": { + "treatArrayAsString": false, + "treatStructAsString": false, + "flattenColumnDelimiter": "_" } } } @@ -1575,19 +1576,21 @@ ], "ifTrueActivities": [ { - "dependsOn": [], - "description": "Do Not Extract Files - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", "name": "Do Not Extract Files", + "description": "Do Not Extract Files - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", "type": "Fail", + "dependsOn": [], "typeProperties": { - "errorCode": "-999", - "message": "Error: -999 - Do Not Extract Files - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false" + "message": "Error: -999 - Do Not Extract Files - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", + "errorCode": "-999" } } ] } }, { + "name": "Files Validations Failed", + "type": "SetVariable", "dependsOn": [ { "activity": "Extract Files From MGDC", @@ -1596,18 +1599,19 @@ ] } ], - "name": "Files Validations Failed", "policy": { - "secureInput": false, - "secureOutput": false + "secureOutput": false, + "secureInput": false }, - "type": "SetVariable", "typeProperties": { - "value": "true", - "variableName": "FilesExtractValidationsFailed" + "variableName": "FilesExtractValidationsFailed", + "value": "true" } }, { + "name": "Site Validations Failed", + "description": "true", + "type": "SetVariable", "dependsOn": [ { "activity": "Extract Sites From MGDC", @@ -1616,19 +1620,19 @@ ] } ], - "description": "true", - "name": "Site Validations Failed", "policy": { - "secureInput": false, - "secureOutput": false + "secureOutput": false, + "secureInput": false }, - "type": "SetVariable", "typeProperties": { - "value": "true", - "variableName": "FilesExtractValidationsFailed" + "variableName": "FilesExtractValidationsFailed", + "value": "true" } }, { + "name": "Pipeline Success", + "description": "If this activity completed then Pipeline Completed Successfully", + "type": "SetVariable", "dependsOn": [ { "activity": "Sites View - Lakehouse SQLEndpoint", @@ -1649,19 +1653,19 @@ ] } ], - "description": "If this activity completed then Pipeline Completed Successfully", - "name": "Pipeline Success", "policy": { - "secureInput": false, - "secureOutput": false + "secureOutput": false, + "secureInput": false }, - "type": "SetVariable", "typeProperties": { - "value": "Pipeline Completed Successfully", - "variableName": "PipelineFinalStatus" + "variableName": "PipelineFinalStatus", + "value": "Pipeline Completed Successfully" } }, { + "name": "Wait 1 Min", + "description": "Some times SQL Endpoint does not understand the newly created delta lake tables. Need to wait some time for newly created tables to get reflected in SQL catalog.", + "type": "Wait", "dependsOn": [ { "activity": "Merge Sites and Files To Final Table", @@ -1670,13 +1674,13 @@ ] } ], - "name": "Wait 1 Min", - "type": "Wait", "typeProperties": { "waitTimeInSeconds": 60 } }, { + "name": "Files Aggs View - Lakehouse SQLEndpoint", + "type": "Script", "dependsOn": [ { "activity": "Wait 1 Min", @@ -1685,105 +1689,102 @@ ] } ], - "externalReferences": { - "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" - }, - "name": "Files Aggs View - Lakehouse SQLEndpoint", "policy": { + "timeout": "0.12:00:00", "retry": 2, "retryIntervalInSeconds": 300, - "secureInput": false, "secureOutput": false, - "timeout": "0.12:00:00" + "secureInput": false }, - "type": "Script", "typeProperties": { - "database": { - "type": "Expression", - "value": "@pipeline().parameters.LakehouseName" - }, - "scriptBlockExecutionTimeout": "02:00:00", "scripts": [ { + "type": "NonQuery", "text": { - "type": "Expression", - "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FileAggsView" - }, - "type": "NonQuery" + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).FileAggsView", + "type": "Expression" + } } - ] + ], + "scriptBlockExecutionTimeout": "02:00:00", + "database": { + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + } + }, + "externalReferences": { + "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" } } ], - "lastModifiedByObjectId": "909f00ae-4251-400b-9be6-4af2e0aa2c1c", - "lastPublishTime": "2025-07-10T01:55:59Z", "parameters": { - "EndTime": { - "type": "string" - }, - "FilesFilter": { - "type": "string" - }, "FilesTableName": { - "defaultValue": "Files", - "type": "string" + "type": "string", + "defaultValue": "Files" }, - "ForceFullWhenDataExists": { - "defaultValue": false, - "type": "bool" + "SitesTableName": { + "type": "string", + "defaultValue": "Sites" + }, + "StagingSuffix": { + "type": "string", + "defaultValue": "_Staging" }, "LakehouseName": { - "defaultValue": "<>", - "type": "string" + "type": "string", + "defaultValue": "<>" }, - "SitesFilter": { + "StartTime": { "type": "string" }, - "SitesTableName": { - "defaultValue": "Sites", + "EndTime": { "type": "string" }, - "StagingSuffix": { - "defaultValue": "_Staging", + "SitesFilter": { "type": "string" }, - "StartTime": { + "FilesFilter": { "type": "string" + }, + "ForceFullWhenDataExists": { + "type": "bool", + "defaultValue": false } }, "variables": { - "EndTime": { - "type": "String" - }, - "FilesDataExists": { - "defaultValue": false, - "type": "Boolean" - }, - "FilesExtractValidationsFailed": { - "defaultValue": "false", + "StartTimeForSites": { "type": "String" }, - "PipelineFinalStatus": { + "EndTime": { "type": "String" }, - "SiteExtractValidationsFailed": { - "defaultValue": "false", + "StartTimeForFiles": { "type": "String" }, "SitesDataExists": { - "defaultValue": false, - "type": "Boolean" + "type": "Boolean", + "defaultValue": false }, - "StartTimeForFiles": { - "type": "String" + "FilesDataExists": { + "type": "Boolean", + "defaultValue": false }, - "StartTimeForSites": { + "SiteExtractValidationsFailed": { + "type": "String", + "defaultValue": "false" + }, + "FilesExtractValidationsFailed": { + "type": "String", + "defaultValue": "false" + }, + "PipelineFinalStatus": { "type": "String" } - } + }, + "lastModifiedByObjectId": "909f00ae-4251-400b-9be6-4af2e0aa2c1c", + "lastPublishTime": "2025-07-10T01:55:59Z" }, - "type": "pipelines" + "dependsOn": [] } - ], - "variables": {} + ] } \ No newline at end of file diff --git a/templates/Unlock SharePoint Capacity Insights/manifest.json b/templates/Unlock SharePoint Capacity Insights/manifest.json index 9e108bee..329cecc5 100644 --- a/templates/Unlock SharePoint Capacity Insights/manifest.json +++ b/templates/Unlock SharePoint Capacity Insights/manifest.json @@ -1,14 +1,11 @@ { - "annotations": [], - "author": "Microsoft", - "categories": [], + "name": "Unlock SharePoint Capacity Insights", + "image": "NotebookMerge Sites andFiles To Final TableSet variableSet End SnapshotDateNotebookRead Last SnapshotDatesSet variableSet Sites SnapshotDateSet variableSet Files SnapshotDateScriptSites View -Lakehouse...ScriptFiles View -Lakehouse...Set variableDoes Files DataExistsSet variableDoes Sites DataExistsIf ConditionExtract Sites FromMGDCTrueDo NotExtract...+FalseExtractSites To...+If ConditionExtract Files FromMGDCTrueDo NotExtract...+FalseExtractFiles To...+Set variableFiles ValidationsFailedSet variableSite ValidationsFailedSet variablePipeline SuccessWaitWait 1 MinScriptFiles Aggs View -Lakehouse...", "icons": [ "TridentNotebook", "SetVariable", "TridentNotebook" ], - "image": "NotebookMerge Sites andFiles To Final TableSet variableSet End SnapshotDateNotebookRead Last SnapshotDatesSet variableSet Sites SnapshotDateSet variableSet Files SnapshotDateScriptSites View -Lakehouse...ScriptFiles View -Lakehouse...Set variableDoes Files DataExistsSet variableDoes Sites DataExistsIf ConditionExtract Sites FromMGDCTrueDo NotExtract...+FalseExtractSites To...+If ConditionExtract Files FromMGDCTrueDo NotExtract...+FalseExtractFiles To...+Set variableFiles ValidationsFailedSet variableSite ValidationsFailedSet variablePipeline SuccessWaitWait 1 MinScriptFiles Aggs View -Lakehouse...", - "name": "Unlock SharePoint Capacity Insights", "requires": { "linkedservices": { "LS_Lakehouse-SQLEndPoint": { @@ -23,8 +20,11 @@ } } }, + "author": "Microsoft", + "annotations": [], + "services": [], + "categories": [], "scope": [ "PBI" - ], - "services": [] + ] } \ No newline at end of file diff --git a/templates/Unlock SharePoint Oversharing Insights/Unlock SharePoint Oversharing Insights.json b/templates/Unlock SharePoint Oversharing Insights/Unlock SharePoint Oversharing Insights.json index 7d6e0158..1d767ba9 100644 --- a/templates/Unlock SharePoint Oversharing Insights/Unlock SharePoint Oversharing Insights.json +++ b/templates/Unlock SharePoint Oversharing Insights/Unlock SharePoint Oversharing Insights.json @@ -16,7 +16,6 @@ "type": "pipelines", "apiVersion": "2018-06-01", "properties": { - "description": "This Pipeline will extract the data from Microsoft Graph Data connect to Lakehouse for getting oversharing insights in your organization. This pipeline needs some dependencies and steps documented here. https://go.microsoft.com/fwlink/?linkid=2300611", "activities": [ { "name": "Merge Sites and Permissions To Final Table", @@ -43,7 +42,7 @@ "secureInput": false }, "typeProperties": { - "notebookId": "044e9c94-b555-42eb-af0d-b9b316b0c501", + "notebookId": "de1fae8a-2f39-4c73-9a71-f8e8097f9387", "workspaceId": { "value": "@pipeline().DataFactory", "type": "Expression" @@ -137,7 +136,7 @@ "secureInput": false }, "typeProperties": { - "notebookId": "31fc030f-0e0c-46bc-ae89-281bd5c9272a", + "notebookId": "e964108f-ea31-426a-9459-0c549409fed8", "workspaceId": { "value": "@pipeline().DataFactory", "type": "Expression" @@ -253,7 +252,7 @@ "type": "Script", "dependsOn": [ { - "activity": "Merge Sites and Permissions To Final Table", + "activity": "Wait 1 Min", "dependencyConditions": [ "Succeeded" ] @@ -261,8 +260,8 @@ ], "policy": { "timeout": "0.12:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, + "retry": 2, + "retryIntervalInSeconds": 300, "secureOutput": false, "secureInput": false }, @@ -291,7 +290,7 @@ "type": "Script", "dependsOn": [ { - "activity": "Merge Sites and Permissions To Final Table", + "activity": "Wait 1 Min", "dependencyConditions": [ "Succeeded" ] @@ -299,8 +298,8 @@ ], "policy": { "timeout": "0.12:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, + "retry": 2, + "retryIntervalInSeconds": 300, "secureOutput": false, "secureInput": false }, @@ -1722,6 +1721,22 @@ "variableName": "PipelineFinalStatus", "value": "Pipeline Completed Successfully" } + }, + { + "name": "Wait 1 Min", + "description": "Some times SQL Endpoint does not understand the newly created delta lake tables. Need to wait some time for newly created tables to get reflected in SQL catalog.", + "type": "Wait", + "dependsOn": [ + { + "activity": "Merge Sites and Permissions To Final Table", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "waitTimeInSeconds": 60 + } } ], "parameters": { @@ -1788,9 +1803,8 @@ "type": "String" } }, - "lastModifiedByObjectId": "1a2fbd2b-16c9-4b04-96de-be6e856e5333", - "lastPublishTime": "2025-01-03T20:19:45Z", - "extraTridentInfo": "{\"folderObjectId\":\"0cc8ced7-35a7-4dcb-94af-3b4f54c381b0\",\"capacityObjectId\":\"633251C5-A944-4D3E-B523-E61846F67005\",\"artifactType\":\"Pipeline\",\"provisionState\":\"Active\",\"lastUpdatedDate\":\"2025-01-03T20:19:43.984Z\",\"createdDate\":\"2024-11-11T21:15:17.805Z\",\"ownerUser\":{\"id\":619869,\"name\":\"Admin Archimedes\",\"objectId\":\"1a2fbd2b-16c9-4b04-96de-be6e856e5333\",\"userPrincipalName\":\"admin@odspimax.onmicrosoft.com\",\"aadAppId\":null,\"type\":null},\"ownerUserId\":619869,\"createdByUser\":{\"id\":619869,\"name\":\"Admin Archimedes\",\"objectId\":\"1a2fbd2b-16c9-4b04-96de-be6e856e5333\",\"userPrincipalName\":\"admin@odspimax.onmicrosoft.com\",\"aadAppId\":null,\"type\":null},\"createdByUserId\":619869,\"modifiedByUser\":{\"id\":619869,\"name\":\"Admin Archimedes\",\"objectId\":\"1a2fbd2b-16c9-4b04-96de-be6e856e5333\",\"userPrincipalName\":\"admin@odspimax.onmicrosoft.com\",\"aadAppId\":null,\"type\":null},\"modifiedByUserId\":619869,\"artifactRelations\":[{\"dependentArtifactObjectId\":\"044e9c94-b555-42eb-af0d-b9b316b0c501\",\"settings\":0},{\"dependentArtifactObjectId\":\"31fc030f-0e0c-46bc-ae89-281bd5c9272a\",\"settings\":0}],\"permissions\":71,\"artifactPermissions\":7,\"systemArtifactType\":\"None\",\"etag\":\"\\\"602479d8aa384b309be9371e34286d2a\\\"\"}" + "lastModifiedByObjectId": "909f00ae-4251-400b-9be6-4af2e0aa2c1c", + "lastPublishTime": "2025-07-10T23:43:54Z" }, "dependsOn": [] } diff --git a/templates/Unlock SharePoint Oversharing Insights/manifest.json b/templates/Unlock SharePoint Oversharing Insights/manifest.json index 1e31861e..b93dbcf6 100644 --- a/templates/Unlock SharePoint Oversharing Insights/manifest.json +++ b/templates/Unlock SharePoint Oversharing Insights/manifest.json @@ -1,7 +1,6 @@ { "name": "Unlock SharePoint Oversharing Insights", - "description": "This Pipeline will extract the data from Microsoft Graph Data connect to Lakehouse for getting oversharing insights in your organization. This pipeline needs some dependencies and steps documented here. https://go.microsoft.com/fwlink/?linkid=2300611", - "image": "NotebookMerge Sites andPermissions To...Set variableSet End SnapshotDateNotebookRead Last SnapshotDatesSet variableSet Sites SnapshotDateSet variableSet PermissionsSnapshot DateScriptSites View -Lakehouse...ScriptPermissions View -Lakehouse...Set variableDoes PermissionsData ExistsSet variableDoes Sites DataExistsIf ConditionExtract Sites FromMGDCTrueDo NotExtract...+FalseExtractSites To...+If ConditionExtract PermissionsFrom MGDCTrueDo NotExtract...+FalseExtractPermissio...+Set variablePermissionValidations FailedSet variableSite ValidationsFailedSet variablePipeline Success", + "image": "NotebookMerge Sites andPermissions To...Set variableSet End SnapshotDateNotebookRead Last SnapshotDatesSet variableSet Sites SnapshotDateSet variableSet PermissionsSnapshot DateScriptSites View -Lakehouse...ScriptPermissions View -Lakehouse...Set variableDoes PermissionsData ExistsSet variableDoes Sites DataExistsIf ConditionExtract Sites FromMGDCTrueDo NotExtract...+FalseExtractSites To...+If ConditionExtract PermissionsFrom MGDCTrueDo NotExtract...+FalseExtractPermissio...+Set variablePermissionValidations FailedSet variableSite ValidationsFailedSet variablePipeline SuccessWaitWait 1 Min", "icons": [ "TridentNotebook", "SetVariable",