{ "name": "PL_transform_dataflow_ADLS_AzureSQL_SinglePLFailure", "properties": { "activities": [ { "name": "Lk_Xref_ObjectType_Reference", "type": "Lookup", "dependsOn": [ { "activity": "Lk_Start_Logs", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": "SELECT ObjectType, LoadStatus FROM [dbo].[TBL_LOG_FAILURE_PIPELINE]\nWHERE LoadStatus='ResetMode'", "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "DS_XrefObjectTypeUpdate", "type": "DatasetReference" }, "firstRowOnly": false } }, { "name": "ForEach_Xref_ObjectType_Reference", "description": "using switch activity to rotate 40 data flows", "type": "ForEach", "dependsOn": [ { "activity": "Lk_Xref_ObjectType_Reference", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Lk_Xref_ObjectType_Reference').output.value", "type": "Expression" }, "isSequential": false, "activities": [ { "name": "SwitchStartFromFailureTable", "type": "Switch", "dependsOn": [], "userProperties": [], "typeProperties": { "on": { "value": "@item().ObjectType", "type": "Expression" }, "cases": [ { "value": "actor", "activities": [ { "name": "DF_User Activity_audit_log_actor", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_actor", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActor": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(),'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "actor_roles", "activities": [ { "name": "DF_User Activity_audit_log_actor_roles", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_actor_roles", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "auditlogactorroles": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(),'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_Base", "activities": [ { "name": "DF_User Activity_audit_log_activities_base", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_base", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesBase": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(),'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_meet", "activities": [ { "name": "DF_User Activity_audit_log_activities_meet", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_meet", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesMeet": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_meet_users", "activities": [ { "name": "DF_User Activity_audit_log_activities_meet_users", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_meet_users", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesMeet": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_presence", "activities": [ { "name": "DF_User Activity_audit_log_activities_presence", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_presence", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesPresence": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_user", "activities": [ { "name": "DF_User Activity_audit_log_activities_user", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_user", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesUser": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_user_users", "activities": [ { "name": "DF_User Activity_audit_log_activities_user_users", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_user_users", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesUserUsers": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_binder", "activities": [ { "name": "DF_User Activity_audit_log_activities_binder", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_binder", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesBinder": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_binder_users", "activities": [ { "name": "DF_User Activity_audit_log_activities_binder_users", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_binder_users", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesBinderUsers": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_binder_updated_users", "activities": [ { "name": "DF_User Act_audit_log_activities_binder_updated_users", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Act_audit_log_activities_binder_updated_users", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesBinderUpdatedUsers": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_binder_tag", "activities": [ { "name": "DF_User Activity_audit_log_activities_binder_tag", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_binder_tag", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesBinderTag": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] }, { "value": "activities_binder_tag_tags", "activities": [ { "name": "DF_User Activity_audit_log_activities_binder_tag_tags", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "DF_User Activity_audit_log_activities_binder_tag_tags", "type": "DataFlowReference", "parameters": { "df_ETL": { "value": "'@{variables('v_ETL')}'", "type": "Expression" } }, "datasetParameters": { "AuditLogActivitiesBinderTagTags": { "Folder_Path": { "value": "@{concat(formatDateTime(utcNow(), 'yyyy/MM/dd'))}", "type": "Expression" } } } }, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ] } ], "defaultActivities": [ { "name": "WaitOther", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ] } } ] } }, { "name": "SP_Restart_Failure_Pipeline", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "ForEach_Xref_ObjectType_Reference", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[dbo].[TBL_LOG_FAILURE_PIPELINE_RESETALL]" }, "linkedServiceName": { "referenceName": "ls_AZR_SQLDB", "type": "LinkedServiceReference" } }, { "name": "SP_Logs_Status_Success", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "ForEach_Xref_ObjectType_Reference", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[logging].[UPDATE_LOGS]", "storedProcedureParameters": { "Deletes": { "value": null, "type": "Int32" }, "DurationSeconds": { "value": null, "type": "Int32" }, "EndTime": { "value": { "value": "@utcnow()", "type": "Expression" }, "type": "DateTime" }, "ErrorMessage": { "value": null, "type": "String" }, "Inserts": { "value": null, "type": "Int32" }, "LogId": { "value": null, "type": "Int32" }, "StatusMessage": { "value": null, "type": "String" }, "Updates": { "value": null, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "ls_AZR_SQLDB", "type": "LinkedServiceReference" } }, { "name": "Lk_Start_Logs", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[logging].[INSERT_LOGS]", "storedProcedureParameters": { "Deletes": { "type": "Int32", "value": null }, "DurationSeconds": { "type": "Int32", "value": null }, "EndTime": { "type": "DateTime", "value": null }, "ErrorMessage": { "type": "String", "value": null }, "Inserts": { "type": "Int32", "value": null }, "PipelineName": { "type": "String", "value": { "value": "@pipeline().Pipeline", "type": "Expression" } }, "PipelineRunID": { "type": "String", "value": { "value": "@pipeline().RunId", "type": "Expression" } }, "StartTime": { "type": "DateTime", "value": { "value": "@utcnow()", "type": "Expression" } }, "StatusMessage": { "type": "String", "value": null }, "Updates": { "type": "Int32", "value": null } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "DS_XrefObjectTypeUpdate", "type": "DatasetReference" } } }, { "name": "SP_Logs_Status_Failure", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "ForEach_Xref_ObjectType_Reference", "dependencyConditions": [ "Failed" ] } ], "policy": { "timeout": "0.12:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[logging].[UPDATE_LOGS]", "storedProcedureParameters": { "Deletes": { "value": null, "type": "Int32" }, "DurationSeconds": { "value": null, "type": "Int32" }, "EndTime": { "value": { "value": "@utcnow()", "type": "Expression" }, "type": "DateTime" }, "ErrorMessage": { "value": "", "type": "String" }, "Inserts": { "value": "", "type": "Int32" }, "LogId": { "value": "", "type": "Int32" }, "StatusMessage": { "value": "", "type": "String" }, "Updates": { "value": "", "type": "Int32" } } }, "linkedServiceName": { "referenceName": "ls_AZR_SQLDB", "type": "LinkedServiceReference" } } ], "variables": { "test_var": { "type": "String" }, "pipeline_status": { "type": "String" }, "v_ETL": { "type": "String" }, "lookvalue": { "type": "String" }, "var_test": { "type": "String" } }, "folder": { "name": "dbo DB/transformation_final" }, "annotations": [], "lastPublishTime": "2024-02-09T14:58:05Z" }, "type": "Microsoft.DataFactory/factories/pipelines" }