{ "runStatus": { "computeAcquisitionDuration": 115060, "version": "20220324.3", "profile": { "DerivedColumn1": { "computed": [], "lineage": {}, "dropped": 0, "drifted": 0, "newer": 2, "total": 294, "updated": 0 }, "Join1": { "computed": [], "lineage": {}, "dropped": 0, "drifted": 0, "newer": 0, "total": 266, "updated": 0 }, "source1": { "computed": [], "lineage": {}, "dropped": 0, "drifted": 0, "newer": 7, "total": 7, "updated": 0 }, "Join2": { "computed": [], "lineage": {}, "dropped": 0, "drifted": 0, "newer": 0, "total": 292, "updated": 0 }, "source2": { "computed": [], "lineage": {}, "dropped": 0, "drifted": 0, "newer": 259, "total": 259, "updated": 0 }, "sink1": { "computed": [ { "source": "source3", "columns": [ "ClassID", "Company" ] }, { "source": "source1", "columns": [ "ForeignSysRowID" ] }, { "source": "source2", "columns": [ "SysRowID", "Company", "ClassID" ] } ], "lineage": { "PART_DESC": { "mapped": false, "from": [ { "source": "source2", "columns": [ "PartDescriptions" ] } ] }, "SYSROW_ID": { "mapped": false, "from": [ { "source": "source2", "columns": [ "SysRowID" ] } ] }, "PART_NUM": { "mapped": true, "from": [ { "source": "source2", "columns": [ "PartNum" ] } ] }, "CLASS_ID": { "mapped": false, "from": [ { "source": "source2", "columns": [ "ClassID" ] } ] }, "LOAD_DATETIME": { "mapped": false, "from": [ { "source": "DerivedColumn1", "columns": [ "LOAD_DATETIME" ] } ] }, "COMPANY": { "mapped": false, "from": [ { "source": "source2", "columns": [ "Company" ] } ] }, "UNIT_PRICE": { "mapped": false, "from": [ { "source": "source2", "columns": [ "UnitPrice" ] } ] }, "SRC_SYS": { "mapped": false, "from": [ { "source": "DerivedColumn1", "columns": [ "SRC_SYS" ] } ] } }, "dropped": 282, "drifted": 0, "newer": 6, "total": 8, "updated": 2 }, "Aggregate1": { "computed": [], "lineage": {}, "dropped": 0, "drifted": 0, "newer": 0, "total": 259, "updated": 259 }, "source3": { "computed": [], "lineage": {}, "dropped": 0, "drifted": 0, "newer": 26, "total": 26, "updated": 0 } }, "metrics": { "sink1": { "format": "table", "stages": [ { "stage": 4, "partitionTimes": [ 211 ], "recordsWritten": 0, "lastUpdateTime": "2022-04-05 07:24:59.106", "bytesWritten": 0, "recordsRead": 84, "bytesRead": 15577, "partitionStatus": "Success", "streams": { "source3": { "count": 84, "cached": false, "totalPartitions": 1, "partitionStatus": "Success", "partitionCounts": [ 84 ], "type": "source" } }, "target": "sink1", "time": 293, "progressState": "Completed" }, { "stage": 3, "partitionTimes": [ 470 ], "recordsWritten": 0, "lastUpdateTime": "2022-04-05 07:24:59.335", "bytesWritten": 0, "recordsRead": 16867, "bytesRead": 1592104, "partitionStatus": "Success", "streams": { "source1": { "count": 16867, "cached": false, "totalPartitions": 1, "partitionStatus": "Success", "partitionCounts": [ 16867 ], "type": "source" } }, "target": "sink1", "time": 573, "progressState": "Completed" }, { "stage": 5, "partitionTimes": [ 1408, 1412, 1415, 1408 ], "recordsWritten": 16867, "lastUpdateTime": "2022-04-05 07:25:02.054", "bytesWritten": 0, "recordsRead": 16867, "bytesRead": 17752068, "partitionStatus": "Success", "streams": { "source2": { "count": 16867, "cached": false, "totalPartitions": 4, "partitionStatus": "Success", "partitionCounts": [ 5209, 5249, 5212, 1197 ], "type": "source" } }, "target": "sink1", "time": 2358, "progressState": "Completed" }, { "stage": 6, "partitionTimes": [ 1823, 1858, 1838, 1824, 440, 432, 429, 433, 519, 470, 602, 385, 384, 388, 350, 385, 385, 821, 363, 395, 375, 386, 346, 384, 387, 368, 511, 364, 478, 509, 366, 340, 342, 375, 353, 395, 339, 338, 366, 383, 495, 381, 370, 341, 350, 342, 345, 342, 345, 323, 531, 342, 615, 369, 330, 341, 425, 341, 368, 366, 402, 550, 344, 323, 318, 322, 578, 333, 401, 310, 353, 399, 345, 382, 360, 331, 513, 434, 500, 502, 807, 347, 361, 603, 457, 350, 537, 344, 323, 300, 326, 313, 290, 313, 531, 311, 308, 382, 325, 290, 315, 309, 294, 290, 387, 344, 292, 499, 362, 345, 363, 352, 327, 299, 344, 330, 413, 323, 304, 293, 283, 320, 1104, 976, 938, 1050, 307, 366, 304, 289, 322, 345, 349, 325, 300, 305, 308, 310, 731, 342, 401, 305, 280, 352, 322, 450, 600, 398, 615, 301, 290, 294, 324, 307, 334, 322, 334, 285, 307, 318, 540, 296, 325, 298, 305, 343, 338, 330, 423, 329, 316, 306, 307, 322, 319, 341, 311, 307, 300, 332, 521, 287, 305, 326, 310, 309, 319, 312, 544, 523, 583, 588, 349, 305, 309, 292, 348, 339, 344, 349 ], "recordsWritten": 16833, "lastUpdateTime": "2022-04-05 07:25:24.416", "bytesWritten": 1883466, "recordsRead": 16867, "bytesRead": 0, "partitionStatus": "Success", "streams": { "DerivedColumn1": { "count": 16833, "cached": false, "totalPartitions": 200, "partitionStatus": "Success", "partitionCounts": [ 86, 76, 93, 83, 85, 91, 97, 84, 74, 96, 91, 74, 85, 92, 88, 91, 94, 79, 95, 75, 106, 95, 82, 80, 92, 85, 89, 87, 90, 79, 82, 92, 80, 94, 80, 90, 99, 85, 80, 94, 86, 101, 98, 82, 92, 81, 72, 87, 75, 88, 92, 83, 74, 91, 87, 84, 67, 84, 87, 95, 89, 77, 82, 88, 70, 93, 77, 91, 98, 88, 75, 94, 100, 86, 75, 89, 74, 71, 86, 83, 87, 78, 85, 101, 98, 79, 66, 84, 92, 74, 89, 92, 85, 69, 113, 77, 80, 72, 73, 85, 82, 77, 90, 78, 82, 73, 97, 79, 85, 79, 87, 90, 82, 74, 77, 83, 90, 75, 93, 73, 78, 98, 84, 85, 101, 81, 69, 83, 78, 89, 90, 75, 69, 66, 67, 77, 83, 86, 68, 82, 99, 95, 71, 98, 78, 82, 105, 81, 89, 82, 72, 89, 73, 85, 84, 72, 78, 91, 63, 76, 97, 107, 77, 66, 75, 75, 79, 104, 76, 87, 89, 71, 100, 67, 73, 100, 73, 75, 91, 80, 89, 97, 83, 73, 83, 70, 80, 94, 74, 87, 78, 91, 78, 84, 86, 96, 83, 97, 91, 78 ], "type": "derive" }, "Join1": { "count": 16867, "cached": false, "totalPartitions": 200, "partitionStatus": "Success", "partitionCounts": [ 86, 76, 93, 83, 85, 91, 97, 85, 74, 96, 91, 74, 85, 92, 89, 91, 94, 79, 95, 75, 106, 95, 82, 80, 94, 85, 89, 87, 90, 79, 82, 92, 82, 94, 80, 90, 99, 85, 80, 95, 86, 101, 98, 82, 92, 81, 72, 87, 76, 88, 92, 83, 74, 91, 87, 84, 67, 85, 87, 96, 89, 77, 82, 88, 71, 93, 77, 91, 100, 89, 75, 94, 100, 86, 75, 89, 74, 72, 86, 83, 88, 78, 85, 101, 98, 79, 66, 84, 92, 76, 90, 92, 85, 69, 113, 78, 80, 72, 73, 85, 82, 78, 90, 78, 82, 74, 97, 79, 85, 79, 87, 91, 82, 74, 77, 83, 91, 75, 93, 73, 78, 98, 84, 86, 101, 81, 69, 83, 78, 89, 90, 75, 69, 66, 67, 77, 83, 86, 68, 82, 99, 95, 71, 98, 78, 82, 105, 81, 89, 82, 72, 89, 73, 85, 84, 72, 78, 91, 63, 76, 97, 107, 77, 66, 75, 75, 81, 104, 76, 88, 90, 71, 100, 67, 73, 100, 75, 75, 91, 80, 90, 97, 83, 73, 83, 70, 80, 94, 74, 87, 78, 92, 78, 84, 86, 97, 83, 97, 91, 78 ], "type": "join" }, "Join2": { "count": 16833, "cached": false, "totalPartitions": 200, "partitionStatus": "Success", "partitionCounts": [ 86, 76, 93, 83, 85, 91, 97, 84, 74, 96, 91, 74, 85, 92, 88, 91, 94, 79, 95, 75, 106, 95, 82, 80, 92, 85, 89, 87, 90, 79, 82, 92, 80, 94, 80, 90, 99, 85, 80, 94, 86, 101, 98, 82, 92, 81, 72, 87, 75, 88, 92, 83, 74, 91, 87, 84, 67, 84, 87, 95, 89, 77, 82, 88, 70, 93, 77, 91, 98, 88, 75, 94, 100, 86, 75, 89, 74, 71, 86, 83, 87, 78, 85, 101, 98, 79, 66, 84, 92, 74, 89, 92, 85, 69, 113, 77, 80, 72, 73, 85, 82, 77, 90, 78, 82, 73, 97, 79, 85, 79, 87, 90, 82, 74, 77, 83, 90, 75, 93, 73, 78, 98, 84, 85, 101, 81, 69, 83, 78, 89, 90, 75, 69, 66, 67, 77, 83, 86, 68, 82, 99, 95, 71, 98, 78, 82, 105, 81, 89, 82, 72, 89, 73, 85, 84, 72, 78, 91, 63, 76, 97, 107, 77, 66, 75, 75, 79, 104, 76, 87, 89, 71, 100, 67, 73, 100, 73, 75, 91, 80, 89, 97, 83, 73, 83, 70, 80, 94, 74, 87, 78, 91, 78, 84, 86, 96, 83, 97, 91, 78 ], "type": "join" }, "sink1": { "count": 16833, "cached": false, "totalPartitions": 200, "partitionStatus": "Success", "partitionCounts": [ 86, 76, 93, 83, 85, 91, 97, 84, 74, 96, 91, 74, 85, 92, 88, 91, 94, 79, 95, 75, 106, 95, 82, 80, 92, 85, 89, 87, 90, 79, 82, 92, 80, 94, 80, 90, 99, 85, 80, 94, 86, 101, 98, 82, 92, 81, 72, 87, 75, 88, 92, 83, 74, 91, 87, 84, 67, 84, 87, 95, 89, 77, 82, 88, 70, 93, 77, 91, 98, 88, 75, 94, 100, 86, 75, 89, 74, 71, 86, 83, 87, 78, 85, 101, 98, 79, 66, 84, 92, 74, 89, 92, 85, 69, 113, 77, 80, 72, 73, 85, 82, 77, 90, 78, 82, 73, 97, 79, 85, 79, 87, 90, 82, 74, 77, 83, 90, 75, 93, 73, 78, 98, 84, 85, 101, 81, 69, 83, 78, 89, 90, 75, 69, 66, 67, 77, 83, 86, 68, 82, 99, 95, 71, 98, 78, 82, 105, 81, 89, 82, 72, 89, 73, 85, 84, 72, 78, 91, 63, 76, 97, 107, 77, 66, 75, 75, 79, 104, 76, 87, 89, 71, 100, 67, 73, 100, 73, 75, 91, 80, 89, 97, 83, 73, 83, 70, 80, 94, 74, 87, 78, 91, 78, 84, 86, 96, 83, 97, 91, 78 ], "type": "sink" }, "Aggregate1": { "count": 16867, "cached": false, "totalPartitions": 200, "partitionStatus": "Success", "partitionCounts": [ 86, 76, 93, 83, 85, 91, 97, 85, 74, 96, 91, 74, 85, 92, 89, 91, 94, 79, 95, 75, 106, 95, 82, 80, 94, 85, 89, 87, 90, 79, 82, 92, 82, 94, 80, 90, 99, 85, 80, 95, 86, 101, 98, 82, 92, 81, 72, 87, 76, 88, 92, 83, 74, 91, 87, 84, 67, 85, 87, 96, 89, 77, 82, 88, 71, 93, 77, 91, 100, 89, 75, 94, 100, 86, 75, 89, 74, 72, 86, 83, 88, 78, 85, 101, 98, 79, 66, 84, 92, 76, 90, 92, 85, 69, 113, 78, 80, 72, 73, 85, 82, 78, 90, 78, 82, 74, 97, 79, 85, 79, 87, 91, 82, 74, 77, 83, 91, 75, 93, 73, 78, 98, 84, 86, 101, 81, 69, 83, 78, 89, 90, 75, 69, 66, 67, 77, 83, 86, 68, 82, 99, 95, 71, 98, 78, 82, 105, 81, 89, 82, 72, 89, 73, 85, 84, 72, 78, 91, 63, 76, 97, 107, 77, 66, 75, 75, 81, 104, 76, 88, 90, 71, 100, 67, 73, 100, 75, 75, 91, 80, 90, 97, 83, 73, 83, 70, 80, 94, 74, 87, 78, 92, 78, 84, 86, 97, 83, 97, 91, 78 ], "type": "aggregate" } }, "target": "sink1", "time": 22342, "progressState": "Completed" } ], "sinkPostProcessingTime": 0, "store": "sqldw", "rowsWritten": 16833, "details": { "postSQLDuration": [ 13 ], "preCommandsDuration": [ 1 ], "tableOperationSQLDuration": [ 1198 ], "writeStageDuration": [ 28 ], "preSQLDuration": [ 1 ], "tempTable": [ "dbo.T_6828_d171255578da411d9c303dfeb940329c" ], "postCommandsDuration": [ 2 ] }, "progressState": "Completed", "sources": { "source1": { "rowsRead": 16867, "store": "blob", "details": { "fileSystemInitDuration": [ 10613 ], "moveFilesDuration": [ 96 ], "pathResolutionDuration": [ 71 ], "fileLoadDuration": [ 10485 ] }, "format": "delimited" }, "source3": { "rowsRead": 84, "store": "blob", "details": { "fileSystemInitDuration": [ 331 ], "moveFilesDuration": [ 93 ], "pathResolutionDuration": [ 17 ], "fileLoadDuration": [ 310 ] }, "format": "delimited" }, "source2": { "rowsRead": 16867, "store": "blob", "details": { "fileSystemInitDuration": [ 713 ], "moveFilesDuration": [ 119 ], "pathResolutionDuration": [ 38 ], "fileLoadDuration": [ 666 ] }, "format": "delimited" } }, "sinkProcessingTime": 49338 } }, "dsl": "\nsource() ~> source3\n\nsource() ~> source1\n\nsource() ~> source2\n\nsource2 aggregate() ~> Aggregate1\n\nsource1, Aggregate1 join() ~> Join1\n\nJoin1, source3 join() ~> Join2\n\nJoin2 derive() ~> DerivedColumn1\n\nDerivedColumn1 sink() ~> sink1" }, "effectiveIntegrationRuntime": "AutoResolveIntegrationRuntime (East US)", "billingReference": { "activityType": "executedataflow", "billableDuration": [ { "meterType": "General", "duration": 0.5027138406666667, "unit": "coreHour", "sessionType": "JobCluster" } ] }, "reportLineageToPurview": { "status": "NotReported" } }