{
    "name": "csv_to_delta",
    "properties": {
        "type": "MappingDataFlow",
        "typeProperties": {
            "sources": [
                {
                    "dataset": {
                        "referenceName": "etl_02_csv",
                        "type": "DatasetReference"
                    },
                    "name": "CSV",
                    "description": "Import CSV files from Blob Storage"
                }
            ],
            "sinks": [
                {
                    "linkedService": {
                        "referenceName": "ETL NOAA ADLS2 Storage",
                        "type": "LinkedServiceReference"
                    },
                    "name": "DeltaLake",
                    "description": "Write to Delta table"
                }
            ],
            "transformations": [
                {
                    "name": "SetUpsertPolicy",
                    "description": "Upsert all rows"
                }
            ],
            "script": "source(output(\n\t\tSTATION as string,\n\t\tDATE as timestamp 'yyyy-MM-dd\\'T\\'HH:mm:ss',\n\t\tSOURCE as string,\n\t\tLATITUDE as double,\n\t\tLONGITUDE as double,\n\t\tELEVATION as double,\n\t\tNAME as string,\n\t\tREPORT_TYPE as string,\n\t\tCALL_SIGN as string,\n\t\tQUALITY_CONTROL as string,\n\t\tWND as string,\n\t\tCIG as string,\n\t\tVIS as string,\n\t\tTMP as string,\n\t\tDEW as string,\n\t\tSLP as string,\n\t\tAA1 as string,\n\t\tAA2 as string,\n\t\tAA3 as string,\n\t\tAA4 as string,\n\t\tAB1 as string,\n\t\tAC1 as string,\n\t\tAD1 as string,\n\t\tAE1 as string,\n\t\tAG1 as string,\n\t\tAH1 as string,\n\t\tAH2 as string,\n\t\tAH3 as string,\n\t\tAH4 as string,\n\t\tAH5 as string,\n\t\tAH6 as string,\n\t\tAI1 as string,\n\t\tAI2 as string,\n\t\tAI3 as string,\n\t\tAI4 as string,\n\t\tAI5 as string,\n\t\tAI6 as string,\n\t\tAJ1 as string,\n\t\tAK1 as string,\n\t\tAL1 as string,\n\t\tAL2 as string,\n\t\tAL3 as string,\n\t\tAL4 as string,\n\t\tAM1 as string,\n\t\tAN1 as string,\n\t\tAO1 as string,\n\t\tAO2 as string,\n\t\tAO3 as string,\n\t\tAO4 as string,\n\t\tAP1 as string,\n\t\tAP2 as string,\n\t\tAP3 as string,\n\t\tAP4 as string,\n\t\tAT1 as string,\n\t\tAT2 as string,\n\t\tAT3 as string,\n\t\tAT4 as string,\n\t\tAT5 as string,\n\t\tAT6 as string,\n\t\tAT7 as string,\n\t\tAT8 as string,\n\t\tAU1 as string,\n\t\tAU2 as string,\n\t\tAU3 as string,\n\t\tAU4 as string,\n\t\tAU5 as string,\n\t\tAU6 as string,\n\t\tAU7 as string,\n\t\tAU8 as string,\n\t\tAU9 as string,\n\t\tAW1 as string,\n\t\tAW2 as string,\n\t\tAW3 as string,\n\t\tAW4 as string,\n\t\tAX1 as string,\n\t\tAX2 as string,\n\t\tAX3 as string,\n\t\tAX4 as string,\n\t\tAX5 as string,\n\t\tAX6 as string,\n\t\tAY1 as string,\n\t\tAY2 as string,\n\t\tAZ1 as string,\n\t\tAZ2 as string,\n\t\tCB1 as string,\n\t\tCB2 as string,\n\t\tCF1 as string,\n\t\tCF2 as string,\n\t\tCF3 as string,\n\t\tCG1 as string,\n\t\tCG2 as string,\n\t\tCG3 as string,\n\t\tCH1 as string,\n\t\tCH2 as string,\n\t\tCI1 as string,\n\t\tCN1 as string,\n\t\tCN2 as string,\n\t\tCN3 as string,\n\t\tCN4 as string,\n\t\tCO1 as string,\n\t\tCO2 as string,\n\t\tCO3 as string,\n\t\tCO4 as string,\n\t\tCO5 as string,\n\t\tCO6 as string,\n\t\tCO7 as string,\n\t\tCO8 as string,\n\t\tCO9 as string,\n\t\tCR1 as string,\n\t\tCT1 as string,\n\t\tCT2 as string,\n\t\tCT3 as string,\n\t\tCU1 as string,\n\t\tCU2 as string,\n\t\tCU3 as string,\n\t\tCV1 as string,\n\t\tCV2 as string,\n\t\tCV3 as string,\n\t\tCW1 as string,\n\t\tCX1 as string,\n\t\tCX2 as string,\n\t\tCX3 as string,\n\t\tED1 as string,\n\t\tGA1 as string,\n\t\tGA2 as string,\n\t\tGA3 as string,\n\t\tGA4 as string,\n\t\tGA5 as string,\n\t\tGA6 as string,\n\t\tGD1 as string,\n\t\tGD2 as string,\n\t\tGD3 as string,\n\t\tGD4 as string,\n\t\tGD5 as string,\n\t\tGD6 as string,\n\t\tGE1 as string,\n\t\tGF1 as string,\n\t\tGG1 as string,\n\t\tGG2 as string,\n\t\tGG3 as string,\n\t\tGG4 as string,\n\t\tGG5 as string,\n\t\tGG6 as string,\n\t\tGH1 as string,\n\t\tGJ1 as string,\n\t\tGK1 as string,\n\t\tGL1 as string,\n\t\tGM1 as string,\n\t\tGN1 as string,\n\t\tGO1 as string,\n\t\tGP1 as string,\n\t\tGQ1 as string,\n\t\tGR1 as string,\n\t\tHA1 as string,\n\t\tIA1 as string,\n\t\tIA2 as string,\n\t\tIB1 as string,\n\t\tIB2 as string,\n\t\tIC1 as string,\n\t\tKA1 as string,\n\t\tKA2 as string,\n\t\tKA3 as string,\n\t\tKA4 as string,\n\t\tKB1 as string,\n\t\tKB2 as string,\n\t\tKB3 as string,\n\t\tKC1 as string,\n\t\tKC2 as string,\n\t\tKD1 as string,\n\t\tKD2 as string,\n\t\tKE1 as string,\n\t\tKF1 as string,\n\t\tKG1 as string,\n\t\tKG2 as string,\n\t\tMA1 as string,\n\t\tMD1 as string,\n\t\tME1 as string,\n\t\tMF1 as string,\n\t\tMG1 as string,\n\t\tMH1 as string,\n\t\tMK1 as string,\n\t\tMV1 as string,\n\t\tMV2 as string,\n\t\tMV3 as string,\n\t\tMV4 as string,\n\t\tMV5 as string,\n\t\tMV6 as string,\n\t\tMV7 as string,\n\t\tMW1 as string,\n\t\tMW2 as string,\n\t\tMW3 as string,\n\t\tMW4 as string,\n\t\tMW5 as string,\n\t\tMW6 as string,\n\t\tMW7 as string,\n\t\tOA1 as string,\n\t\tOA2 as string,\n\t\tOA3 as string,\n\t\tOB1 as string,\n\t\tOB2 as string,\n\t\tOC1 as string,\n\t\tOD1 as string,\n\t\tOD2 as string,\n\t\tOD3 as string,\n\t\tRH1 as string,\n\t\tRH2 as string,\n\t\tRH3 as string,\n\t\tSA1 as string,\n\t\tST1 as string,\n\t\tUA1 as string,\n\t\tUG1 as string,\n\t\tUG2 as string,\n\t\tWA1 as string,\n\t\tWD1 as string,\n\t\tWG1 as string,\n\t\tWJ1 as string,\n\t\tREM as string,\n\t\tEQD as string,\n\t\tQNN as string\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: true,\n\tpurgeFiles: true,\n\twildcardPaths:['*.csv']) ~> CSV\nCSV alterRow(upsertIf(true())) ~> SetUpsertPolicy\nSetUpsertPolicy sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tinput(\n\t\tSTATION as string,\n\t\tDATE as timestamp,\n\t\tSOURCE as string,\n\t\tLATITUDE as double,\n\t\tLONGITUDE as double,\n\t\tELEVATION as double,\n\t\tNAME as string,\n\t\tREPORT_TYPE as string,\n\t\tCALL_SIGN as string,\n\t\tQUALITY_CONTROL as string,\n\t\tWND as string,\n\t\tCIG as string,\n\t\tVIS as string,\n\t\tTMP as string,\n\t\tDEW as string,\n\t\tSLP as string,\n\t\tAA1 as string,\n\t\tAA2 as string,\n\t\tAA3 as string,\n\t\tAA4 as string,\n\t\tAB1 as string,\n\t\tAC1 as string,\n\t\tAD1 as string,\n\t\tAE1 as string,\n\t\tAG1 as string,\n\t\tAH1 as string,\n\t\tAH2 as string,\n\t\tAH3 as string,\n\t\tAH4 as string,\n\t\tAH5 as string,\n\t\tAH6 as string,\n\t\tAI1 as string,\n\t\tAI2 as string,\n\t\tAI3 as string,\n\t\tAI4 as string,\n\t\tAI5 as string,\n\t\tAI6 as string,\n\t\tAJ1 as string,\n\t\tAK1 as string,\n\t\tAL1 as string,\n\t\tAL2 as string,\n\t\tAL3 as string,\n\t\tAL4 as string,\n\t\tAM1 as string,\n\t\tAN1 as string,\n\t\tAO1 as string,\n\t\tAO2 as string,\n\t\tAO3 as string,\n\t\tAO4 as string,\n\t\tAP1 as string,\n\t\tAP2 as string,\n\t\tAP3 as string,\n\t\tAP4 as string,\n\t\tAT1 as string,\n\t\tAT2 as string,\n\t\tAT3 as string,\n\t\tAT4 as string,\n\t\tAT5 as string,\n\t\tAT6 as string,\n\t\tAT7 as string,\n\t\tAT8 as string,\n\t\tAU1 as string,\n\t\tAU2 as string,\n\t\tAU3 as string,\n\t\tAU4 as string,\n\t\tAU5 as string,\n\t\tAU6 as string,\n\t\tAU7 as string,\n\t\tAU8 as string,\n\t\tAU9 as string,\n\t\tAW1 as string,\n\t\tAW2 as string,\n\t\tAW3 as string,\n\t\tAW4 as string,\n\t\tAX1 as string,\n\t\tAX2 as string,\n\t\tAX3 as string,\n\t\tAX4 as string,\n\t\tAX5 as string,\n\t\tAX6 as string,\n\t\tAY1 as string,\n\t\tAY2 as string,\n\t\tAZ1 as string,\n\t\tAZ2 as string,\n\t\tCB1 as string,\n\t\tCB2 as string,\n\t\tCF1 as string,\n\t\tCF2 as string,\n\t\tCF3 as string,\n\t\tCG1 as string,\n\t\tCG2 as string,\n\t\tCG3 as string,\n\t\tCH1 as string,\n\t\tCH2 as string,\n\t\tCI1 as string,\n\t\tCN1 as string,\n\t\tCN2 as string,\n\t\tCN3 as string,\n\t\tCN4 as string,\n\t\tCO1 as string,\n\t\tCO2 as string,\n\t\tCO3 as string,\n\t\tCO4 as string,\n\t\tCO5 as string,\n\t\tCO6 as string,\n\t\tCO7 as string,\n\t\tCO8 as string,\n\t\tCO9 as string,\n\t\tCR1 as string,\n\t\tCT1 as string,\n\t\tCT2 as string,\n\t\tCT3 as string,\n\t\tCU1 as string,\n\t\tCU2 as string,\n\t\tCU3 as string,\n\t\tCV1 as string,\n\t\tCV2 as string,\n\t\tCV3 as string,\n\t\tCW1 as string,\n\t\tCX1 as string,\n\t\tCX2 as string,\n\t\tCX3 as string,\n\t\tED1 as string,\n\t\tGA1 as string,\n\t\tGA2 as string,\n\t\tGA3 as string,\n\t\tGA4 as string,\n\t\tGA5 as string,\n\t\tGA6 as string,\n\t\tGD1 as string,\n\t\tGD2 as string,\n\t\tGD3 as string,\n\t\tGD4 as string,\n\t\tGD5 as string,\n\t\tGD6 as string,\n\t\tGE1 as string,\n\t\tGF1 as string,\n\t\tGG1 as string,\n\t\tGG2 as string,\n\t\tGG3 as string,\n\t\tGG4 as string,\n\t\tGG5 as string,\n\t\tGG6 as string,\n\t\tGH1 as string,\n\t\tGJ1 as string,\n\t\tGK1 as string,\n\t\tGL1 as string,\n\t\tGM1 as string,\n\t\tGN1 as string,\n\t\tGO1 as string,\n\t\tGP1 as string,\n\t\tGQ1 as string,\n\t\tGR1 as string,\n\t\tHA1 as string,\n\t\tIA1 as string,\n\t\tIA2 as string,\n\t\tIB1 as string,\n\t\tIB2 as string,\n\t\tIC1 as string,\n\t\tKA1 as string,\n\t\tKA2 as string,\n\t\tKA3 as string,\n\t\tKA4 as string,\n\t\tKB1 as string,\n\t\tKB2 as string,\n\t\tKB3 as string,\n\t\tKC1 as string,\n\t\tKC2 as string,\n\t\tKD1 as string,\n\t\tKD2 as string,\n\t\tKE1 as string,\n\t\tKF1 as string,\n\t\tKG1 as string,\n\t\tKG2 as string,\n\t\tMA1 as string,\n\t\tMD1 as string,\n\t\tME1 as string,\n\t\tMF1 as string,\n\t\tMG1 as string,\n\t\tMH1 as string,\n\t\tMK1 as string,\n\t\tMV1 as string,\n\t\tMV2 as string,\n\t\tMV3 as string,\n\t\tMV4 as string,\n\t\tMV5 as string,\n\t\tMV6 as string,\n\t\tMV7 as string,\n\t\tMW1 as string,\n\t\tMW2 as string,\n\t\tMW3 as string,\n\t\tMW4 as string,\n\t\tMW5 as string,\n\t\tMW6 as string,\n\t\tMW7 as string,\n\t\tOA1 as string,\n\t\tOA2 as string,\n\t\tOA3 as string,\n\t\tOB1 as string,\n\t\tOB2 as string,\n\t\tOC1 as string,\n\t\tOD1 as string,\n\t\tOD2 as string,\n\t\tOD3 as string,\n\t\tRH1 as string,\n\t\tRH2 as string,\n\t\tRH3 as string,\n\t\tSA1 as string,\n\t\tST1 as string,\n\t\tUA1 as string,\n\t\tUG1 as string,\n\t\tUG2 as string,\n\t\tWA1 as string,\n\t\tWD1 as string,\n\t\tWG1 as string,\n\t\tWJ1 as string,\n\t\tREM as string,\n\t\tEQD as string,\n\t\tQNN as string\n\t),\n\tformat: 'delta',\n\tcompressionType: 'snappy',\n\tcompressionLevel: 'Fastest',\n\tfileSystem: 'etl-03-delta',\n\ttruncate: false,\n\tvacuum: 0,\n\tdeletable:false,\n\tinsertable:false,\n\tupdateable:false,\n\tupsertable:true,\n\tkeys:['STATION','DATE','LATITUDE','LONGITUDE','SOURCE','REPORT_TYPE'],\n\tmapColumn(\n\t\tSTATION,\n\t\tDATE,\n\t\tSOURCE,\n\t\tLATITUDE,\n\t\tLONGITUDE,\n\t\tELEVATION,\n\t\tNAME,\n\t\tREPORT_TYPE,\n\t\tCALL_SIGN,\n\t\tQUALITY_CONTROL,\n\t\tWND,\n\t\tCIG,\n\t\tVIS,\n\t\tTMP,\n\t\tDEW,\n\t\tSLP,\n\t\tAA1,\n\t\tAA2,\n\t\tAA3,\n\t\tAA4,\n\t\tAB1,\n\t\tAC1,\n\t\tAD1,\n\t\tAE1,\n\t\tAG1,\n\t\tAH1,\n\t\tAH2,\n\t\tAH3,\n\t\tAH4,\n\t\tAH5,\n\t\tAH6,\n\t\tAI1,\n\t\tAI2,\n\t\tAI3,\n\t\tAI4,\n\t\tAI5,\n\t\tAI6,\n\t\tAJ1,\n\t\tAK1,\n\t\tAL1,\n\t\tAL2,\n\t\tAL3,\n\t\tAL4,\n\t\tAM1,\n\t\tAN1,\n\t\tAO1,\n\t\tAO2,\n\t\tAO3,\n\t\tAO4,\n\t\tAP1,\n\t\tAP2,\n\t\tAP3,\n\t\tAP4,\n\t\tAT1,\n\t\tAT2,\n\t\tAT3,\n\t\tAT4,\n\t\tAT5,\n\t\tAT6,\n\t\tAT7,\n\t\tAT8,\n\t\tAU1,\n\t\tAU2,\n\t\tAU3,\n\t\tAU4,\n\t\tAU5,\n\t\tAU6,\n\t\tAU7,\n\t\tAU8,\n\t\tAU9,\n\t\tAW1,\n\t\tAW2,\n\t\tAW3,\n\t\tAW4,\n\t\tAX1,\n\t\tAX2,\n\t\tAX3,\n\t\tAX4,\n\t\tAX5,\n\t\tAX6,\n\t\tAY1,\n\t\tAY2,\n\t\tAZ1,\n\t\tAZ2,\n\t\tCB1,\n\t\tCB2,\n\t\tCF1,\n\t\tCF2,\n\t\tCF3,\n\t\tCG1,\n\t\tCG2,\n\t\tCG3,\n\t\tCH1,\n\t\tCH2,\n\t\tCI1,\n\t\tCN1,\n\t\tCN2,\n\t\tCN3,\n\t\tCN4,\n\t\tCO1,\n\t\tCO2,\n\t\tCO3,\n\t\tCO4,\n\t\tCO5,\n\t\tCO6,\n\t\tCO7,\n\t\tCO8,\n\t\tCO9,\n\t\tCR1,\n\t\tCT1,\n\t\tCT2,\n\t\tCT3,\n\t\tCU1,\n\t\tCU2,\n\t\tCU3,\n\t\tCV1,\n\t\tCV2,\n\t\tCV3,\n\t\tCW1,\n\t\tCX1,\n\t\tCX2,\n\t\tCX3,\n\t\tED1,\n\t\tGA1,\n\t\tGA2,\n\t\tGA3,\n\t\tGA4,\n\t\tGA5,\n\t\tGA6,\n\t\tGD1,\n\t\tGD2,\n\t\tGD3,\n\t\tGD4,\n\t\tGD5,\n\t\tGD6,\n\t\tGE1,\n\t\tGF1,\n\t\tGG1,\n\t\tGG2,\n\t\tGG3,\n\t\tGG4,\n\t\tGG5,\n\t\tGG6,\n\t\tGH1,\n\t\tGJ1,\n\t\tGK1,\n\t\tGL1,\n\t\tGM1,\n\t\tGN1,\n\t\tGO1,\n\t\tGP1,\n\t\tGQ1,\n\t\tGR1,\n\t\tHA1,\n\t\tIA1,\n\t\tIA2,\n\t\tIB1,\n\t\tIB2,\n\t\tIC1,\n\t\tKA1,\n\t\tKA2,\n\t\tKA3,\n\t\tKA4,\n\t\tKB1,\n\t\tKB2,\n\t\tKB3,\n\t\tKC1,\n\t\tKC2,\n\t\tKD1,\n\t\tKD2,\n\t\tKE1,\n\t\tKF1,\n\t\tKG1,\n\t\tKG2,\n\t\tMA1,\n\t\tMD1,\n\t\tME1,\n\t\tMF1,\n\t\tMG1,\n\t\tMH1,\n\t\tMK1,\n\t\tMV1,\n\t\tMV2,\n\t\tMV3,\n\t\tMV4,\n\t\tMV5,\n\t\tMV6,\n\t\tMV7,\n\t\tMW1,\n\t\tMW2,\n\t\tMW3,\n\t\tMW4,\n\t\tMW5,\n\t\tMW6,\n\t\tMW7,\n\t\tOA1,\n\t\tOA2,\n\t\tOA3,\n\t\tOB1,\n\t\tOB2,\n\t\tOC1,\n\t\tOD1,\n\t\tOD2,\n\t\tOD3,\n\t\tRH1,\n\t\tRH2,\n\t\tRH3,\n\t\tSA1,\n\t\tST1,\n\t\tUA1,\n\t\tUG1,\n\t\tUG2,\n\t\tWA1,\n\t\tWD1,\n\t\tWG1,\n\t\tWJ1,\n\t\tREM,\n\t\tEQD,\n\t\tQNN\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> DeltaLake"
        }
    }
}