{
    "name": "HTTP fetch and iterate",
    "properties": {
        "description": "https://docs.microsoft.com/en-us/answers/questions/648990/download-all-files-in-a-folder-from-a-http-source.html?childToView=649708#answer-649708",
        "activities": [
            {
                "name": "Get base url content",
                "description": "fetch the http content of page.",
                "type": "WebActivity",
                "dependsOn": [],
                "policy": {
                    "timeout": "7.00:00:00",
                    "retry": 0,
                    "retryIntervalInSeconds": 30,
                    "secureOutput": false,
                    "secureInput": false
                },
                "userProperties": [],
                "typeProperties": {
                    "url": "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/",
                    "method": "GET",
                    "headers": {
                        "Accept": "application/JSON"
                    }
                }
            },
            {
                "name": "Separate via anchor tag",
                "description": "@split(activity('Get base url content').output.Response,'<a href=')",
                "type": "SetVariable",
                "dependsOn": [
                    {
                        "activity": "Get base url content",
                        "dependencyConditions": [
                            "Succeeded"
                        ]
                    }
                ],
                "userProperties": [],
                "typeProperties": {
                    "variableName": "raw",
                    "value": {
                        "value": "@split(activity('Get base url content').output.Response,'<a href=\"')",
                        "type": "Expression"
                    }
                }
            },
            {
                "name": "remove header entry",
                "type": "SetVariable",
                "dependsOn": [
                    {
                        "activity": "Separate via anchor tag",
                        "dependencyConditions": [
                            "Succeeded"
                        ]
                    }
                ],
                "userProperties": [],
                "typeProperties": {
                    "variableName": "dirty entries",
                    "value": {
                        "value": "@skip(variables('raw'),2)",
                        "type": "Expression"
                    }
                }
            },
            {
                "name": "Iterate over entries",
                "description": "Clean up and then copy each url",
                "type": "ForEach",
                "dependsOn": [
                    {
                        "activity": "remove header entry",
                        "dependencyConditions": [
                            "Succeeded"
                        ]
                    }
                ],
                "userProperties": [],
                "typeProperties": {
                    "items": {
                        "value": "@variables('dirty entries')",
                        "type": "Expression"
                    },
                    "activities": [
                        {
                            "name": "Demonstrate cleanup",
                            "type": "SetVariable",
                            "dependsOn": [],
                            "userProperties": [],
                            "typeProperties": {
                                "variableName": "clean entry",
                                "value": {
                                    "value": "@substring(\nitem(),\n0,\nindexof(item(),'\">')\n)",
                                    "type": "Expression"
                                }
                            }
                        }
                    ]
                }
            }
        ],
        "variables": {
            "raw": {
                "type": "Array"
            },
            "dirty entries": {
                "type": "Array"
            },
            "clean entry": {
                "type": "String"
            }
        },
        "annotations": []
    }
}