using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Microsoft.Azure.Storage; using Microsoft.Azure.Storage.Blob; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Host; using Microsoft.Extensions.Logging; using Newtonsoft.Json; namespace func_UpsertJSONCon { public class Function1 { [FunctionName("FuncBlobTriggerJIRA")] public async Task Run([BlobTrigger("jiradata/Support_Data/Daily_Updated_Data_ALLProjects/{Daily_Updated_Issues_With_ChangeLog_All_Projects}", Connection = "AzureWebJobsStorage")] Stream myBlob, ILogger log) { log.LogInformation($"C# Blob trigger function Processed blob\n \n Size: {myBlob.Length} Bytes"); try { var serializer = new JsonSerializer(); List updatedFileobjects = new List(); using (var sr = new StreamReader(myBlob)) { while (!sr.EndOfStream) { var jsonTextReader = await sr.ReadLineAsync(); dynamic data = JsonConvert.DeserializeObject(jsonTextReader); updatedFileobjects.Add(data); } } log.LogInformation($"Read updatedFileobjects {updatedFileobjects.Count}"); int checkCount = 0; if (updatedFileobjects.Count == 1) { if (updatedFileobjects[0].issues.Count > 0) { checkCount = 1; } } else if (updatedFileobjects.Count > 1) { checkCount = 1; } log.LogInformation($"Read Check Count {checkCount}"); if (checkCount == 1) { List oldFileobjects = new List(); string connectionString = GetEnvironmentVariable("AzureWebJobsStorage"); string rawJsonFileContainerName = GetEnvironmentVariable("RawJsonFileContainerName"); string rawDataFile = GetEnvironmentVariable("Raw_Data_File"); log.LogInformation($"Read OldFileObjects Conection Values {connectionString}"); log.LogInformation($"Read rawJsonFileContainerName Values {rawJsonFileContainerName}"); log.LogInformation($"Read rawDataFile Values {rawDataFile}"); BlobServiceClient blobServiceClient = new BlobServiceClient("DefaultEndpointsProtocol=https;AccountName=blobncus;AccountKey=aUq/brYjF5WKNSIz77/9N/Lw0nvXiGcy+h2W4/BdSyeuKSx9kqCjezbSu3Vf3vygVN0guNdWYEfS+AStJB48/Q==;BlobEndpoint=https://blobncus.blob.core.windows.net/;TableEndpoint=https://blobncus.table.core.windows.net/;QueueEndpoint=https://blobncus.queue.core.windows.net/;FileEndpoint=https://blobncus.file.core.windows.net/"); BlobContainerClient rawcontainerClient = blobServiceClient.GetBlobContainerClient("jiradata"); BlobClient rawblobClient = rawcontainerClient.GetBlobClient("All_Issues_With_ChangeLog_All_Projects.json"); log.LogInformation($"Read BLOB CLIENT SUCCESSFULLY"); if (await rawblobClient.ExistsAsync()) { log.LogInformation($"BLOB CLIENT EXIST"); var response = await rawblobClient.DownloadAsync(); using (var streamReader = new StreamReader(response.Value.Content)) { log.LogInformation($"READING WHILE LOOP"); while (!streamReader.EndOfStream) { var lines = await streamReader.ReadLineAsync(); dynamic j = JsonConvert.DeserializeObject(lines); //log.LogInformation($"DESERIALIZE OBJECT {j}"); oldFileobjects.Add(j); } } } log.LogInformation($"READ OLD FILE OBJECTS COUNT {oldFileobjects.Count}"); StringBuilder sb = new StringBuilder(); sb.AppendLine("{"); sb.AppendLine("\"issues\": ["); foreach (var obj in oldFileobjects) { foreach (var issObj in obj.issues) { sb.AppendLine(issObj.ToString()); sb.Append(","); } } sb.ToString().Trim(','); sb.Append("]"); sb.Append("}"); var keysList = new List(); StringBuilder updsb = new StringBuilder(); updsb.AppendLine("{"); updsb.AppendLine("\"issues\": ["); foreach (var obj in updatedFileobjects) { foreach (var issObj in obj.issues) { keysList.Add(issObj.key.ToString()); /*Storing all the key of updated json in a List*/ updsb.AppendLine(issObj.ToString()); updsb.Append(","); } } updsb.ToString().Trim(','); updsb.Append("]"); updsb.Append("}"); dynamic oldDeldata = JsonConvert.DeserializeObject(sb.ToString()); //var d = oldDeldata.issues.Where(x => x.key = "k"); dynamic remainingItemsFromOldJSON = null; /*Looping through each key of updated json*/ foreach (var key in keysList.Distinct().ToList()) { int j = 0; while (j < oldDeldata.issues.Count) { if (key == oldDeldata.issues[j].key.ToString()) /*Comparing key with each item of old json*/ { oldDeldata.issues[j].Remove(); /*If key value of old json matches with key value of updated json then deleting old date key from old json*/ remainingItemsFromOldJSON = oldDeldata.issues; j = 0; } j++; } } dynamic remainingItemsDeserialize = JsonConvert.DeserializeObject(remainingItemsFromOldJSON.ToString()); dynamic newdataupd = JsonConvert.DeserializeObject(updsb.ToString()); /*Add remaining key data from old json to updated json*/ foreach (var addo in remainingItemsDeserialize) { newdataupd.issues.Add(addo); } log.LogInformation($"Read newdataupd : {newdataupd}"); WriteToFile(newdataupd, log); log.LogInformation($"Write Succesully"); //using (StreamWriter writer = new StreamWriter(@"C:\Users\amar1\Downloads\UPSERT\Modified-Output-20ThOct.json")) //{ // writer.WriteLine(newdataupd); //} } } catch (Exception ex) { log.LogInformation($"C# Blob trigger function EXCEPTION\n \n MESSAGE: {ex.Message}"); } } public static void WriteToFile(dynamic newdataupd, ILogger log) { /*Writing Vaild Json to output file*/ string ConnectionString = GetEnvironmentVariable("AzureWebJobsStorage"); string containerName = GetEnvironmentVariable("WriteJsonFileContainerName"); string writeFileName = GetEnvironmentVariable("WriteFileName"); var storageAccount = CloudStorageAccount.Parse(ConnectionString); var blobClient = storageAccount.CreateCloudBlobClient(); var container = blobClient.GetContainerReference(containerName); var blob = container.GetBlockBlobReference(writeFileName); blob.Properties.ContentType = "application/json"; log.LogInformation($"Read newdataupd content : {newdataupd.ToString()}"); string blobContents = newdataupd.ToString(); blob.UploadText(blobContents); log.LogInformation($"UploadText successfully"); } public static string GetEnvironmentVariable(string name) { return System.Environment.GetEnvironmentVariable(name, EnvironmentVariableTarget.Process); } } }