Web API - Get progress when uploading to Azure storage

Question

The task i want to accomplish is to create a Web API service in order to upload a file to Azure storage. At the same time, i would like to have a progress indicator that reflects the actual upload progress. After some research and studying i found out two important things:

First is that i have to split the file manually into chunks, and upload them asynchronously using the PutBlockAsync method from Microsoft.WindowsAzure.Storage.dll.

Second, is that i have to receive the file in my Web API service in Streamed mode and not in Buffered mode.

So until now i have the following implementation:

UploadController.cs

using System.Configuration;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using System.Web.Http;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using WebApiFileUploadToAzureStorage.Infrastructure;
using WebApiFileUploadToAzureStorage.Models;

namespace WebApiFileUploadToAzureStorage.Controllers
{
    public class UploadController : ApiController
    {
        [HttpPost]
        public async Task<HttpResponseMessage> UploadFile()
        {
            if (!Request.Content.IsMimeMultipartContent("form-data"))
            {
                return Request.CreateResponse(HttpStatusCode.UnsupportedMediaType,
                    new UploadStatus(null, false, "No form data found on request.", string.Empty, string.Empty));
            }

            var streamProvider = new MultipartAzureBlobStorageProvider(GetAzureStorageContainer());
            var result = await Request.Content.ReadAsMultipartAsync(streamProvider);

            if (result.FileData.Count < 1)
            {
                return Request.CreateResponse(HttpStatusCode.BadRequest,
                    new UploadStatus(null, false, "No files were uploaded.", string.Empty, string.Empty));
            }

            return Request.CreateResponse(HttpStatusCode.OK);
        }

        private static CloudBlobContainer GetAzureStorageContainer()
        {
            var storageConnectionString = ConfigurationManager.AppSettings["AzureBlobStorageConnectionString"];
            var storageAccount = CloudStorageAccount.Parse(storageConnectionString);

            var blobClient = storageAccount.CreateCloudBlobClient();
            blobClient.DefaultRequestOptions.SingleBlobUploadThresholdInBytes = 1024 * 1024;

            var container = blobClient.GetContainerReference("photos");

            if (container.Exists())
            {
                return container;
            }

            container.Create();

            container.SetPermissions(new BlobContainerPermissions
            {
                PublicAccess = BlobContainerPublicAccessType.Container
            });

            return container;
        }
    }
}

MultipartAzureBlobStorageProvider.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.WindowsAzure.Storage.Blob;

namespace WebApiFileUploadToAzureStorage.Infrastructure
{
    public class MultipartAzureBlobStorageProvider : MultipartFormDataStreamProvider
    {
        private readonly CloudBlobContainer _blobContainer;

        public MultipartAzureBlobStorageProvider(CloudBlobContainer blobContainer) : base(Path.GetTempPath())
        {
            _blobContainer = blobContainer;
        }

        public override Task ExecutePostProcessingAsync()
        {
            const int blockSize = 256 * 1024;
            var fileData = FileData.First();
            var fileName = Path.GetFileName(fileData.Headers.ContentDisposition.FileName.Trim('"'));
            var blob = _blobContainer.GetBlockBlobReference(fileName);
            var bytesToUpload = (new FileInfo(fileData.LocalFileName)).Length;
            var fileSize = bytesToUpload;

            blob.Properties.ContentType = fileData.Headers.ContentType.MediaType;
            blob.StreamWriteSizeInBytes = blockSize;

            if (bytesToUpload < blockSize)
            {
                var cancellationToken = new CancellationToken();

                using (var fileStream = new FileStream(fileData.LocalFileName, FileMode.Open, FileAccess.ReadWrite))
                {
                    var upload = blob.UploadFromStreamAsync(fileStream, cancellationToken);

                    Debug.WriteLine($"Status {upload.Status}.");

                    upload.ContinueWith(task =>
                    {
                        Debug.WriteLine($"Status {task.Status}.");
                        Debug.WriteLine("Upload is over successfully.");
                    }, TaskContinuationOptions.OnlyOnRanToCompletion);

                    upload.ContinueWith(task =>
                    {
                        Debug.WriteLine($"Status {task.Status}.");

                        if (task.Exception != null)
                        {
                            Debug.WriteLine("Task could not be completed." + task.Exception.InnerException);
                        }
                    }, TaskContinuationOptions.OnlyOnFaulted);

                    upload.Wait(cancellationToken);
                }
            }
            else
            {
                var blockIds = new List<string>();
                var index = 1;
                long startPosition = 0;
                long bytesUploaded = 0;

                do
                {
                    var bytesToRead = Math.Min(blockSize, bytesToUpload);
                    var blobContents = new byte[bytesToRead];

                    using (var fileStream = new FileStream(fileData.LocalFileName, FileMode.Open))
                    {
                        fileStream.Position = startPosition;
                        fileStream.Read(blobContents, 0, (int)bytesToRead);
                    }

                    var manualResetEvent = new ManualResetEvent(false);
                    var blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(index.ToString("d6")));
                    Debug.WriteLine($"Now uploading block # {index.ToString("d6")}");
                    blockIds.Add(blockId);
                    var upload = blob.PutBlockAsync(blockId, new MemoryStream(blobContents), null);

                    upload.ContinueWith(task =>
                    {
                        bytesUploaded += bytesToRead;
                        bytesToUpload -= bytesToRead;
                        startPosition += bytesToRead;
                        index++;
                        var percentComplete = (double)bytesUploaded / fileSize;
                        Debug.WriteLine($"Percent complete: {percentComplete.ToString("P")}");
                        manualResetEvent.Set();
                    });

                    manualResetEvent.WaitOne();
                } while (bytesToUpload > 0);

                Debug.WriteLine("Now committing block list.");
                var putBlockList = blob.PutBlockListAsync(blockIds);

                putBlockList.ContinueWith(task =>
                {
                    Debug.WriteLine("Blob uploaded completely.");
                });

                putBlockList.Wait();
            }

            File.Delete(fileData.LocalFileName);
            return base.ExecutePostProcessingAsync();
        }
    }
}

I also enabled Streamed mode as this blog post suggests. This approach works great, in terms that the file is uploaded successfully to Azure storage. Then, when i make a call to this service making use of XMLHttpRequest (and subscribing to the progress event) i see the indicator moving to 100% very quickly. If a 5MB file needs around 1 minute to upload, my indicator moves to the end in just 1 second. So probably the problem resides in the way that the server informs the client about the upload progress. Any thoughts about this? Thank you.

================================ Update 1 ===================================

That is the JavaScript code i use to call the service

function uploadFile(file, index, uploadCompleted) {
    var authData = localStorageService.get("authorizationData");
    var xhr = new XMLHttpRequest();

    xhr.upload.addEventListener("progress", function (event) {
        fileUploadPercent = Math.floor((event.loaded / event.total) * 100);
        console.log(fileUploadPercent + " %");
    });

    xhr.onreadystatechange = function (event) {
        if (event.target.readyState === event.target.DONE) {

            if (event.target.status !== 200) {
            } else {
                var parsedResponse = JSON.parse(event.target.response);
                uploadCompleted(parsedResponse);
            }

        }
    };

    xhr.open("post", uploadFileServiceUrl, true);
    xhr.setRequestHeader("Authorization", "Bearer " + authData.token);

    var data = new FormData();
    data.append("file-" + index, file);

    xhr.send(data);
}

Answer 1

your progress indicator might be moving rapidly fast, might be because of

public async Task<HttpResponseMessage> UploadFile()

i have encountered this before, when creating an api of async type, im not even sure if it can be awaited, it will just of course just finish your api call on the background, reason your progress indicator instantly finish, because of the async method (fire and forget). the api will immediately give you a response, but will actually finish on the server background (if not awaited).

please kindly try making it just

public HttpResponseMessage UploadFile()

and also try these ones

var result = Request.Content.ReadAsMultipartAsync(streamProvider).Result;
var upload = blob.UploadFromStreamAsync(fileStream, cancellationToken).Result;

OR

var upload = await blob.UploadFromStreamAsync(fileStream, cancellationToken);

hope it helps.

Answer 2

Other way to acomplish what you want (I don't understand how the XMLHttpRequest's progress event works) is using the ProgressMessageHandler to get the upload progress in the request. Then, in order to notify the client, you could use some cache to store the progress, and from the client request the current state in other endpoint, or use SignalR to send the progress from the server to the client

Something like:

//WebApiConfigRegister
var progress = new ProgressMessageHandler();
progress.HttpSendProgress += HttpSendProgress;
config.MessageHandlers.Add(progress);
//End WebApiConfig Register

    private static void HttpSendProgress(object sender, HttpProgressEventArgs e)
    {   
        var request = sender as HttpRequestMessage;
        //todo: check if request is not null
        //Get an Id from the client or something like this to identify the request
        var id = request.RequestUri.Query[0];
        var perc = e.ProgressPercentage;
        var b = e.TotalBytes;
        var bt = e.BytesTransferred;
        Cache.InsertOrUpdate(id, perc);
    }

You can check more documentation on this MSDN blog post (Scroll down to "Progress Notifications" section)

Also, you could calculate the progress based on the data chunks, store the progress in a cache, and notify in the same way as above. Something like this solution