Azure Basic - Windows Storage Enhancement (Blob)

Blob

1. meta data (In Sample1)
2. retry & timeout policy  (In Sample1)
3. parallel upload file using putblock & putblocklist  (In Sample1)
   
4. Conditional Operation (Update, Insert, Delete, Read) (ETag) (In Sample 2)
5. private/public container (In Sample 3)
6. Shared Access Signatures (SAS) share with selected few clients(In Sample 3)
7. Public containers read-only access to anyone(In Sample 3)

Sample 1

http://blogs.msdn.com/b/kwill/archive/2011/05/30/asynchronous-parallel-block-blob-transfers-with-progress-change-notification.aspx

Web.config

 <system.web>
    <httpRuntime
          executionTimeout="900"
          maxRequestLength="409600"/>

Code

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.StorageClient;
using System.IO;
using System.Threading;
using System.Text;
using System.Security.Cryptography;

namespace WebApplication1
{
    public partial class _Default : System.Web.UI.Page
    {
        protected void Page_Load(object sender, EventArgs e)
        {


        }

        public void ParallelUploadToFile(CloudBlockBlob blob, BlobRequestOptions options, string fileName)
        {
            //Initilize
            //int maxBlockSize = 4 * 1024 * 1024;//4MB
            int maxBlockSize = 4 * 1024;//4KB
            int numThreads = 10;
            List<string> blockList = new List<string>();
            List<Thread> threads = new List<Thread>();
            Queue<KeyValuePair<int, int>> queue = new Queue<KeyValuePair<int, int>>();//record every small block blob name&length

            //Calculate File Size
            FileInfo finfo = new FileInfo(fileName);
            long FileInBytes = finfo.Length;
            var blockCount = ((int)Math.Floor((double)(FileInBytes / maxBlockSize))) + 1;

            //Record each block blob name and length and put into Queue
            int blockLength = 0;
            int blockId = 0;
            while (FileInBytes > 0)
            {
                blockLength = (int)Math.Min(maxBlockSize, FileInBytes);
                string blockIdString = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}",
                    blockId.ToString("0000000"))));
                KeyValuePair<int, int> kvp = new KeyValuePair<int, int>(blockId++, blockLength);
                queue.Enqueue(kvp);
                blockList.Add(blockIdString);
                FileInBytes -= blockLength;
            }

            //Create multiple thread to parall upload file
            for (int idxThread = 0; idxThread < numThreads; idxThread++)
            {
                Thread t = new Thread(new ThreadStart(() =>
                {
                    using (FileStream fs = new FileStream(fileName,
                        FileMode.Open, FileAccess.Read))
                    {
                        while (true)
                        {
                            KeyValuePair<int, int> blockIdAndLength = new KeyValuePair<int, int>(-1, -1);
                            lock (queue)
                            {
                                if (queue.Count <= 0)
                                {
                                    break;
                                }

                                blockIdAndLength = queue.Dequeue();
                            }

                            byte[] buff = new byte[blockIdAndLength.Value];
                            BinaryReader br = new BinaryReader(fs);

                            // Move the file system reader to the proper position
                            fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin);
                            br.Read(buff, 0, blockIdAndLength.Value);

                            // Upload block.
                            string blockName = Convert.ToBase64String(BitConverter.GetBytes(
                                blockIdAndLength.Key));

                            using (MemoryStream ms = new MemoryStream(buff, 0, blockIdAndLength.Value))
                            {
                                string blockHash = GetMD5HashFromStream(buff);
                                blob.PutBlock(blockList[blockIdAndLength.Key], ms, blockHash, options);
                            }
                        }
                    }
                }
                    )
                    );
                t.Start();
                threads.Add(t);
            }

            // Wait for all threads to complete uploading data.
            foreach (Thread t in threads)
            {
                t.Join();
            }

            // Commit the blocklist.
            blob.PutBlockList(blockList, options);
        }

        private string GetMD5HashFromStream(byte[] data)
        {
            MD5 md5 = new MD5CryptoServiceProvider();
            byte[] blockHash = md5.ComputeHash(data);
            return Convert.ToBase64String(blockHash, 0, 16);
        }

        protected void btnParallelUpload_Click(object sender, EventArgs e)
        {
            var storageAccount =
                CloudStorageAccount.FromConfigurationSetting("DataConnectionString");

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Create Container
            CloudBlobContainer cloudContainer = blobClient.GetContainerReference("containername");
            bool hasCreated = cloudContainer.CreateIfNotExist();

            // Access Blob in the Container
            CloudBlockBlob cloudBlob = cloudContainer.GetBlockBlobReference("blobname");
            cloudBlob.DeleteIfExists();

            //BlobRequestOptions has retry policy, timeout etc.
            BlobRequestOptions options = new BlobRequestOptions()
            {
                RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultMaxBackoff),
                Timeout = TimeSpan.FromSeconds(90)
            };

            //Meta Data
            cloudBlob.Metadata["NameOfFile"] = "Alex";

            //Upload File In Parallel
            ParallelUploadToFile(cloudBlob, options, Server.MapPath(FileUpload1.FileName));
        }
    }
}

     
     

      
      
     
     
Sample 2

We use following property to implement the condition opeartion, e.g.: we will only update the blob which is not been update by other thread after I read

1. IfNoneMatch
2. ETag

Base on the Sample 1: Add following code at the end of method ParallelUploadToFile()

// Commit the blocklist.
            blob.PutBlockList(blockList, options);
            HiddenField1.Value = blob.Properties.ETag;

To save ETag value to a hidden field

Then when download the blob, we check whether the blob has been changed or not, If it has been change, then throw a exception

protected void Button1_Click(object sender, EventArgs e)
        {
            var storageAccount =
                            CloudStorageAccount.FromConfigurationSetting("DataConnectionString");

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Create Container
            CloudBlobContainer cloudContainer = blobClient.GetContainerReference("containername");
            bool hasCreated = cloudContainer.CreateIfNotExist();

            // Access Blob in the Container
            CloudBlockBlob cloudBlob = cloudContainer.GetBlockBlobReference("blobname");

            BlobRequestOptions options = new BlobRequestOptions()
            {
                AccessCondition = AccessCondition.IfMatch(HiddenField1.Value)
            };

            try
            {
                byte[] DownloadByteArraySecondTimeOnward = cloudBlob.
                DownloadByteArray(options);
                Response.Write(DownloadByteArraySecondTimeOnward);
                Response.End();

            }
            catch (StorageClientException ex)
            {
              Response.Write("The File is not exist!!!!!!!!!!!!!!!!");
            }

Sample 3

As the code in Sample 1, We does not define the container access level, so by default it is private, in this case, we cannot access our blob via URL directly. If we do so, we will get the following error:

http://127.0.0.1:10000/devstoreaccount1/containername/blobname

<Error>
<Code>ResourceNotFound</Code>
<Message>
The specified resource does not exist. RequestId:5259965f-7ba7-4495-a5bf-9c1cae3c44fc Time:2011-11-19T15:05:16.7015163Z
</Message>
</Error>

Here is some resolutions

S1: Use Azure SDK

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.StorageClient;

namespace ConsoleApplication1
{
    class Program
    {
        static void Main(string[] args)
        {
            string accountName = "devstoreaccount1";
            string key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
            string containerName = "containername";
            string blobName = "blobname";

            StorageCredentialsAccountAndKey credentials =
    new StorageCredentialsAccountAndKey(accountName, key);

            string baseUri = string.Format("http://127.0.0.1:10000/{0}", accountName);
            CloudBlobClient blobClient = new CloudBlobClient(baseUri, credentials);

            // Create Container
            CloudBlobContainer cloudContainer = blobClient.GetContainerReference(containerName);
            bool hasCreated = cloudContainer.CreateIfNotExist();

            // Access Blob in the Container
            CloudBlob cloudBlob = cloudContainer.GetBlobReference(blobName);
            byte[] DownloadByteArraySecondTimeOnward = cloudBlob.
                DownloadByteArray();

            Console.Write(DownloadByteArraySecondTimeOnward.ToString());
            Console.Read();
        }
    }
}

 

S2: Add SAS in container level

http://127.0.0.1:10000/devstoreaccount1/containername/blobname?sr=c&si=partneraccess&sig=6IZLCb4%2Bgss8PpfGvd242SLsVxVmynziCHNWlumyVwg%3D

 // Create Container
            CloudBlobContainer cloudContainer = blobClient.GetContainerReference(containerName);
            bool hasCreated = cloudContainer.CreateIfNotExist();

            SharedAccessPolicy sap = new SharedAccessPolicy();
            sap.Permissions = SharedAccessPermissions.Read | SharedAccessPermissions.Write;
            sap.SharedAccessStartTime = DateTime.UtcNow;
            sap.SharedAccessExpiryTime = DateTime.UtcNow + TimeSpan.FromDays(100);

            BlobContainerPermissions bcp = new BlobContainerPermissions();
            bcp.PublicAccess = BlobContainerPublicAccessType.Off;
            bcp.SharedAccessPolicies.Clear();
            bcp.SharedAccessPolicies.Add("partneraccess", sap);

            cloudContainer.SetPermissions(bcp);

            var sas = cloudContainer.GetSharedAccessSignature(new SharedAccessPolicy() { }, "partneraccess");
            Console.WriteLine(sas);

S3:  Add SAS in blob level

http://127.0.0.1:10000/devstoreaccount1/containername/blobname?st=2011-11-19T15%3A43%3A32Z&se=2011-11-19T15%3A48%3A32Z&sr=b&sp=r&sig=QT9zguLKhV2cHXMZycK7FOQBJLq%2BpzyrHCr9J5yGFLc%3D

// Access Blob in the Container
            CloudBlob cloudBlob = cloudContainer.GetBlobReference(blobName);

            // create the SAS for a specified duration 
            var sas = cloudBlob.GetSharedAccessSignature(new SharedAccessPolicy
            {
                Permissions = SharedAccessPermissions.Read,
                SharedAccessStartTime = DateTime.UtcNow,
                SharedAccessExpiryTime = DateTime.UtcNow + TimeSpan.FromMinutes(5)
            });

S4:  Use Restful API

To Be Filled In