Archiving the SharePoint Office 365 Documents to Azure Cold Storage

posted on

May 26, 2016

Archiving the SharePoint Office 365 Documents to Azure Cold Storage

What is SharePoint office 365?

SharePoint online is cloud based solution to reduce the dependency on the internal resources. SharePoint office 365 provides the many features of SharePoint without managing the infrastructure on your own.

For more details about the SharePoint office 355, you can refer the below link

https://products.office.com/en-us/sharepoint/sharepoint-online-collaboration-software

What is Azure cool storage?

Cool storage is one of access tier mechanism in Azure Blob storage. Purpose of cool storage is to store your secondary data at a lower storage cost.

For more details about azure storage account, you can refer the below link

https://azure.microsoft.com/en-in/documentation/articles/storage-blob-storage-tiers/

Today I would like to explain “How to move the document(s) from SharePoint Document library to Azure cold Storage?”

Moving the documents from SharePoint office 365 to Azure cool storage is not a straight forward mechanism. Above image represents the quick glance of moving documents to azure storage. To achieve this, you need to follow the below steps

Move the Office 365 documents to Azure storage queue.

In this Process, include the “Move to Azure” option in the SharePoint document library ECB menu. The moment you click on the “Move to Azure” will create a queue in the azure storage and it put a message into the queue with document information details like “Document ID and Document url, etc.”

You can use the below JavaScript code for inserting the “Move to Azure” option in the SharePoint document library ECB menu.

$(document).ready(function () {
    SP.SOD.executeFunc("callout.js", "Callout", function () {
        var itemCtx = {};
        itemCtx.Templates = {};
        itemCtx.BaseViewID = 'Callout';
        itemCtx.ListTemplateType = 101;
        itemCtx.Templates.Footer = function (itemCtx) {
            return CalloutRenderFooterTemplate(itemCtx, AddCustomAction, true);
        };
        SPClientTemplates.TemplateManager.RegisterTemplateOverrides(itemCtx);
    });
});

function AddCustomAction(renderCtx, calloutActionMenu) {
    calloutActionMenu.addAction(new CalloutAction({
        text: "Move to Azure",
        tooltip: 'Move to Azure Blob storage',
        onClickCallback: function () {
        }
    }));
}

$(document).ready(function () {

SP.SOD.executeFunc("callout.js", "Callout", function () {

var itemCtx = {};

itemCtx.Templates = {};

itemCtx.BaseViewID = 'Callout';

itemCtx.ListTemplateType = 101;

itemCtx.Templates.Footer = function (itemCtx) {

return CalloutRenderFooterTemplate(itemCtx, AddCustomAction, true);

};

SPClientTemplates.TemplateManager.RegisterTemplateOverrides(itemCtx);

});

function AddCustomAction(renderCtx, calloutActionMenu) {

calloutActionMenu.addAction(new CalloutAction({

text: "Move to Azure",

tooltip: 'Move to Azure Blob storage',

onClickCallback: function () {

}

}));

}

Azure storage queue expose a rest API to create a queue, blobs and table. Based on that, I am creating a queue by using Queue service Rest Api. To create a queue, first we need to generate the authentication token and sample is given below for token generation.

function GetToken()
{
    var stringToSign = "PUT\n\n\n0\n\n\n\n\n\n\n\n\nx-ms-date:" + dateInUTC + "\nx-ms-version:" + apiVersion + "\n/spdocumentstorage/samplequeue3";
    var StorageAccountKey = "maVja38Ns6b4AXM+fMSOQApRuijoBJmXsQiqALYJqi6DQNbeVg3uu31S0ErZo0nLx6s74nrmk1zc5YBXGZ5jaA==";
    var signature = CryptoJS.enc.Base64.stringify(CryptoJS.HmacSHA256(CryptoJS.enc.Utf8.parse(stringToSign), CryptoJS.enc.Base64.parse(StorageAccountKey)));
    console.log(signature);
    return signature;
}

function GetToken()

{

var stringToSign = "PUT\n\n\n0\n\n\n\n\n\n\n\n\nx-ms-date:" + dateInUTC + "\nx-ms-version:" + apiVersion + "\n/spdocumentstorage/samplequeue3";

var StorageAccountKey = "maVja38Ns6b4AXM+fMSOQApRuijoBJmXsQiqALYJqi6DQNbeVg3uu31S0ErZo0nLx6s74nrmk1zc5YBXGZ5jaA==";

var signature = CryptoJS.enc.Base64.stringify(CryptoJS.HmacSHA256(CryptoJS.enc.Utf8.parse(stringToSign), CryptoJS.enc.Base64.parse(StorageAccountKey)));

console.log(signature);

return signature;

}

In the above example Verb, date and CanonicalizedResource are required parameters. For building the “stringToSign” string, you need to follow the below msdn reference link

https://msdn.microsoft.com/en-us/library/azure/dd179428.aspx

You can user the below code to create the queue in the azure storage

var queueurl = "https://spdocumentstorage.queue.core.windows.net/samplequeue1";
var accountName = "spdocumentstorage";
var dt = new Date(); // "Wed, 18 May 2016 07:21:48 GMT";
var dateInUTC = "Wed, " + dt.getDate() + " May 2016 " + dt.getHours() + ":" + dt.getMinutes() + ":" + dt.getSeconds() + " GMT";
var apiVersion = "2009-09-19";
var token = GetToken(stringToSign);
              $.ajax({
                 url: queueurl,
                 type: 'PUT',
                 success: function (data) {
                   alert("Queue is created");
                   },
                 beforeSend: function (xhr) {
                    xhr.setRequestHeader('Authorization', "SharedKey " + accountName + ":" + token);
                    xhr.setRequestHeader('x-ms-date',"" + dateInUTC);
                    xhr.setRequestHeader('x-ms-version', apiVersion);
                 },
                 error: function (rcvData) {
                    console.log(rcvData);
                 }
               });

var queueurl = "https://spdocumentstorage.queue.core.windows.net/samplequeue1";

var accountName = "spdocumentstorage";

var dt = new Date(); // "Wed, 18 May 2016 07:21:48 GMT";

var dateInUTC = "Wed, " + dt.getDate() + " May 2016 " + dt.getHours() + ":" + dt.getMinutes() + ":" + dt.getSeconds() + " GMT";

var apiVersion = "2009-09-19";

var token = GetToken(stringToSign);

$.ajax({

url: queueurl,

type: 'PUT',

success: function (data) {

alert("Queue is created");

beforeSend: function (xhr) {

xhr.setRequestHeader('Authorization', "SharedKey " + accountName + ":" + token);

xhr.setRequestHeader('x-ms-date',"" + dateInUTC);

xhr.setRequestHeader('x-ms-version', apiVersion);

error: function (rcvData) {

console.log(rcvData);

}

});

You can use the following code to put a message into the queue.

var msg = "<Document id=\"1\"> https://company.sharepoint.com/sites/sitecollection/DocLibname/TestDocument.docx "<Document>";
var Base64 = { _keyStr: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=", encode: function (e) { var t = ""; var n, r, i, s, o, u, a; var f = 0; e = Base64._utf8_encode(e); while (f < e.length) { n = e.charCodeAt(f++); r = e.charCodeAt(f++); i = e.charCodeAt(f++); s = n >> 2; o = (n & 3) << 4 | r >> 4; u = (r & 15) << 2 | i >> 6; a = i & 63; if (isNaN(r)) { u = a = 64 } else if (isNaN(i)) { a = 64 } t = t + this._keyStr.charAt(s) + this._keyStr.charAt(o) + this._keyStr.charAt(u) + this._keyStr.charAt(a) } return t }, decode: function (e) { var t = ""; var n, r, i; var s, o, u, a; var f = 0; e = e.replace(/[^A-Za-z0-9+/=]/g, ""); while (f < e.length) { s = this._keyStr.indexOf(e.charAt(f++)); o = this._keyStr.indexOf(e.charAt(f++)); u = this._keyStr.indexOf(e.charAt(f++)); a = this._keyStr.indexOf(e.charAt(f++)); n = s << 2 | o >> 4; r = (o & 15) << 4 | u >> 2; i = (u & 3) << 6 | a; t = t + String.fromCharCode(n); if (u != 64) { t = t + String.fromCharCode(r) } if (a != 64) { t = t + String.fromCharCode(i) } } t = Base64._utf8_decode(t); return t }, _utf8_encode: function (e) { e = e.replace(/rn/g, "n"); var t = ""; for (var n = 0; n < e.length; n++) { var r = e.charCodeAt(n); if (r < 128) { t += String.fromCharCode(r) } else if (r > 127 && r < 2048) { t += String.fromCharCode(r >> 6 | 192); t += String.fromCharCode(r & 63 | 128) } else { t += String.fromCharCode(r >> 12 | 224); t += String.fromCharCode(r >> 6 & 63 | 128); t += String.fromCharCode(r & 63 | 128) } } return t }, _utf8_decode: function (e) { var t = ""; var n = 0; var r = c1 = c2 = 0; while (n < e.length) { r = e.charCodeAt(n); if (r < 128) { t += String.fromCharCode(r); n++ } else if (r > 191 && r < 224) { c2 = e.charCodeAt(n + 1); t += String.fromCharCode((r & 31) << 6 | c2 & 63); n += 2 } else { c2 = e.charCodeAt(n + 1); c3 = e.charCodeAt(n + 2); t += String.fromCharCode((r & 15) << 12 | (c2 & 63) << 6 | c3 & 63); n += 3 } } return t } }
var messageBodyBase64 = Base64.encode(msg);
var message = "<QueueMessage><MessageText>" + messageBodyBase64 + "</MessageText></QueueMessage>";
var messagestring = "POST\n\n\n" + message.length + "\n\n\n\n\n\n\ntext/plain; charset=UTF-8\nx-ms-date:" + dateInUTC + "\nx-ms-version:" + apiVersion + "\n/spdocumentstorage/samplequeue1/messages";
console.log("messagestring" + messagestring);
var msgtoken = GetToken(messagestring);
var queueurl = "https://spdocumentstorage.queue.core.windows.net/samplequeue1/messages";
$.ajax({
        url: queueurl,
        type: 'POST',
        success: function (data) {
            alert("Messages is pushed to queue");
        },
        data: message,
        contentType: "text/plain; charset=UTF-8",
        beforeSend: function (xhr) {
            xhr.setRequestHeader('Authorization', "SharedKey " + accountName + ":" + msgtoken);
            xhr.setRequestHeader('x-ms-date', "" + dateInUTC);
            xhr.setRequestHeader('x-ms-version', apiVersion);
            xhr.setRequestHeader('Content-Length', message.length);
        },
        error: function (rcvData) {
        console.log(rcvData);
        } 
   });

var msg = "<Document id=\"1\"> https://company.sharepoint.com/sites/sitecollection/DocLibname/TestDocument.docx "<Document>";

var messageBodyBase64 = Base64.encode(msg);

var message = "<QueueMessage><MessageText>" + messageBodyBase64 + "</MessageText></QueueMessage>";

var messagestring = "POST\n\n\n" + message.length + "\n\n\n\n\n\n\ntext/plain; charset=UTF-8\nx-ms-date:" + dateInUTC + "\nx-ms-version:" + apiVersion + "\n/spdocumentstorage/samplequeue1/messages";

console.log("messagestring" + messagestring);

var msgtoken = GetToken(messagestring);

var queueurl = "https://spdocumentstorage.queue.core.windows.net/samplequeue1/messages";

$.ajax({

url: queueurl,

type: 'POST',

success: function (data) {

alert("Messages is pushed to queue");

data: message,

contentType: "text/plain; charset=UTF-8",

beforeSend: function (xhr) {

xhr.setRequestHeader('Authorization', "SharedKey " + accountName + ":" + msgtoken);

xhr.setRequestHeader('x-ms-date', "" + dateInUTC);

xhr.setRequestHeader('x-ms-version', apiVersion);

xhr.setRequestHeader('Content-Length', message.length);

error: function (rcvData) {

console.log(rcvData);

}

});

Read the messages from the Azure storage queue and move to Azure cool storage

In this Process, Azure web job read the message form the queue and it process the data. My job picks the document library information from the message and downloading the documents based on the message url and it will move the documents to azure cold storage. Also it removes the document from the office 365 document library.

For more information related to azure jobs, refer the below url

https://azure.microsoft.com/en-in/documentation/articles/web-sites-create-web-jobs/