PROACTIVE COPY TO HOT SPARE ON EMC VNX ARRAY

// // // <![CDATA[
function showError(errorSelector) {
$(errorSelector).show();
$('#pn-loading').hide();
}

$(function() {
setupChromeExtension();
});

function loadPf() {
pf.setup();
pf.start();
$('#pn-loading').hide();
$('#pfcom-wrapper').show();
if(pf.userSettings.hideImages) {$('#wri').click();}
}

function showErrorOnSwod() {
if($('#pf-content').html() == '' && $('.pfcom-message-wrapper:visible').length == 0) {
showError('#pf-swod');
}
callRemote = false;
}

function process(data) {
$(function() {
$('#pf-content').html(data.content);
$('#pf-title').html(data.title);
if(data.author == '') {
$('#pf-author').remove();
} else {
$('#pf-author').html(data.author);
}
pf.dir = data.dir;
retrievedTitle = data.title;
$('#pn-loading').hide();
$('#pfcom-nav, #pfcom-feature, #pfcom-wrapper').show();
loadPf();
});
}

function poll() {
jQuery.ajax({
data: ({print_id: printId }), dataType: 'json',
type: 'get', url:'/print/poll.js',
timeout: 5000,
cache: false,
error:function(request, textStatus, errorThrown){
if(textStatus == 'timeout' && callRemote) {
window.setTimeout(poll,1000);
} else {
$(function() {
showError('#pf-error');
});
}
},
success: function(resp){
callRemote = false;
if(resp.status == 'fetching') {
callRemote = true;
} else if(resp.status == 'success') {
process(resp.data);
} else if(resp.status.match(/failed/)) {
$(function() {
if(resp.status.match(/failed-parse/)) {
showError('#pf-parse-error');
} else {
if(document.domain == 'www.printfriendly.com') {
// some URLs are not reachable from ec2. Try it from hetzner
// TODO: We should be using local hostnames instead of domain names
window.location.replace(window.location.href.replace('www','pf-nl-1'));
} else {
showError('#pf-error');
}
}
});
} else if(resp.status == 'timeout') {
$(function() {
showError('#pf-timeout');
});
}
if(callRemote) {window.setTimeout(poll,100)}
}});
}

function submitUrl() {
jQuery.ajax({
data: ({url: siteUrl, dummy: 1}), dataType:'json',
type:'post', url:'/print/create.js',
timeout: 3000,
cache: false,
error:function(request, textStatus, errorThrown){
if(textStatus == 'timeout') {
window.setTimeout(submitUrl,1000);
} else if(request.status == 503) {
$(function() {
showError('#high-load-error');
});
} else {
$(function() {
showError('#pf-error');
});
}
},
success:function(data){
printId = data.print_id;
window.setTimeout(poll,100);
}
});
}

if(hasContent) {
$(function() {
$('#pfcom-nav, #pfcom-feature').show();
loadPf();
});
} else {
if(error == 'invalid-url' || error == 'banned-domain') {
$(function() {
showError('#pf-' + error + '-error');
});
} else if(siteUrl != '') {
submitUrl();
} else {
$(function() {
showError('#pf-error');
});
}
}
window.setTimeout(showErrorOnSwod, 50000);

Cookie = {
set: function(name,value,days) {
if (days) {
var date = new Date();
date.setTime(date.getTime()+(days*24*60*60*1000));
var expires = "; expires="+date.toGMTString();
}
else var expires = "";
document.cookie = name+"="+value+expires+"; path=/";
},

get: function(name) {
var nameEQ = name + "=";
var ca = document.cookie.split(';');
for(var i=0;i < ca.length;i++) {
var c = ca[i];
while (c.charAt(0)==' ') c = c.substring(1,c.length);
if (c.indexOf(nameEQ) == 0) return c.substring(nameEQ.length,c.length);
}
return null;
},

erase: function(name) {
createCookie(name,"",-1);
}
};

changeFont = function(fontClass) {
$('#pf-content').removeClass('pf-9 pf-10 pf-11 pf-12 pf-13 pf-14 pf-15').addClass(fontClass);
};

var getLocation = function(href) {
var l = document.createElement("a");
l.href = href;
return l;
};

var pf = {
deletedNodes: [],
deletedNodesCss: [],
showAds: false,
dir: window['dir'],
host: requestDomain,
domain: null,
pdfClick: false,
imageDisplayStyle: 'right',

userSettings: {
imageDisplayStyle: 'right',
disablePDF: false,
disablePrint: false,
disableEmail: false,
hideImages: false
},

removeImages: function() {
$('#pf-content img').toggleClass('pf-hidden');
$('#pf-content img.thumbimage').parents('.thumbinner').toggleClass('pf-hidden');
$('#pf-content img').parents('.img-separator').toggleClass('pf-hidden');
},

openGCPDialog: function() {
var gadget = new cloudprint.Gadget();
gadget.setPrintDocument("url", retrievedTitle, window.location.href);
gadget.openPrintDialog();
},

setup: function() {
var settings = this.userSettings;
if(settings.disablePDF) $('#w-pdf').remove();
if(settings.disableEmail) $('#w-email').remove();
if(settings.disablePrint) $('#w-print').remove();
if($.inArray(settings.imageDisplayStyle, ['left','right','block','none']) !== -1) pf.imageDisplayStyle = settings.imageDisplayStyle;
},

start: function() {
pf.domain = pf.host.split(':')[0].split('www.').pop();

$('#textsize').change(function() {
changeFont($(this).val());
Cookie.set('pf-font-class', $(this).val(), 365);
});

$('#w-print').click(function(){
$("#pf-dialog").css({'display': 'block'});
$("body").css('overflow', 'hidden');
$("#pf-dialog-pdf").hide();
$("#pf-dialog-print").show();
if(pf.showAds) {
$('#pf-dialog-ads').show();
}
window.print();
return false;
});

$('#w-pdf').click(function(){
var title = '';
$("#pdf-frame-container").html('http://cdn.printfriendly.com/IEneeds/iframe_blank.html‘);
$(“body”).css(‘overflow’, ‘hidden’);
$(“#pf-dialog”).css({‘display’: ‘block’});
$(“#pf-dialog-print”).hide();
$(“#pf-dialog-pdf”).show();
if(pf.showAds) {
$(‘#pf-dialog-ads’).show();
}
$(“#pf-pdf-form input[name=title]”).val(retrievedTitle);
$(“#pf-pdf-form input[name=hostname]”).val(requestDomain);
$(“#pf-pdf-form input[name=code]”).val($(‘#pf-body’).html());
$(“#pf-pdf-form input[name=dir]”).val(pf.dir);
$(“#pf-pdf-form”).submit();
return false;
});

$(‘#w-email’).click(function(e){
var a = typeof window.screenX != ‘undefined’ ? window.screenX : window.screenLeft;
var i = typeof window.screenY != ‘undefined’ ? window.screenY : window.screenTop;
var g = typeof window.outerWidth!=’undefined’ ? window.outerWidth : document.documentElement.clientWidth;
var f = typeof window.outerHeight != ‘undefined’ ? window.outerHeight: (document.documentElement.clientHeight – 22);
var h = (a 0,
figWidth,
figHeight;

/**
* If the image has no area, there’s nothing to fix.
* In addition, the image may not have loaded properly yet,
* so we should just leave it be.
**/
if(!width || !height) {
return;
}

/**
* If the image is too small to be worth displaying, hide it.
*/
if (width <= limg.imageHideThreshold || height limg.imageWidthThreshold) {
$img.addClass(“blockImage”);
} else {
$img.addClass(“smallImage”);
if(pf.imageDisplayStyle == ‘block’) {
$img.css({‘display’: ‘block’, ‘clear’: ‘both’, ‘float’: ‘none’, ‘margin’: ‘1em 0’});
} else {
if(pf.imageDisplayStyle == ‘none’) {
margin = ‘0’
} else if(pf.imageDisplayStyle == ‘left’) {
margin = ‘1em 1.5em 1em 0’
} else {
margin = ‘1em 0 1em 1.5em’
}
$img.css({‘float’: pf.imageDisplayStyle, ‘clear’: pf.imageDisplayStyle, ‘margin’: margin});
}
}
}
if($img.hasClass(‘caption-img’)) {
$img.parents(‘.wp-caption, .caption, .tr-caption-container’).width($img[0].offsetWidth).addClass(‘pf-caption’);
if($img.hasClass(‘thumbimage’)) {
$img.parents(‘.thumbinner’).width($img[0].offsetWidth).addClass(‘pf-caption’);
}
}

}, 250);

},

init: function () {
$(‘#pf-content img’).each(function() {
var $img = $(this);
if ($img.width() === 0) {
/* If the image was not cached, call layoutImage when it’s loaded. */
$img.load(function () {
pf.layoutImages.layoutImage($img);
});
// IE sometimes doesn’t fire the load event. This is the failsafe
window.setTimeout(function(){pf.layoutImages.layoutImage($img)}, 2000);
} else {
/* If the image was cached, you call it immediately because onload will never fire. */
pf.layoutImages.layoutImage($img);
}
});
}
},

// This function is called after page parsing
complete: function() {
$(‘.pf-init-iframe, .sociable, #sociable, .addthis, #addthis, .printfriendly, .pf-print, #pf-print, .wp-socializer, .editsection, .a2a_dd, .addtoany_share_save, .addtoany_share_save_container, .simply-social-wrapper, #pf-mask, .social_button, #socialbookmarks, .articleFeedback, .print-no, .no-print, .ftwit, .famos-toolbar, .famos-fstar, .ftwit-drawer’, $(‘#’)).remove();
if($(‘body’).hasClass(‘wikipedia’)) {
$(‘.noprint, div#jump-to-nav, .mw-jump, div.top, div#column-one, #colophon, .editsection, .toctoggle, .tochidden, div#f-poweredbyico, div#f-copyrightico, li#viewcount, li#about, li#disclaimer, li#mobileview, li#privacy, #footer-places, .mw-hidden-catlinks, tr.mw-metadata-show-hide-extended, span.mw-filepage-other-resolutions, #filetoc, .usermessage, #ca-delete, span.brokenref, .compact-ambox table .mbox-image,.compact-ambox table .mbox-imageright,.compact-ambox table .mbox-empty-cell, .compact-ambox .hide-when-compact, .check-icon a.new, .geo-nondefault,.geo-multi-punct, .nonumtoc .tocnumber, .nonumtoc .tocnumber, .toclimit-2 .toclevel-1 ul,.toclimit-3 .toclevel-2 ul,.toclimit-4 .toclevel-3 ul,.toclimit-5 .toclevel-4 ul,.toclimit-6 .toclevel-5 ul,.toclimit-7 .toclevel-6 ul, .mw-special-Watchlist #mw-watchlist-resetbutton, .wpb .wpb-header, .wpbs-inner .wpb-outside, .sysop-show,.accountcreator-show, .inputbox-hidecheckboxes form .inputbox-element, #editpage-specialchars, .wikipedia .ambox,.wikipedia .navbox,.wikipedia .vertical-navbox,.wikipedia .infobox.sisterproject,.wikipedia .dablink,.wikipedia .metadata,.editlink,a.NavToggle,span.collapseButton,span.mw-collapsible-toggle, #content cite a.external.text:after,.nourlexpansion a.external.text:after,.nourlexpansion a.external.autonumber:after, .skin-simple div#column-one,.skin-simple div#f-poweredbyico,.skin-simple div#f-copyrightico,.skin-simple .editsection, #siteNotice, div.magnify’).remove();
}

try {
$(‘#pf-content a’).each(function(i) {
var $this = $(this);
var content = $.trim($this.html().replace(‘ ‘,’ ‘,’g’));
if(content === ”) {
$this.remove();
}
});

// Blogger wraps images in a div.separator. Takes up space
// even after image is hidden
$(‘#pf-content div.separator’).each(function(i) {
var $this = $(this);
if($this.children().size() == $this.find(‘a,br’).size()) {
$this.addClass(‘img-separator’);
}
});

} catch(e) {}

var l = getLocation(decodeURIComponent(siteUrl));
var host = l.host;
var path = l.path || l.pathname

host = host.replace(/^www./i, ”); // Remove www
host = host.replace(//$/, ”); // Remove ‘/’ from the end
host = host.replace(/:(80|443)$/, ”); //Remove default port for IE

if (path[0] != ‘/’)
path = ‘/’ + path;

$([

‘,


].join(”)).insertAfter(‘#pf-title’);

var elements = “p, img, blockquote, h1, h2, h3, h4, ul, li, a, table, tr, pre, span, b, strong, i”;
var elementsToSkip = ‘#pf-title, #pf-src, #pf-src *, #copyright, .copyright, .delete-off, .delete-no’;
$(‘#pf-print-area’).delegate(elements,’mouseover mouseout’, function (e) {
if(!$(this).is(elementsToSkip)) {
if(e.type == ‘mouseover’) {
$(this).addClass(“hilight”);
return false;
} else {
$(this).removeClass(“hilight”);
return false;
}
}
});

$(‘#pf-print-area’).delegate(elements, ‘click’, function () {
if(!$(this).is(elementsToSkip)) {
pf.deletedNodes.push($(this));
pf.deletedNodesCss.push($(this).css(‘display’));
$(this).css({‘display’:’none’});
}
// stops event propagation and default event.
// Otherwise event propagtes to parent/child
// and it becomes a mess
return false;
});

$(‘.wp-caption img, .caption img, .tr-caption-container img, .thumbinner img.thumbimage’).each(function(){
$(this).addClass(‘caption-img’);
});

$(‘#ajax_loader’).hide();
pf.layoutImages.init();
if(pf.dir == ‘rtl’) {
$(‘#pf-content, #pf-title’).css({“direction”: pf.dir});
}
}
}
// ]]>// = 15) {
$i.click(function() {
chrome.webstore.install(url, installSuccess, installError);
return false;
});
} else {
$i.attr(‘target’, ‘_blank’);
$i.attr(‘href’, url);
}

$(‘#chrome-extension-message-wrapper’).show();
}
}
img.src = ‘chrome-extension://ohlencieiipommannpdfcmfdpjjmeolj/images/print-friendly-16px.png’;
}
}

function installSuccess() {
$(‘#chrome-extension-message-wrapper’).hide();
}

function installError(msg) {
}

function optOut() {
$(‘#chrome-extension-message-wrapper’).hide();
Cookie.set(‘pf-chrome-optout’, 1, 365);
}

// ]]>

Proactive copy to hot spare on EMC VNX array

extremesanity

 SAN errors, oh no!

The process started when the array emailed me a couple of soft media errors, so I glanced at the SPA and SPB event logs in Navisphere and saw this:

2014.3.5_initial_errors

Notice that the majority of errors showed up as informational and not warning or critical, meaning the array will not indicate to anyone that this drive is about to fail, yeesh.

Note also that all the errors occur on the same disk, Bus 0 Enclosure 1 Disk 7, a NLSAS 2TB drive in my environment.

Event codes included 0x6a0, 0x820, and 0x801 with descriptions of disk soft media error, soft scsi bus error, and soft media error.  My suggestion is to filter only on description, and do searches for “error” to find all the messages.

Reviewing the disks within the navisphere GUI showed that no disks were faulted, the dashboard showed no errors, and the hot spare was not in use, meaning the array did not believe the drive should be failed yet.


Time to be proactive…

I opened a case with EMC support, and sent them screenshots of the SP event logs, and SPCollects from both SPA and SPB, and noted that the error occurred on the same drive over 100 times in one day.  The support representative immediately requested I do a proactive copy to the hot spare disk, and ordered a replacement disk sent to me.

A proactive copy is preferred because instead of requiring the array to rebuild the RAID array to a new disk (and endure the performance degradation inherent in this procedure), it copies the data from one disk to another, then tells the RAID array to use the hot spare disk, then disables the failing disk, skipping the rebuild process altogether and hence no RAID rebuild performance degradation.

I first tried to do the proactive copy from the navisphere gui without success, below.

2014.3.5_proactivecopy_greyedout

Note the option to copy is greyed out.  Apparently VNXs new mixed RAID storage pools prevents this option from being used, so I moved onto the CLI.

Passing the command
naviseccli -Address -User -Password -Scope 0 copytohotspare 0_1_7 -initiate
(where 0_1_7 is the failing disk) worked correctly, starting the proactive copy from the failing disk to the hotspare of the same type.

Checking progress of the proactive copy

Now to check progress of the copy…

First I tried looking at the disks in the GUI

2014.3.5_noprogress_gui_1

The disk state is listed as “Copying to Hot Spare(100%)”.  Hmm, 100% doesn’t seem right, I just started this procedure.  (Looking at the RAID LUN within the GUI showed the state as “transitioning” without any progress indicators)

Then I tried through the CLI

2014.3.5_noprogress_cli

Well that also doesn’t look right, either.  I continued on looking at SPCollect logs, SP event logs, and looking all over the Internet, including the EMC community forums, without finding any answers.  (running getlun on the RAID LUN from the CLI didn’t show any progress indicators also)

Update: I figured out a way to gauge progress, though a bit crude.  See the update at the bottom.

Eventually after 19 hours (NLSAS 2TB) the process completed, throwing event logs 0x6b0, 0x604, 0x67d, 0x6a8, 0x67c, 0x7a7, 0x6ab0 along with many others indicating it had marked the failing drive as failed as expected.

2014.3.5_proactivecopy_completed

Complete list of event logs thrown as part of the proactive copy completion:
0x6b0, 0x712d4601, 0x906, 0x7a7, 0x608, 0x6a1, 0x602, 0x7a5, 0x6a8, 0x712789a0, 0x67b, 0x67c, 0x603, 0x602, 0x712d0508, 0x604, 0x712d0507, 0x2580, 0x906, 0x7a6, 0x799, 0x712d4602, 0x712d4601, 0x67d,  0x7400, 0x740a, 0x2580and probably some others I missed.

At this point I gave the EMC CE a call and scheduled replacement of the drive.  Once the replacement drive is in place, the array should copy the information on the hot spare back to the replacement drive, then mark the hot spare drive as available again.


Update
: It appears a progress indicator of sorts is included in the lustat command in the SPCollect logs.  By running SPCollects over and over again I can gauge the progress of proactive copies and equalizations when the drive is replaced.

progress_indicator_found

This information is contained within the SPCollect zip file, within the *_sus zip file, within the SPx_cfg_info.txt file.  By looking at the EQZ percentage, I can gauge roughly when it will finish, and more importantly that the equalization or proactive copy is progressing.

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s