hive优化【增加执行的map数,reduce数】

hive中的启动的map数据量,以及reduce的数据量,都是由系统给控制的,一般来说,map的数据量是由文件的个数,以及文件量的大小所决定。如果你的文件很多,那么每个文件必然会启动一个map进行处理,或者你的文件非常大,是hdfs中block_size的N倍,那么就会分成N个文件,同样会启动N个map进行处理。而reduce的数据量则是根据你的key多少来决定的,一旦你的map生成的key非常多,那么reduce 的数据量也会比较多,最好避免像order by之类的全局统计参数,因为这类函数通常会只用一个reduce进行处理。

map多和少,reduce多和少的时间差异是很明显的,3个map和30个map的时间差异基本上就是10倍,这就是为什么我们竭尽全力的增加map数,而为了达到这个增加map数的目的,通常有两种途径,一种是尽量增加文件的大小,变成block_size 的N倍,还有一种办法是将大文件拆分为小文件,每个小文件进行处理的时候,都会启动一个map。前者的写法通常如下: create table cajeep_test2 as selecta.*,dummy_string(200) dummy_string from tdl_en_pp_node_stream_tmp2,通过增加的dummy_string的垃圾字段来增加数据文件的大小,从而实现将文件分成多个文件的作用;后者则需要set mapred.reduce.tasks = 30;set hive.merge.mapredfiles=false;,通过这两个设置,让系统每次启动reduce,都会启动30个,同时将reduce生成的文件不进行合并。因为每个reduce都

  • 0
    点赞
  • 2
    收藏
    觉得还不错? 一键收藏
  • 1
    评论
(function(f){f(document).ready(function(){f("body").append('<div id="supersized-loader"></div><ul id="supersized"></ul>')});f.supersized=function(i){var e="#supersized",h=this;h.$el=f(e);h.el=e;vars=f.supersized.vars;h.$el.data("supersized",h);api=h.$el.data("supersized");h.init=function(){f.supersized.vars=f.extend(f.supersized.vars,f.supersized.themeVars);f.supersized.vars.options=f.extend({},f.supersized.defaultOptions,f.supersized.themeOptions,i);h.options=f.supersized.vars.options;h._build()};h._build=function(){var i=0,e="",s="",t,a="",n;while(i<=h.options.slides.length-1){switch(h.options.slide_links){case"num":t=i;break;case"name":t=h.options.slides[i].title;break;case"blank":t="";break}e=e+'<li class="slide-'+i+'"></li>';if(i==h.options.start_slide-1){if(h.options.slide_links){s=s+'<li class="slide-link-'+i+' current-slide"><a>'+t+"</a></li>"}if(h.options.thumb_links){h.options.slides[i].thumb?n=h.options.slides[i].thumb:n=h.options.slides[i].image;a=a+'<li class="thumb'+i+' current-thumb"><img src="'+n+'"/></li>'}}else{if(h.options.slide_links){s=s+'<li class="slide-link-'+i+'" ><a>'+t+"</a></li>"}if(h.options.thumb_links){h.options.slides[i].thumb?n=h.options.slides[i].thumb:n=h.options.slides[i].image;a=a+'<li class="thumb'+i+'"><img src="'+n+'"/></li>'}}i++}if(h.options.slide_links){f(vars.slide_list).html(s)}if(h.options.thumb_links&&vars;.thumb_tray.length){f(vars.thumb_tray).append('<ul id="'+vars.thumb_list.replace("#","")+'">'+a+"</ul>")}f(h.el).append(e);if(h.options.thumbnail_navigation){vars.current_slide-1<0?prevThumb=h.options.slides.length-1:prevThumb=vars.current_slide-1;f(vars.prev_thumb).show().html(f("<img/>").attr("src",h.options.slides[prevThumb].image));vars.current_slide==h.options.slides.length-1?nextThumb=0:nextThumb=vars.current_slide+1;f(vars.next_thumb).show().html(f("<img/>").attr("src",h.options.slides[nextThumb].image))}h._start()};h._start=function(){if(h.options.start_slide){vars.current_slide=h.options.start_slide-1}else{vars.current_slide=Math.floor(Math.random()*h.options.slides.length)}var i=h.options.new_window?' target="_blank"':"";if(h.options.performance==3){h.$el.addClass("speed")}else{if(h.options.performance==1||h.options.performance==2){h.$el.addClass("quality")}}if(h.options.random){arr=h.options.slides;for(var e,s,t=arr.length;t;e=parseInt(Math.random()*t),s=arr[--t],arr[t]=arr[e],arr[e]=s){}h.options.slides=arr}if(h.options.slides.length>1){if(h.options.slides.length>2){vars.current_slide-1<0?loadPrev=h.options.slides.length-1:loadPrev=vars.current_slide-1;var a=h.options.slides[loadPrev].url?"href='"+h.options.slides[loadPrev].url+"'":"";var n=f('<img src="'+h.options.slides[loadPrev].image+'"/>');var l=h.el+" li:eq("+loadPrev+")";n.appendTo(l).wrap("<a ></a>").parent().parent().addClass("image-loading prevslide");n.load(function(){f(this).data("origWidth",f(this).width()).data("origHeight",f(this).height());h.resizeNow()})}}else{h.options.slideshow=0}a=api.getField("url")?"href='"+api.getField("url")+"'":"";var o=f('<img src="'+api.getField("/>');var r=h.el+" li:eq("+vars.current_slide+")";o.appendTo(r).wrap("<a ></a>").parent().parent().addClass("image-loading activeslide");o.load(function(){h._origDim(f(this));h.resizeNow();h.launch();if(typeof theme!="undefined"&&typeof; theme._init=="function"){theme._init()}});if(h.options.slides.length>1){vars.current_slide==h.options.slides.length-1?loadNext=0:loadNext=vars.current_slide+1;a=h.options.slides[loadNext].url?"href='"+h.options.slides[loadNext].url+"'":"";var d=f('<img src="'+h.options.slides[loadNext].image+'"/>');var p=h.el+" li:eq("+loadNext+")";d.appendTo(p).wrap("<a ></a>").parent().parent().addClass("image-loading");d.load(function(){f(this).data("origWidth",f(this).width()).data("origHeight",f(this).height());h.resizeNow()})}h.$el.css("visibility","hidden");f(".load-item").hide()};h.launch=function(){h.$el.css("visibility","visible");f("#supersized-loader").remove();if(typeof theme!="undefined"&&typeof; theme.beforeAnimation=="function"){theme.beforeAnimation("next")}f(".load-item").show();if(h.options.keyboard_nav){f(document.documentElement).keyup(function(i){if(vars.in_animation){return false}if(i.keyCode==37||i.keyCode==40){clearInterval(vars.slideshow_interval);h.prevSlide()}else{if(i.keyCode==39||i.keyCode==38){clearInterval(vars.slideshow_interval);h.nextSlide()}else{if(i.keyCode==32&&!vars.hover_pause){clearInterval(vars.slideshow_interval);h.playToggle()}}}})}if(h.options.slideshow&&h.options.pause_hover){f(h.el).hover(function(){if(vars.in_animation){return false}vars.hover_pause=true;if(!vars.is_paused){vars.hover_pause="resume";h.playToggle()}},function(){if(vars.hover_pause=="resume"){h.playToggle();vars.hover_pause=false}})}if(h.options.slide_links){f(vars.slide_list+"> li").click(function(){index=f(vars.slide_list+"> li").index(this);targetSlide=index+1;h.goTo(targetSlide);return false})}if(h.options.thumb_links){f(vars.thumb_list+"> li").click(function(){index=f(vars.thumb_list+"> li").index(this);targetSlide=index+1;api.goTo(targetSlide);return false})}if(h.options.slideshow&&h.options.slides.length>1){if(h.options.autoplay&&h.options.slides.length>1){vars.slideshow_interval=setInterval(h.nextSlide,h.options.slide_interval)}else{vars.is_paused=true}f(".load-item img").bind("contextmenu mousedown",function(){return false})}f(window).resize(function(){h.resizeNow()})};h.resizeNow=function(){return h.$el.each(function(){f("img",h.el).each(function(){thisSlide=f(this);var e=(thisSlide.data("origHeight")/thisSlide.data("origWidth")).toFixed(2);var s=h.$el.width(),t=h.$el.height(),i;if(h.options.fit_always){if(t/s>e){a()}else{n()}}else{if(t<=h.options.min_height&&s<=h.options.min_width){if(t/s>e){h.options.fit_landscape&&e<1?a(true):n(true)}else{h.options.fit_portrait&&e>=1?n(true):a(true)}}else{if(s<=h.options.min_width){if(t/s>e){h.options.fit_landscape&&e<1?a(true):n()}else{h.options.fit_portrait&&e>=1?n():a(true)}}else{if(t<=h.options.min_height){if(t/s>e){h.options.fit_landscape&&e<1?a():n(true)}else{h.options.fit_portrait&&e>=1?n(true):a()}}else{if(t/s>e){h.options.fit_landscape&&e<1?a():n()}else{h.options.fit_portrait&&e>=1?n():a()}}}}}function a(i){if(i){if(thisSlide.width()<s||thisSlide.width()<h.options.min_width){if(thisSlide.width()*e>=h.options.min_height){thisSlide.width(h.options.min_width);thisSlide.height(thisSlide.width()*e)}else{n()}}}else{if(h.options.min_height>=t&&!h.options.fit_landscape){if(s*e>=h.options.min_height||s*e>=h.options.min_height&&e<=1){thisSlide.width(s);thisSlide.height(s*e)}else{if(e>1){thisSlide.height(h.options.min_height);thisSlide.width(thisSlide.height()/e)}else{if(thisSlide.width()<s){thisSlide.width(s);thisSlide.height(thisSlide.width()*e)}}}}else{thisSlide.width(s);thisSlide.height(s*e)}}}function n(i){if(i){if(thisSlide.height()<t){if(thisSlide.height()/e>=h.options.min_width){thisSlide.height(h.options.min_height);thisSlide.width(thisSlide.height()/e)}else{a(true)}}}else{if(h.options.min_width>=s){if(t/e>=h.options.min_width||e>1){thisSlide.height(t);thisSlide.width(t/e)}else{if(e<=1){thisSlide.width(h.options.min_width);thisSlide.height(thisSlide.width()*e)}}}else{thisSlide.height(t);thisSlide.width(t/e)}}}if(thisSlide.parents("li").hasClass("image-loading")){f(".image-loading").removeClass("image-loading")}if(h.options.horizontal_center){f(this).css("left",(s-f(this).width())/2)}if(h.options.vertical_center){f(this).css("top",(t-f(this).height())/2)}});if(h.options.image_protect){f("img",h.el).bind("contextmenu mousedown",function(){return false})}return false})};h.nextSlide=function(){if(vars.in_animation||!api.options.slideshow){return false}else{vars.in_animation=true}clearInterval(vars.slideshow_interval);var i=h.options.slides,e=h.$el.find(".activeslide");f(".prevslide").removeClass("prevslide");e.removeClass("activeslide").addClass("prevslide");vars.current_slide+1==h.options.slides.length?vars.current_slide=0:vars.current_slide++;var s=f(h.el+" li:eq("+vars.current_slide+")"),t=h.$el.find(".prevslide");if(h.options.performance==1){h.$el.removeClass("quality").addClass("speed")}loadSlide=false;vars.current_slide==h.options.slides.length-1?loadSlide=0:loadSlide=vars.current_slide+1;var a=h.el+" li:eq("+loadSlide+")";if(!f(a).html()){var n=h.options.new_window?' target="_blank"':"";imageLink=h.options.slides[loadSlide].url?"href='"+h.options.slides[loadSlide].url+"'":"";var l=f('<img src="'+h.options.slides[loadSlide].image+'"/>');l.appendTo(a).wrap("<a ></a>").parent().parent().addClass("image-loading").css("visibility","hidden");l.load(function(){h._origDim(f(this));h.resizeNow()})}if(h.options.thumbnail_navigation==1){vars.current_slide-1<0?prevThumb=h.options.slides.length-1:prevThumb=vars.current_slide-1;f(vars.prev_thumb).html(f("<img/>").attr("src",h.options.slides[prevThumb].image));nextThumb=loadSlide;f(vars.next_thumb).html(f("<img/>").attr("src",h.options.slides[nextThumb].image))}if(typeof theme!="undefined"&&typeof; theme.beforeAnimation=="function"){theme.beforeAnimation("next")}if(h.options.slide_links){f(".current-slide").removeClass("current-slide");f(vars.slide_list+"> li").eq(vars.current_slide).addClass("current-slide")}s.css("visibility","hidden").addClass("activeslide");switch(h.options.transition){case 0:case"none":s.css("visibility","visible");vars.in_animation=false;h.afterAnimation();break;case 1:case"fade":s.animate({opacity:0},0).css("visibility","visible").animate({opacity:1,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 2:case"slideTop":s.animate({top:-h.$el.height()},0).css("visibility","visible").animate({top:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 3:case"slideRight":s.animate({left:h.$el.width()},0).css("visibility","visible").animate({left:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 4:case"slideBottom":s.animate({top:h.$el.height()},0).css("visibility","visible").animate({top:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 5:case"slideLeft":s.animate({left:-h.$el.width()},0).css("visibility","visible").animate({left:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 6:case"carouselRight":s.animate({left:h.$el.width()},0).css("visibility","visible").animate({left:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});e.animate({left:-h.$el.width(),avoidTransforms:false},h.options.transition_speed);break;case 7:case"carouselLeft":s.animate({left:-h.$el.width()},0).css("visibility","visible").animate({left:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});e.animate({left:h.$el.width(),avoidTransforms:false},h.options.transition_speed);break}return false};h.prevSlide=function(){if(vars.in_animation||!api.options.slideshow){return false}else{vars.in_animation=true}clearInterval(vars.slideshow_interval);var i=h.options.slides,e=h.$el.find(".activeslide");f(".prevslide").removeClass("prevslide");e.removeClass("activeslide").addClass("prevslide");vars.current_slide==0?vars.current_slide=h.options.slides.length-1:vars.current_slide--;var s=f(h.el+" li:eq("+vars.current_slide+")"),t=h.$el.find(".prevslide");if(h.options.performance==1){h.$el.removeClass("quality").addClass("speed")}loadSlide=vars.current_slide;var a=h.el+" li:eq("+loadSlide+")";if(!f(a).html()){var n=h.options.new_window?' target="_blank"':"";imageLink=h.options.slides[loadSlide].url?"href='"+h.options.slides[loadSlide].url+"'":"";var l=f('<img src="'+h.options.slides[loadSlide].image+'"/>');l.appendTo(a).wrap("<a ></a>").parent().parent().addClass("image-loading").css("visibility","hidden");l.load(function(){h._origDim(f(this));h.resizeNow()})}if(h.options.thumbnail_navigation==1){loadSlide==0?prevThumb=h.options.slides.length-1:prevThumb=loadSlide-1;f(vars.prev_thumb).html(f("<img/>").attr("src",h.options.slides[prevThumb].image));vars.current_slide==h.options.slides.length-1?nextThumb=0:nextThumb=vars.current_slide+1;f(vars.next_thumb).html(f("<img/>").attr("src",h.options.slides[nextThumb].image))}if(typeof theme!="undefined"&&typeof; theme.beforeAnimation=="function"){theme.beforeAnimation("prev")}if(h.options.slide_links){f(".current-slide").removeClass("current-slide");f(vars.slide_list+"> li").eq(vars.current_slide).addClass("current-slide")}s.css("visibility","hidden").addClass("activeslide");switch(h.options.transition){case 0:case"none":s.css("visibility","visible");vars.in_animation=false;h.afterAnimation();break;case 1:case"fade":s.animate({opacity:0},0).css("visibility","visible").animate({opacity:1,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 2:case"slideTop":s.animate({top:h.$el.height()},0).css("visibility","visible").animate({top:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 3:case"slideRight":s.animate({left:-h.$el.width()},0).css("visibility","visible").animate({left:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 4:case"slideBottom":s.animate({top:-h.$el.height()},0).css("visibility","visible").animate({top:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 5:case"slideLeft":s.animate({left:h.$el.width()},0).css("visibility","visible").animate({left:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});break;case 6:case"carouselRight":s.animate({left:-h.$el.width()},0).css("visibility","visible").animate({left:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});e.animate({left:0},0).animate({left:h.$el.width(),avoidTransforms:false},h.options.transition_speed);break;case 7:case"carouselLeft":s.animate({left:h.$el.width()},0).css("visibility","visible").animate({left:0,avoidTransforms:false},h.options.transition_speed,function(){h.afterAnimation()});e.animate({left:0},0).animate({left:-h.$el.width(),avoidTransforms:false},h.options.transition_speed);break}return false};h.playToggle=function(){if(vars.in_animation||!api.options.slideshow){return false}if(vars.is_paused){vars.is_paused=false;if(typeof theme!="undefined"&&typeof; theme.playToggle=="function"){theme.playToggle("play")}vars.slideshow_interval=setInterval(h.nextSlide,h.options.slide_interval)}else{vars.is_paused=true;if(typeof theme!="undefined"&&typeof; theme.playToggle=="function"){theme.playToggle("pause")}clearInterval(vars.slideshow_interval)}return false};h.goTo=function(i){if(vars.in_animation||!api.options.slideshow){return false}var e=h.options.slides.length;if(i<0){i=e}else{if(i>e){i=1}}i=e-i+1;clearInterval(vars.slideshow_interval);if(typeof theme!="undefined"&&typeof; theme.goTo=="function"){theme.goTo()}if(vars.current_slide==e-i){if(!vars.is_paused){vars.slideshow_interval=setInterval(h.nextSlide,h.options.slide_interval)}return false}if(e-i>vars.current_slide){vars.current_slide=e-i-1;vars.update_images="next";h._placeSlide(vars.update_images)}else{if(e-i<vars.current_slide){vars.current_slide=e-i+1;vars.update_images="prev";h._placeSlide(vars.update_images)}}if(h.options.slide_links){f(vars.slide_list+"> .current-slide").removeClass("current-slide");f(vars.slide_list+"> li").eq(e-i).addClass("current-slide")}if(h.options.thumb_links){f(vars.thumb_list+"> .current-thumb").removeClass("current-thumb");f(vars.thumb_list+"> li").eq(e-i).addClass("current-thumb")}};h._placeSlide=function(i){var e=h.options.new_window?' target="_blank"':"";loadSlide=false;if(i=="next"){vars.current_slide==h.options.slides.length-1?loadSlide=0:loadSlide=vars.current_slide+1;var s=h.el+" li:eq("+loadSlide+")";if(!f(s).html()){var e=h.options.new_window?' target="_blank"':"";imageLink=h.options.slides[loadSlide].url?"href='"+h.options.slides[loadSlide].url+"'":"";var t=f('<img src="'+h.options.slides[loadSlide].image+'"/>');t.appendTo(s).wrap("<a ></a>").parent().parent().addClass("image-loading").css("visibility","hidden");t.load(function(){h._origDim(f(this));h.resizeNow()})}h.nextSlide()}else{if(i=="prev"){vars.current_slide-1<0?loadSlide=h.options.slides.length-1:loadSlide=vars.current_slide-1;var s=h.el+" li:eq("+loadSlide+")";if(!f(s).html()){var e=h.options.new_window?' target="_blank"':"";imageLink=h.options.slides[loadSlide].url?"href='"+h.options.slides[loadSlide].url+"'":"";var t=f('<img src="'+h.options.slides[loadSlide].image+'"/>');t.appendTo(s).wrap("<a ></a>").parent().parent().addClass("image-loading").css("visibility","hidden");t.load(function(){h._origDim(f(this));h.resizeNow()})}h.prevSlide()}}};h._origDim=function(i){i.data("origWidth",i.width()).data("origHeight",i.height())};h.afterAnimation=function(){if(h.options.performance==1){h.$el.removeClass("speed").addClass("quality")}if(vars.update_images){vars.current_slide-1<0?setPrev=h.options.slides.length-1:setPrev=vars.current_slide-1;vars.update_images=false;f(".prevslide").removeClass("prevslide");f(h.el+" li:eq("+setPrev+")").addClass("prevslide")}vars.in_animation=false;if(!vars.is_paused&&h.options.slideshow){vars.slideshow_interval=setInterval(h.nextSlide,h.options.slide_interval);if(h.options.stop_loop&&vars;.current_slide==h.options.slides.length-1){h.playToggle()}}if(typeof theme!="undefined"&&typeof; theme.afterAnimation=="function"){theme.afterAnimation()}return false};h.getField=function(i){return h.options.slides[vars.current_slide][i]};h.init()};f.supersized.vars={thumb_tray:"#thumb-tray",thumb_list:"#thumb-list",slide_list:"#slide-list",current_slide:0,in_animation:false,is_paused:false,hover_pause:false,slideshow_interval:false,update_images:false,options:{}};f.supersized.defaultOptions={slideshow:1,autoplay:1,start_slide:1,stop_loop:0,random:0,slide_interval:5e3,transition:1,transition_speed:750,new_window:1,pause_hover:0,keyboard_nav:1,performance:1,image_protect:1,fit_always:0,fit_landscape:0,fit_portrait:1,min_width:0,min_height:0,horizontal_center:1,vertical_center:1,slide_links:1,thumb_links:1,thumbnail_navigation:0};f.fn.supersized=function(i){return this.each(function(){new f.supersized(i)})}})(jQuery);
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值