hadoop 集群执行java_本地idea开发mapreduce程序提交到远程hadoop集群执行

package com;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Counter;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.GenericOptionsParser;

import org.apache.hadoop.util.StringUtils;

import java.io.BufferedReader;

import java.io.FileReader;

import java.io.IOException;

import java.net.URI;

import java.util.*;/**

* @author wangxiaolei(王小雷)

* @since 2018/11/22*/

public classwc {public static classTokenizerMapper

extends Mapper{static enumCountersEnum { INPUT_WORDS }private final static IntWritable one = new IntWritable(1);private Text word = newText();privateboolean caseSensitive;private Set patternsToSkip = new HashSet();privateConfiguration conf;privateBufferedReader fis;

@Overridepublic voidsetup(Context context) throws IOException,

InterruptedException {

conf=context.getConfiguration();

caseSensitive= conf.getBoolean("wordcount.case.sensitive", true);if (conf.getBoolean("wordcount.skip.patterns", false)) {

URI[] patternsURIs=Job.getInstance(conf).getCacheFiles();for(URI patternsURI : patternsURIs) {

Path patternsPath= newPath(patternsURI.getPath());

String patternsFileName=patternsPath.getName().toString();

parseSkipFile(patternsFileName);

}

}

}private voidparseSkipFile(String fileName) {try{

fis= new BufferedReader(newFileReader(fileName));

String pattern= null;while ((pattern = fis.readLine()) != null) {

patternsToSkip.add(pattern);

}

}catch(IOException ioe) {

System.err.println("Caught exception while parsing the cached file ‘"

+StringUtils.stringifyException(ioe));

}

}

@Overridepublic voidmap(Object key, Text value, Context context

) throws IOException, InterruptedException {

String line= (caseSensitive) ?value.toString() : value.toString().toLowerCase();for(String pattern : patternsToSkip) {

line= line.replaceAll(pattern, "");

}

StringTokenizer itr= newStringTokenizer(line);while(itr.hasMoreTokens()) {

word.set(itr.nextToken());

context.write(word, one);

Counter counter= context.getCounter(CountersEnum.class.getName(),

CountersEnum.INPUT_WORDS.toString());

counter.increment(1);

}

}

}public static classIntSumReducer

extends Reducer{private IntWritable result = newIntWritable();public void reduce(Text key, Iterablevalues,

Context context

) throws IOException, InterruptedException {int sum = 0;for(IntWritable val : values) {

sum+= val.get();

}

result.set(sum);

context.write(key, result);

}

}public static voidmain(String[] args) throws Exception {

Configuration conf= newConfiguration();

conf.set("yarn.resourcemanager.address", "192.168.56.101:8050");

conf.set("mapreduce.framework.name", "yarn");

conf.set("fs.defaultFS", "hdfs://vbusuanzi:9000/");//conf.set("mapred.jar", "mapreduce/build/libs/mapreduce-0.1.jar");//也可以在这里设置刚刚编译好的jar

conf.set("mapred.job.tracker", "vbusuanzi:9001");//conf.set("mapreduce.app-submission.cross-platform", "true");//Windows开发者需要设置跨平台

args = new String[]{"/tmp/test/LICENSE.txt","/tmp/test/out30"};

GenericOptionsParser optionParser= newGenericOptionsParser(conf, args);

String[] remainingArgs=optionParser.getRemainingArgs();if ((remainingArgs.length != 2) && (remainingArgs.length != 4)) {

System.err.println("Usage: wordcount [-skip skipPatternFile]");

System.exit(2);

}

Job job= Job.getInstance(conf,"test");

job.setJar("mapreduce/build/libs/mapreduce-0.1.jar");

job.setJarByClass(com.wc.class);

job.setMapperClass(TokenizerMapper.class);

job.setCombinerClass(IntSumReducer.class);

job.setReducerClass(IntSumReducer.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

List otherArgs = new ArrayList();for (int i=0; i < remainingArgs.length; ++i) {if ("-skip".equals(remainingArgs[i])) {

job.addCacheFile(new Path(remainingArgs[++i]).toUri());

job.getConfiguration().setBoolean("wordcount.skip.patterns", true);

}else{

otherArgs.add(remainingArgs[i]);

}

}

FileInputFormat.addInputPath(job,new Path(otherArgs.get(0)));

FileOutputFormat.setOutputPath(job,new Path(otherArgs.get(1)));

job.waitForCompletion(true);

System.exit(job.waitForCompletion(true) ? 0 : 1);

}

}