Hadoop WorldCount程序
--aaa.txt
hello world hadoop
hello lhj hadoop
good luck to lhj
nice to me lhj
--Mapper
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class WcMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
@Override
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
String line=value.toString();
StringTokenizer st=new StringTokenizer(line);
while(st.hasMoreTokens()){
String world=st.nextToken();
context.write(new Text(world), new IntWritable(1));
}
}
}
--Reducer
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class WcReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> iterable,Context context)
throws IOException, InterruptedException {
int sum=0;
for(IntWritable i:iterable){
sum=sum+i.get();
}
context.write(key, new IntWritable(sum));
}
}
--JobRun
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class JobRun {
public static void main(String[] args) {
Configuration conf=new Configuration();
// conf.set("fs.default.name", "hdfs://node1:9000");
// conf.set("mapred.job.tracker", "node1:9001");
// conf.set("mapred.jar", "C:\\Documents and Settings\\Administrator\\桌面");
try {
Job job=new Job(conf);
job.setJarByClass(JobRun.class);
job.setMapperClass(WcMapper.class);
job.setReducerClass(WcReduce.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setNumReduceTasks(1);//默认也是1
FileInputFormat.addInputPath(job, new Path("/user/hadoop/input/wc"));
FileOutputFormat.setOutputPath(job, new Path("/user/hadoop/output/wc"));
System.exit(job.waitForCompletion(true) ? 0:1);
} catch (Exception e) {
e.printStackTrace();
}
}
}
--上传jar包
hadoop fs -put wc.jar /user/hadoop/input/wc
--hadoop中执行
hadoop jar wc.jar JobRun //wc.jar是导出的jar包,如果有包的话要加上包名, JobRun是有main方法的类
--监控执行的进度
http://node1:50030/jobtracker.jsp
[hadoop@node1 ~]$ hadoop fs -cat /user/hadoop/output/wc/part-r-00000
good 1
hadoop 2
hello 2
lhj 3
luck 1
me 1
nice 1
to 2
world 1
Hadoop WorldCount程序
最新推荐文章于 2020-09-08 19:04:24 发布