编写java文件:
项目驱动:
package com.hadoop.weblog;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* 该项目的 驱动类
* @author naixi
*
*/
public class WebLogDriver extends Configured implements Tool {
public static void main(String[] args) {
Configuration conf = new Configuration();
try {
int status = ToolRunner.run(conf, new WebLogDriver(), args);
System.exit(status);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public int run(String[] args) throws Exception {
Job job = Job.getInstance(this.getConf(),"baidu-test");
job.setJarByClass(WebLogDriver.class);
//input
Path inputPaths = new Path(args[0]);
FileInputFormat.setInputPaths(job, inputPaths);
//map
job.setMapperClass(WebLogMapper.class);
job.setMapOutputKeyClass(WebLogWritable.class);
job.setMapOutputValueClass(NullWritable.class);
//shuffle
//reduce
//我们在这里项目案例中 , 我们做的就是数据清洗 清洗一条输出一条 不需要聚合
job.setNumReduceTasks(0); //将reduce任务设置为0
//output
Path Paths = new Path(args[1]);
FileSystem fileSystem = FileSystem.get(this.getConf()); //实例化一个Hadoop的文件系统对象
if(fileSystem.exists(Paths)) { //如果这个文件夹存在就删除
fileSystem.delete(Paths,true); //使用递归删除该文件夹
}
FileOutputFormat.setOutputPath(job, Paths);
boolean flag = job.waitForCompletion(t