hadoop客户端提交job的java代码

最新推荐文章于 2021-02-23 19:04:01 发布

madbluesky

最新推荐文章于 2021-02-23 19:04:01 发布

阅读量381

点赞数

分类专栏： java 文章标签：大数据 java

本文链接：https://blog.csdn.net/madbluesky/article/details/84473670

版权

java 专栏收录该内容

14 篇文章 1 订阅

订阅专栏


package com.xhb1.test;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import com.test.UserBroserCount;

public class RunBroserCount {
	public static void main(String[] args) throws Exception {
		String inputPath = "/logdata/userLogFile.log";
		String outputPath = "/oozie/test1";
		new RunBroserCount().runBroserCount(inputPath, outputPath);
	}

	public void runBroserCount(String inputPath, String outputPath)
			throws IOException, InterruptedException, ClassNotFoundException {
		Configuration conf = new Configuration();
		conf.set("fs.default.name", "hdfs://192.168.0.170:4310");
		//conf.set("hadoop.job.user", "root");
		conf.set("mapred.job.tracker", "192.168.0.170:4311");
		Job job = new Job(conf, "browserCount");
		((JobConf) job.getConfiguration()).setJar("E:/workspaces/pmp/BrowserCount/target/pr.jar");  
		job.setJarByClass(UserBroserCount.class);
		job.setMapOutputValueClass(IntWritable.class);
		job.setMapperClass(UserBroserCount.UserBroserCountMapper.class);
		job.setReducerClass(UserBroserCount.UserBroserCountReducer.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		FileInputFormat.addInputPath(job, new Path(inputPath));
		FileOutputFormat.setOutputPath(job, new Path(outputPath));
		job.waitForCompletion(true);

	}

}