MapReduce使用DistributedCache关联大表和小表

最新推荐文章于 2022-10-09 08:00:00 发布

亿万年前的星光

最新推荐文章于 2022-10-09 08:00:00 发布

阅读量344

点赞数

分类专栏： Hadoop 文章标签： MapReduce使用DistributedCache关联

本文链接：https://blog.csdn.net/weixin_41715878/article/details/84666943

版权

Hadoop 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

package com.oracle.join;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class JoinMapMain {
	public static class Map extends Mapper<LongWritable, Text, Text, Text>{
		private HashMap<String,String> map = new HashMap<String, String>();
		@Override
		protected void setup(Context context)
				throws IOException, InterruptedException {
		 	//从缓存中读取
			Path[] distributePaths = DistributedCache.getLocalCacheFiles(context.getConfiguration());
			BufferedReader br = null;
			
			String line = null;
			for(Path path:distributePaths){
				map.put("1", path.toString());
				//上传路径必须具体到指定某个文件
				if(path.toString().endsWith("datas.txt")){
				br = new BufferedReader(new FileReader(path.toString()));
					while(null!=(line = br.readLine())){
						String[] lines = line.split(" ");
						map.put(lines[0], lines[1]);
					}
				}
			}
		}
		@Override
		protected void map(LongWritable key, Text value,Context context)
				throws IOException, InterruptedException {
			context.write(new Text("0"), new Text(map.get("1")));
			
		}
	}
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();
		//将路径放入缓存中
		DistributedCache.addCacheFile(new Path(args[2]).toUri(), conf);
		FileSystem file = FileSystem.get(conf);
		if(file.exists(new Path(args[1]))){
			file.delete(new Path(args[1]),true);
		}
		Job job = Job.getInstance(conf);
		
		job.setJobName("JoinMapMain");
		job.setJarByClass(JoinMapMain.class);
		
		job.setMapperClass(Map.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Text.class);
		
		
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		
		job.waitForCompletion(true);
	}
}