hadoop个人理解

 

2 hadoop三种方式

Local:

需要改hadoop-site.xml—的java——home

Hadoop-default是主要配置，不需要更改

命令：

hadoop/bin/hadoop demo.hadoop.HadoopGrep log文件所在目录 任意的输出目录 grep的字符串

3 重写例子

package com.hadoop.example; 

import java.io.File;
import java.util.ArrayList;
import java.util.List;

import org.apache.catalina.filters.AddDefaultCharsetFilter;

import com.sun.org.apache.bcel.internal.generic.NEW;


public class HReal {

	public static void main(String[] args) {
		
	}
	public void work(){
		JobConf conf = null;
		Mapper mapper = new Mapper();
		Reducer reducer = new Reducer();
		InputFormat inputFormat = new InputFormat();
		OutputFromat outputFromat = new OutputFromat();
		String inpathString ="";
		String outpathString="";
		
		JobTracker jobTracker = new JobTracker();
		Job job = new Job();
		jobTracker.addJob(job);
		if(job.flg=true){
			
		}
		
	}
}

class Mapper{
	private String kString;
	private String vString;
	public void deal(Class c){
		
	}
	private void deal(String s){
		
	}
	
	
	public SequenceFile deal(String kString,String vString){
		this.kString=kString;
		this.vString=vString;
		return new SequenceFile();
	}
}
class SequenceFile{
	
}
class Reducer{
	private String resultString;
	private String middleString;
	public  String  deal(){
		return resultString;
	}
}
class InputFormat{
	
	public RecordReader newReader(){
		return new RecordReader();
	}
	
	public Object format(String line){
		return line;
	}
	
	public Object  format(String line, Object type){
		if(type instanceof String){
			return line;
		}
		if(type instanceof Class){
			return line.getClass();
		}
		return line;
	}
	
	public FileSplist getSplits(String pathfile){
		File file=new File(pathfile);
		//分割文件处理
		return new FileSplist();
	}
}
class FileSplist{
	
}
class OutputFromat{
	public void write(String result){
		
	}
}
class NameNode{
	
}
class DataNode{
	
}
class Master{
	private JobTracker jobTracker;
	private NameNode nameNode;
}
class JobTracker{
	private MapReduce mapReduce;
	private InputFormat inputFormat;
	List<Job> jobs = new ArrayList<Job>();
	public void addJob(Job job){
		jobs.add(job);
	}
	public void nextJob(){
		
		jobs.get(this.getCurrentFlg());
	}
	public int getCurrentFlg(){
		return 1;
	}
	public void currState(){
		
	}
}
class MapReduce{
	
}
class JobConf{
	
}
class Job{
	static boolean flg = false;
	public void run(){
		System.out.println("deal");
		flg=true;
	}
}
class MapperTask{
	List<FileSplist> lst = new ArrayList<FileSplist>();
	InputFormat inputFormat;
	public void addTask(FileSplist fileSplist){
		lst.add(fileSplist);
	}
	
	public void read(){
//		lst.get(0)
	}
}
class Queue{
	List lst = new ArrayList();
	public void addMapperTask(MapperTask mapperTask){
		lst.add(mapperTask);
	}
}
class TaskTracker{
	int taskCount=0;
	JobTracker jobTracker;
	
	public void nextMapReduce(){
		jobTracker.nextJob();
	}
	public void deal(){
		
	}
	//定时报告任务情况
	public void schele(){
		System.out.println("当前任务完成了多少了");
		jobTracker.currState();
		if(taskCount ==3){
			nextMapReduce();
		}
		try {
			Thread.sleep(10000);
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
}
class RecordReader{
	
}
class ReducerTask{
	Reducer reducer;
	OutputFromat outputFromat;
	public void sort(){
		
	}
	public void merge(){
		
	}
	public void deal(){
		sort();
		merge();
		String resultString = reducer.deal();
		outputFromat.write(resultString);
	}
}