myjob.properties:
IN_PATH=/mrtest/in
OUT_PATH=/mrtest/out/rs.txt
MAPPER_CLASS=com.mydemo.mr.WordCountMapper
1.HdfsWordCount:
public class HdfsWordCount {
public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException, ClassNotFoundException, InstantiationException, IllegalAccessException {
// TODO Auto-generated method stub
//反射,加载配置信息
Properties pro = new Properties();
pro.load(HdfsWordCount.class.getClassLoader().getResourceAsStream("myjob.properties"));
Path inpath = new Path(pro.getProperty("IN_PATH"));
Path outpath = new Path(pro.getProperty("OUT_PATH"));
//构造工具类
Context con_map = new Context();
Class<?> Mapper = Class.forName(pro.getProperty("MAPPER_CLASS"));
WordCountMapper Map = (WordCountMapper) Mapper.newInstance();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://192.168.232.132:9000"), conf, "root");
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(inpath, false);
while(listFiles.hasNext()) {
LocatedFileStatus file = listFiles.next();
FSDataInputStream in = fs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(in, "utf-8"));
String read = null;
while((read=br.readLine())!=null) {
Map.Mapper_map(read, con_map);
}
br.close();
in.close();
}
Path Out = new Path("/mrtest/out");
if(fs.exists(Out)) {
fs.mkdirs(Out);
}
HashMap<Object, Object> contextMap = con_map.get_Conmap();
FSDataOutputStream out1 = fs.create(outpath);
//遍历hashmap
Set<Entry<Object, Object>> entrySet = contextMap.entrySet();
for(Entry<Object, Object> entry:entrySet) {
//写数据
out1.write((entry.getKey().toString()+"\t"+entry.getValue()+"\n").getBytes());
}
out1.close();
fs.close();
System.out.println("数据统计结果完成....");
}
}
2.Mapper:
public interface Mapper {
public void Mapper_map(String line, Context con);
}
3.WordCountMapper:
public class WordCountMapper implements Mapper {
@Override
public void Mapper_map(String line, Context con) {
// TODO Auto-generated method stub
String [] line_sq = line.split(" ");
for(String k: line_sq) {
Object key = con.read_map(k);
if(key==null) {
con.write_map(k, 1);
}else {
Object va = con.read_map(k);
int value = (int) va;
con.write_map(k, value+1);
}
}
}
}
4.Context:
public class Context {
private HashMap<Object, Object> hm = new HashMap<>();
public void write_map(Object key, Object value) {
hm.put(key, value);
}
public Object read_map(Object key) {
return hm.get(key);
}
public HashMap<Object, Object> get_Conmap(){
return hm;
}
}