Hadoop 3.4.0 项目实战

1环境基于 上一篇搭建

高可用分布式集群

2  官方提供MapReduce程序

#评估圆周率

cd  /data/hadoop/share/hadoop/mapreduce/
hadoop  jar  hadoop-mapreduce-examples-3.4.0.jar  pi 2 6

 

 

 3 实例项目分析1

#预分析的文件如，如单词统计

#

#上传文件到hdfs
hdfs dfs  -put word.txt /test/01/
#可以先测试下，在运行计算
[root@master11 01]# cat word.txt   |  python  m.py |python r.py
foo	1
quux	1
labs	1
foo	1
bar	1
quux	1
good	1
six	1
good	1
foo	2
quux	1
labs	1
foo	1
bar	1
quux	1
good	1
six	1
good	1
foo	1
# hadoop jar /data/hadoop/share/hadoop/tools/lib/hadoop-streaming-3.4.0.jar  -D stream.non.zero.exit.is.failure=false -mapper /data/test/01/m.py  -reducer /data/test/01/r.py   -input /test/01/  -output /test/output1/

 

 #拷贝文件到服务器本地

hadoop fs -copyToLocal /test/output1/part-00000 /root/part-00000
#查看
[root@master11 ~]# cat  part-00000 
bar	2
foo	6
good	4
labs	2
quux	4
six	2

[root@master11 01]# cat m.py 
#!/usr/bin/env python  
  
import sys  
  
for line in sys.stdin:  
    line = line.strip()  
    words = line.split()  
    for word in words:  
        print '%s\t%s' % (word, 1)
[root@master11 01]# cat r.py 
#!/usr/bin/env python  
  
from operator import itemgetter  
import sys  
  
current_word = None  
current_count = 0  
word = None  
  
for line in sys.stdin:  
    line = line.strip()  
  
    word, count = line.split('\t', 1)  
  
    try:  
        count = int(count)  
    except ValueError:  
        continue  
  
    if current_word == word:  
        current_count += count  
    else:  
        if current_word:  
            print '%s\t%s' % (current_word, current_count)  
        current_count = count  
        current_word = word  
  
if current_word == word:  
    print '%s\t%s' % (current_word, current_count)

 

4项目实战分析2，nginx日志 IP计数

 

hdfs dfs  -put t.log /test/01/
#执行计算
hadoop jar /data/hadoop/share/hadoop/tools/lib/hadoop-streaming-3.4.0.jar  -D stream.non.zero.exit.is.failure=false -mapper /data/test/01/map.py  -reducer /data/test/01/red.py   -input /test/01/t.log  -output /test/output2/

#下载
hadoop fs -copyToLocal /test/output2/part-00000 /root/part-00000
#查看

 

[root@master11 01]# cat  map.py 
#!/usr/bin/python
import sys
import re
for line in sys.stdin:  
    ipaddress=re.compile(r'([\d.]*)')
    match=ipaddress.match(line)  
    if match:
        ip=match.group(1)
        print ' %s\t%s' % (ip, 1)
[root@master11 01]# cat red.py 
#!/usr/bin/python
#-*-coding:UTF-8 -*-
import sys
import os
import string
res = {}
for line in sys.stdin:
    skey=line[0:-1]
    if(res.has_key(skey)==False): 
        res[skey]=0
    res[skey]=res[skey]+1
for key in res.keys():
     print key+"\t"+str(res[key])

5  欢迎同学们一起交流