日志

创建目录
mkdir -p /letv/data/dsj/logs

upload.sh

#!/usr/bin/env bash
log_date=`date +%Y%m%d`
scp  /letv/logs/nginx/api.ibestv.com/log_backup/access_${log_date}_*.log root@ip:/letv/data/dsj/logs/${log_date}/
ssh ip /letv/data/hadoop-2.6.0/bin/hadoop fs -put /letv/data/dsj/logs/${log_date}/access_${log_date}_*.log /user/${log_date}

定时任务
crontab -e

0 0 * * * /letv/data/upload.sh

python

import sys
import pyhdfs
from hdfs.hfile import Hfile

hostname = 'ip-'
port = 9000
hdfs_path = '/user'
local_path = '/letv/logs/nginx/api.ibestv.com/log_backup/access_20160428_000001.log'

# Let's open local and HDFS files.

hfile = Hfile(hostname, port, hdfs_path, mode='w')
fh = open(local_path)

# Now we'll write lines from a local file into the HDFS file.
for line in fh:
  hfile.write(line)

# And close them.
fh.close()
hfile.close()

自动启动flume脚本

<!--干掉flume-->
kill -9 $(jps -l|grep flume)

hive 中的表

CREATE TABLE if not exists nglog_analyze(  
  remote_addr STRING,  
  http_marketChannelName STRING,  
  http_hwMac STRING,  
  http_hwModel STRING,  
  http_appVerName STRING,  
  time_local STRING, 
  request_uri STRING,   
  status STRING,  
  body_bytes_sent STRING,
  bytes_sent STRING,
  gzip_ratio STRING,
  http_referer STRING,
  http_cookie STRING,
  http_user_agent STRING, 
  http_x_forwarded_for STRING,
  upstream_addr STRING, 
  upstream_response_time STRING,
  upstream_status STRING,
  request_time STRING
  )  
ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n' STORED AS TEXTFILE; 

正式的日志

CREATE TABLE if not exists nglog_analyze3(  
  host STRING,  
  uri STRING,  
  request_method STRING,  
  remote_addr STRING,  
  http_marketChannelName STRING,  
  http_hwMac STRING, 
  http_ethMac STRING,   
  http_hwModel STRING,  
  http_appVerName STRING,
  time_local_status STRING,
  body_bytes_sent STRING,
  bytes_sent STRING,
  gzip_ratio STRING,
  http_referer STRING, 
  http_cookie STRING,
  http_user_agent STRING, 
  http_x_forwarded_for STRING,
  upstream_addr STRING,
  upstream_response_time STRING,
  upstream_status STRING,
  request_time STRING
  )  
ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n' STORED AS TEXTFILE; 

LOAD DATA INPATH '/flume/20160506' INTO TABLE nglog_analyze3;
select * from nglog_analyze3 limit 10;

日志分割
http://coolnull.com/79.html
http://www.1987.name/682.html