apache pig 读取日志处理，取不固定列最后一段数据

pig 脚本

--加载注册所需的 jar 包和方法

REGISTER /usr/hdp/current/pig-client/piggybank.jar;
set  phoenix.schema.isNamespaceMappingEnabled true;
REGISTER /usr/hdp/current/phoenix-client/phoenix-client.jar;


-- source
--提取完整的日志信息
--load log file
	 org_log = load '/staging/logdata/log_*' USING PigStorage('|') as
            (server_name:chararray, http_x_forwarded_for:chararray, request:chararray, request_time:double, 	upstream_response_time:double, upstream_addr:chararray, status:int,
                            body_bytes_sent:int, http_referer:chararray, http_user_agent:chararray, remote_user:chararray, time_local:chararray, args:chararray, body_bytes_sents:chararray, content_length:long, content_type:chararray, uri:chararray, request_uri:chararray, host:chararray, http_host:chararray, http_cookie:chararray, remote_addr:chararray, upstream_cookie_corpsession_id:chararray, request_id:chararray
);

--从所有信息筛选出需要的字段
message = FOREACH org_log GENERATE server_name, request,request_uri, request_time, upstream_response_time, upstream_addr, status, body_bytes_sent, time_local,remote_addr, request_id;

--因为是日志时间，可能会存在错误 对时间做一个判断，筛选出可以处理的日志信息
split message into message_error if (INDEXOF(time_local,'+0800')==-1),message_filter_date if(INDEXOF(time_local,'+0800')!=-1);


--从完整的端口路径里面用 strsplit 方法取出各自不同的 path services 等，整列值比较复杂，就用多个 STRSPLIT（）方法挨个取
requests = FOREACH message_filter_date GENERATE server_name,request,(chararray)STRSPLIT(request,' ',7).$0 as 	request_mode,(chararray)STRSPLIT((STRSPLIT(request,' ',9).$1),'\\?',2).$0 as interface,(chararray)STRSPLIT((STRSPLIT(request,' ',7).$1),'/',7).$1 as path,(chararray)STRSPLIT((STRSPLIT((STRSPLIT(request,' ',7).$1),'/',7).$2),'\\?',2).$0  as service,(chararray)STRSPLIT((STRSPLIT(request,' ',7).$1),'/',7).$6 as short_interface,request_time,upstream_response_time,upstream_addr,(chararray)STRSPLIT(upstream_addr,':',2).$0 as up_ip,(chararray)STRSPLIT(upstream_addr,':',2).$1 as up_port,status,body_bytes_sent,time_local,remote_addr,request_id;



--由于路径长度不等，端口名称所在位置不同，所以只能写一个方法专门取 s_interface 这个值（用 last_index_of 找出最后一个 / 所在位置，作为 substring 方法截取数据的起始点，再用 SIZE 求出整个长度来作为终点位置，刚好取出位于路径最后一个字段的值
requests_plus = FOREACH requests {
t1 = LAST_INDEX_OF(interface,'/');
bb = (chararray) SUBSTRING(interface,(int)t1+1,(int)SIZE(interface));
GENERATE server_name,request,request_mode,interface,path,service,bb as short_interface,request_time,upstream_response_time,upstream_addr,up_ip,up_port,status,body_bytes_sent,time_local,remote_addr,request_id;}


--对时间做处理，整理出需要时时间和 key
cr_time = FOREACH requests_plus GENERATE server_name..time_local, ToDate(STRSPLIT(time_local,' ',2).$0,'dd/MMM/yyyy:HH:mm:ss') as date_time,remote_addr,request_id;

cr_datetime = FOREACH cr_time GENERATE server_name..time_local,date_time, (chararray)GetYear(date_time) as year,(chararray)GetMonth(date_time)as month,(chararray)GetDay(date_time) as day,GetHour(date_time)as hour,GetMinute(date_time) as minute,remote_addr,request_id;



cr_dtk = FOREACH cr_datetime GENERATE request_id, server_name,request_mode,path,service,interface,short_interface,request_time,upstream_response_time,up_ip,up_port,status,body_bytes_sent,(int)CONCAT(CONCAT(year,(SIZE(month)==2?month:CONCAT('0',month))),(SIZE(day)==2?day:CONCAT('0',day))) as datekey,hour*60+(int)FLOOR(minute/5)*5 as timekey,time_local as date_time,'m' as source,remote_addr;




STORE cr_dtk into 'hbase://###.***' using org.apache.phoenix.pig.PhoenixHBaseStorage('de01.etonedu.cn','-batchSize 500');