一、前端实现 -- 按照时间段查询通话记录
-----------------------------------------
1.完善calllog的dao类calllog.class
----------------------------------------------
package com.it18zhang.ssm.domain;
/**
* calllog的domain类 -- 标准javabean
*/
public class Calllog {
private String caller;
private String callee;
private String callTime;
private String callDuration;
//是否是主叫
private boolean flag;
public boolean isFlag() {
return flag;
}
public void setFlag(boolean flag) {
this.flag = flag;
}
public String getCaller() {
return caller;
}
public void setCaller(String caller) {
this.caller = caller;
}
public String getCallee() {
return callee;
}
public void setCallee(String callee) {
this.callee = callee;
}
public String getCallTime() {
return callTime;
}
public void setCallTime(String callTime) {
this.callTime = callTime;
}
public String getCallDuration() {
return callDuration;
}
public void setCallDuration(String callDuration) {
this.callDuration = callDuration;
}
}
2.通话日志时间查询dao类CalllogRange.class
-----------------------------------------------
package com.it18zhang.ssm.domain;
/**
*/
public class CalllogRange {
private String startPoint ;
private String endPoint ;
public String getStartPoint() {
return startPoint;
}
public void setStartPoint(String startPoint) {
this.startPoint = startPoint;
}
public String getEndPoint() {
return endPoint;
}
public void setEndPoint(String endPoint) {
this.endPoint = endPoint;
}
public String toString() {
return startPoint + " - " + endPoint ;
}
}
3.完善Service类 CalllogService.class
------------------------------------------------------
package com.it18zhang.ssm.service;
import com.it18zhang.ssm.domain.Calllog;
import com.it18zhang.ssm.domain.CalllogRange;
import java.util.List;
/**
* Calllog的服务类 -- 用于定制与服务器交互的规则
*/
public interface CalllogService {
//查询所有的calllog
public List<Calllog> findAll();
/**
* 按照范围查询通话记录
*/
public List<Calllog> findCallogs(String call,List<CalllogRange> list);
}
4.完善CalllogServiceImpl类
-----------------------------------------------
package com.it18zhang.ssm.service.impl;
import com.it18zhang.ssm.domain.Calllog;
import com.it18zhang.ssm.domain.CalllogRange;
import com.it18zhang.ssm.service.CalllogService;
import com.it18zhang.ssm.util.CalllogUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.*;
/**
* CalllogService的实现类
*/
@Service("calllogService")
public class CalllogServiceImpl implements CalllogService {
private Table table;
public CalllogServiceImpl()
{
try {
//获取配置文件
Configuration conf = HBaseConfiguration.create();
//工厂类创建连接
Connection conn = ConnectionFactory.createConnection(conf);
//get table
TableName tbName = TableName.valueOf("call:calllogs");
table = conn.getTable(tbName);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 查询所有的calllog
* 全表扫描
* @return
*/
public List<Calllog> findAll() {
List<Calllog> list = new ArrayList<Calllog>();
try {
//扫描
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
Iterator<Result> it = rs.iterator();
byte[] famliy = Bytes.toBytes("f1");
byte[] callerf = Bytes.toBytes("caller");
byte[] calleef = Bytes.toBytes("callee");
byte[] callTimef = Bytes.toBytes("callTime");
byte[] callDurationf = Bytes.toBytes("callDuration");
Calllog calllog = null;
while (it.hasNext()) {
Result next = it.next();
String caller = Bytes.toString(next.getValue(famliy, callerf));
String callee = Bytes.toString(next.getValue(famliy, calleef));
String callTime = Bytes.toString(next.getValue(famliy, callTimef));
String callDuration = Bytes.toString(next.getValue(famliy, callDurationf));
calllog = new Calllog();
calllog.setCaller(caller);
calllog.setCallee(callee);
calllog.setCallTime(callTime);
calllog.setCallDuration(callDuration);
list.add(calllog);
}
} catch (Exception e) {
e.printStackTrace();
}
return list;
}
/**
* 按照范围查询通话记录
*/
public List<Calllog> findCallogs(String call , List<CalllogRange> ranges){
List<Calllog> logs = new ArrayList<Calllog>();
try {
for(CalllogRange range : ranges){
Scan scan = new Scan();
//设置扫描起始行
scan.setStartRow(Bytes.toBytes(CalllogUtil.getStartRowkey(call, range.getStartPoint(),100)));
//设置扫描结束行
scan.setStopRow(Bytes.toBytes(CalllogUtil.getStopRowkey(call, range.getStartPoint(), range.getEndPoint(),100)));
ResultScanner rs = table.getScanner(scan);
Iterator<Result> it = rs.iterator();
byte[] f = Bytes.toBytes("f1");
byte[] caller = Bytes.toBytes("caller");
byte[] callee = Bytes.toBytes("callee");
byte[] callTime = Bytes.toBytes("callTime");
byte[] callDuration = Bytes.toBytes("callDuration");
Calllog log = null;
while (it.hasNext()) {
log = new Calllog();
Result r = it.next();
//rowkey
String rowkey = Bytes.toString(r.getRow());
String flag = rowkey.split(",")[3] ;
log.setFlag(flag.equals("0")?true:false);
//caller
log.setCaller(Bytes.toString(r.getValue(f, caller)));
//callee
log.setCallee(Bytes.toString(r.getValue(f, callee)));
//callTime
log.setCallTime(Bytes.toString(r.getValue(f, callTime)));
//callDuration
log.setCallDuration(Bytes.toString(r.getValue(f, callDuration)));
logs.add(log);
}
}
return logs;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
5.编写Callog工具类
------------------------------------------
package com.it18zhang.ssm.util;
import com.it18zhang.ssm.domain.CalllogRange;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
/**
* calllog工具类
*/
public class CalllogUtil {
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
private static SimpleDateFormat sdfFriend = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
//格式化
private static DecimalFormat df = new DecimalFormat();
/**
* 获取hash值,默认分区数100
*/
public static String getHashcode(String caller, String callTime, int partitions) {
int len = caller.length();
//取出后四位电话号码
String last4Code = caller.substring(len - 4);
//取出时间单位,年份和月份.
String mon = callTime.substring(0, 6);
//
int hashcode = (Integer.parseInt(mon) ^ Integer.parseInt(last4Code)) % partitions;
return df.format(hashcode);
}
/**
* 起始时间
*/
public static String getStartRowkey(String caller, String startTime, int partitions){
String hashcode = getHashcode(caller, startTime,partitions);
return hashcode + "," + caller + "," + startTime ;
}
/**
* 结束时间
*/
public static String getStopRowkey(String caller, String startTime,String endTime, int partitions){
String hashcode = getHashcode(caller, startTime,partitions);
return hashcode + "," + caller + "," + endTime ;
}
/**
* 计算查询时间范围
*/
public static List<CalllogRange> getCallLogRanges(String startStr ,String endStr){
try{
SimpleDateFormat sdfYMD = new SimpleDateFormat("yyyyMMdd");
SimpleDateFormat sdfYM = new SimpleDateFormat("yyyyMM");
DecimalFormat df00 = new DecimalFormat("00");
//
List<CalllogRange> list = new ArrayList<CalllogRange>();
//字符串时间
String startPrefix = startStr.substring(0, 6);
String endPrefix = endStr.substring(0, 6);
int endDay = Integer.parseInt(endStr.substring(6, 8));
//结束点
String endPoint = endPrefix + df00.format(endDay + 1);
//日历对象
Calendar c = Calendar.getInstance();
//同年月
if (startPrefix.equals(endPrefix)) {
CalllogRange range = new CalllogRange();
range.setStartPoint(startStr); //设置起始点
range.setEndPoint(endPoint); //设置结束点
list.add(range);
} else {
//1.起始月
CalllogRange range = new CalllogRange();
range.setStartPoint(startStr);
//设置日历的时间对象
c.setTime(sdfYMD.parse(startStr));
c.add(Calendar.MONTH, 1);
range.setEndPoint(sdfYM.format(c.getTime()));
list.add(range);
//是否是最后一月
while (true) {
//到了结束月份
if (endStr.startsWith(sdfYM.format(c.getTime()))) {
range = new CalllogRange();
range.setStartPoint(sdfYM.format(c.getTime()));
range.setEndPoint(endPoint);
list.add(range);
break;
} else {
range = new CalllogRange();
//起始时间
range.setStartPoint(sdfYM.format(c.getTime()));
//增加月份
c.add(Calendar.MONTH, 1);
range.setEndPoint(sdfYM.format(c.getTime()));
list.add(range);
}
}
}
return list ;
}
catch(Exception e){
e.printStackTrace();
}
return null ;
}
/**
* 对时间进行格式化
*/
public static String formatDate(String timeStr){
try {
return sdfFriend.format(sdf.parse(timeStr));
} catch (Exception e) {
e.printStackTrace();
}
return null ;
}
}
6.编写控制器CalllogController.class
-----------------------------------------
package com.it18zhang.ssm.web.controller;
import com.it18zhang.ssm.domain.Calllog;
import com.it18zhang.ssm.domain.CalllogRange;
import com.it18zhang.ssm.service.CalllogService;
import com.it18zhang.ssm.service.impl.CalllogServiceImpl;
import com.it18zhang.ssm.util.CalllogUtil;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import javax.annotation.Resource;
import java.util.Calendar;
import java.util.List;
@Controller
public class CalllogController {
@Resource(name="calllogService")
private CalllogService cs;
/**
* 发送参数
* @param model
* @return
*/
@RequestMapping("calllog/findAll")
public String findAll(Model model)
{
List<Calllog> list = cs.findAll();
model.addAttribute("calllogs", list);
return "calllog/calllogList";
}
/**
* 跳转到查询界面
* @return
*/
@RequestMapping("calllog/toFindCalllogPage")
public String toFindCalllog()
{
return "calllog/findCalllog";
}
/**
* 接受参数
* @param m
* @return
*/
@RequestMapping(value = "calllog/findCalllog", method = RequestMethod.POST)
public String findCalllog(Model m, @RequestParam("caller") String caller, @RequestParam("startTime") String startTime, @RequestParam("endTime") String endTime)
{
List<CalllogRange> list = CalllogUtil.getCallLogRanges(startTime, endTime);
List<Calllog> logs = cs.findCallogs(caller,list);
m.addAttribute("calllogs", logs);
return "callLog/calllogList" ;
}
}
二、编写协处理器模块CalllogCoprossorModel，实现主叫发生的同时即插入被叫记录
--------------------------------------------------------
1.说明
因为HbaseCustomer消费者，得到的数据都是主叫数据，没有被叫数据
所以，需要做一个协处理器，来处理当主叫发生的时候，同时写入被叫的记录
2.创建协处理器模块CalllogCoprpssorModel，并添加maven依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>calllog.kafka</groupId>
<artifactId>CalllogCustomerModel</artifactId>
<version>1.0-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>0.10.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.2.4</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.2.4</version>
</dependency>
</dependencies>
</project>
3.新建协处理器区域观察者类com.calllog.coprossor.CalllogRegion
-------------------------------------
package com.calllog.coprossor;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* 协处理器
*/
public class CalllogRegion<postScannerNext> extends BaseRegionObserver {
//被叫引用id
private static final String REF_ROW_ID = "refrowid" ;
//通话记录表名
private static final String CALL_LOG_TABLE_NAME = "call:calllogs" ;
/**
* 每次put数据之后调用
*/
@Override
public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, Durability durability) throws IOException {
super.postPut(e, put, edit, durability);
TableName tName = TableName.valueOf("call:calllogs");
TableName tName1 = e.getEnvironment().getRegion().getRegionInfo().getTable();
if (tName.equals(tName1)) {
//得到rowkey
String rowKey = Bytes.toString(put.getRow());
String[] strs = rowKey.split(",");
//如果是被叫，直接返回
if (strs[3].equals("1")) {
return;
}
//66,15733218888,20181002071335,0,18332561111,063
//取出相应的值
String caller = strs[1];
String time = strs[2];
String callee = strs[4];
String duration = strs[5];
//计算区域号
String hash = CalllogUtil.getHashcode(callee, time, 100);
String newRowKey = hash + "," + callee + "," + time + "," + "1" + "," + caller + "," + duration;
//开始put数据
Put p = new Put(Bytes.toBytes(newRowKey));
p.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("refrowid"), Bytes.toBytes(rowKey));
Table tb = e.getEnvironment().getTable(tName);
tb.put(p);
System.out.println("put over");
}
}
/**
* get之后调用 -- 实现被叫查询时，直接返回主叫的记录
* 将之前get的结果替换
*/
@Override
public void postGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results) throws IOException {
//获得表名
String tableName = e.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
//判断表名是否是ns1:calllogs
if(!tableName.equals("call:calllogs")){
super.preGetOp(e, get, results);
}
else{
//得到rowkey
String rowkey = Bytes.toString(get.getRow());
//
String[] arr = rowkey.split(",");
//主叫
if(arr[3].equals("0")){
super.postGetOp(e, get, results);
}
//被叫
else{
//得到主叫方的rowkey
String refrowid = Bytes.toString(CellUtil.cloneValue(results.get(0)));
//
Table tt = e.getEnvironment().getTable(TableName.valueOf("call:calllogs"));
Get g = new Get(Bytes.toBytes(refrowid));
Result r = tt.get(g);
List<Cell> newList = r.listCells();
results.clear();
results.addAll(newList);
}
}
}
/**
* 扫描之后调用
*/
@Override
public boolean postScannerNext(ObserverContext<RegionCoprocessorEnvironment> e, InternalScanner s, List<Result> results, int limit, boolean hasMore) throws IOException {
boolean b = super.postScannerNext(e, s, results, limit, hasMore);
//新集合
List<Result> newList = new ArrayList<Result>();
//获得表名
String tableName = e.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
//判断表名是否是ns1:calllogs
if (tableName.equals(CALL_LOG_TABLE_NAME)) {
Table tt = e.getEnvironment().getTable(TableName.valueOf(CALL_LOG_TABLE_NAME));
for(Result r : results){
//rowkey
String rowkey = Bytes.toString(r.getRow());
String flag = rowkey.split(",")[3] ;
//主叫
if(flag.equals("0")){
newList.add(r) ;
}
//被叫
else{
//取出主叫号码
byte[] refrowkey = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes(REF_ROW_ID)) ;
Get newGet = new Get(refrowkey);
newList.add(tt.get(newGet));
}
}
results.clear();
results.addAll(newList);
}
return b ;
}
}
4.新建工具类CoprossorUtil
----------------------------------------------
package com.calllog.coprossor;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
/**
* Created by Administrator on 2017/4/13.
*/
public class CalllogUtil {
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
private static SimpleDateFormat sdfFriend = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
//格式化
private static DecimalFormat df = new DecimalFormat();
/**
* 获取hash值,默认分区数100
*/
public static String getHashcode(String caller, String callTime,int partitions) {
int len = caller.length();
//取出后四位电话号码
String last4Code = caller.substring(len - 4);
//取出时间单位,年份和月份.
String mon = callTime.substring(0, 6);
//
int hashcode = (Integer.parseInt(mon) ^ Integer.parseInt(last4Code)) % partitions;
return df.format(hashcode);
}
/**
* 起始时间
*/
public static String getStartRowkey(String caller, String startTime, int partitions){
String hashcode = getHashcode(caller, startTime,partitions);
return hashcode + "," + caller + "," + startTime ;
}
/**
* 结束时间
*/
public static String getStopRowkey(String caller, String startTime,String endTime, int partitions){
String hashcode = getHashcode(caller, startTime,partitions);
return hashcode + "," + caller + "," + endTime ;
}
/**
* 对时间进行格式化
*/
public static String formatDate(String timeStr){
try {
return sdfFriend.format(sdf.parse(timeStr));
} catch (Exception e) {
e.printStackTrace();
}
return null ;
}
}
5.打包部署
a.注册协处理器,并分发到所有hbase节点
[hbase-site.xml]
<property>
<name>hbase.coprocessor.region.classes</name>
<value>com.calllog.coprossor.CalllogRegion</value>
</property>
b.将打好的jar包分发到所有节点的/hbase/lib目录下
c.重启hbase集群
d.进入hbase shell,重建表"call:calllogs" "f1" "f2"
$hbase> create 'call:calllogs','f1','f2';
三、生成日志，收集日志
1.开启kafka集群
[s200 s300 s400]
$> /soft/kafka/bin/kafka-server-start.sh -daemon /soft/kafka/config/server.properties
2.在s100和s200上启动flume，开始收集日志
$s100> flume-ng agent -f /soft/flume/conf/calllog.conf -n a1 &
$s200> flume-ng agent -f /soft/flume/conf/calllog.conf -n a1 &
3.开启kafka的消费者--hbase[进入share/calllog目录下，找到CalllogCustomerModel.jar包]
$> java -cp CalllogCustomerModel.jar calllog.kafka.hbase.customer.HbaseCustomer
4.开启日志生成工具
$calllog> ./calllog.sh
5.运行ssm模块，查看webapp界面显示
4802
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
