文章目录
一、项目概述
建立医药电商大数据分析平台采集医药电商的数据、分析电商平台数据、可视化电商平台数据。可以根据医药产品销售和库存管理来提高销售率等等。
二、功能概述
1.流量分析：根据每日、每月、年度分析用户的行为数据，如浏览量、访客数、访问次数、平均访问深度等等
2.经营状态分析。根据月度或年度对销售状态进行统计，统计指标包括下单金额，下单客户，下单量等等
3.大数据可视化系统
三、关键技术
1.Hadoop：分布式计算平台
2.HBase：分布式数据存储数据库
3.Bootstrap：页面搭建框架
4.IQuery：后台交互
5.Echarts:数据可视化
四、系统架构设计
4.1 大数据采集子系统
4.1.1定义
系统以离线批处理方式，推送采集结果数据给大数据分析平台。
4.1.2接口
采集大数据接受的格式、大数据接口定义
五、数据存储设计
5.2数据采集子系统
5.2.1采集用户流量数据
WsSysAgentAccessVo.java采集用户流量数据，实现了Serializable接口。它包含以下属性：
sessionId（String类型）：会话ID
userId（Integer类型）：登录用户的ID
trackUid（String类型）：用户标识
userAgent（String类型）：用户UA
referer（String类型）：访问的页面地址
数据格式
“WsSysAgentAccessVo [sessionId=xxx, userId=xxx, trackUid=xxx,
userAgent=xxx, referer=xxx]”
package receiveVo;
import java.io.Serializable;
public class WsSysAgentAccessVo implements Serializable {
private static final long serialVersionUID = 1L;
private String sessionId; //会话ID
private Integer userId; //登录用户的ID
private String trackUid; //用户标识
private String userAgent; //用户UA
private String referer; //访问的页面地址
public String getSessionId() {
return sessionId;
}
public void setSessionId(String sessionId) {
this.sessionId = sessionId;
}
public Integer getUserId() {
return userId;
}
public void setUserId(Integer userId) {
this.userId = userId;
}
public String getTrackUid() {
return trackUid;
}
public void setTrackUid(String trackUid) {
this.trackUid = trackUid;
}
public String getUserAgent() {
return userAgent;
}
public void setUserAgent(String userAgent) {
this.userAgent = userAgent;
}
public String getReferer() {
return referer;
}
@Override
public String toString() {
return "WsSysAgentAccessVo [sessionId=" + sessionId + ", userId=" + userId + ", trackUid=" + trackUid
+ ", userAgent=" + userAgent + ", referer=" + referer + "]";
}
public void setReferer(String referer) {
this.referer = referer;
}
}
5.2.2采集用户订单数据
WsOrderVo.java采集用户订单数据，实现了Serializable接口。它包含了以下属性：
orderId（int类型）：订单ID
userId（int类型）：下单的用户ID
orderNum（String类型）：订单编号
payment（Double类型）：支付总金额
productTotalAmount（Double类型）：订单商品总金额
isCod（String类型）：是否货到付款
orderStatus（String类型）：订单状态
package receiveVo;
import java.io.Serializable;
public class WsOrderVo implements Serializable{
private static final long serialVersionUID = 1L;
private int orderId; //订单ID
private int userId; //下单的用户ID
private String orderNum; //订单编号
private Double payment; //支付总金额
private Double productTotalAmount; //订单商品总金额
private String isCod; //是否货到付款
private String orderStatus; //订单状态
public WsOrderVo()
{
}
public int getOrderId()
{
return orderId;
}
public void setOrderId(int orderId)
{
this.orderId = orderId;
}
public int getUserId()
{
return userId;
}
public void setUserId(int userId)
{
this.userId = userId;
}
public String getOrderNum()
{
return orderNum;
}
public void setOrderNum(String orderNum)
{
this.orderNum = orderNum;
}
public Double getPayment()
{
return payment;
}
public void setPayment(Double payment)
{
this.payment = payment;
}
public Double getProductTotalAmount()
{
return productTotalAmount;
}
public void setProductTotalAmount(Double productTotalAmount)
{
this.productTotalAmount = productTotalAmount;
}
public String getIsCod()
{
return isCod;
}
public void setIsCod(String isCod)
{
this.isCod = isCod;
}
public String getOrderStatus()
{
return orderStatus;
}
public void setOrderStatus(String orderStatus)
{
this.orderStatus = orderStatus;
}
public String toString()
{
return (new StringBuilder()).append("WsOrderVo{orderId=").append(orderId).append(", userId=").append(userId).append(", orderNum='").append(orderNum).append('\'').append(", payment=").append(payment).append(", productTotalAmount=").append(productTotalAmount).append(", isCod='").append(isCod).append('\'').append(", orderStatus='").append(orderStatus).append('\'').append('}').toString();
}
}
5.2.3 采集用户订单项数
WsOrderItemVo.java定义采集用户订单项数据，实现了Serializable接口。它包含了以下属性：
orderItemId（int类型）：订单项ID
orderId（int类型）：订单ID
productId（int类型）：商品ID
productUnitPrice（Double类型）：商品单价
num（int类型）：商品数量
package receiveVo;
import java.io.Serializable;
public class WsOrderItemVo implements Serializable{
private static final long serialVersionUID = 1L;
private int orderItemId; //订单项ID
private int orderId; //订单ID
private int productId; //商品ID
private Double productUnitPrice; //商品单价
private int num; //商品数量
public WsOrderItemVo()
{
}
public int getOrderItemId()
{
return orderItemId;
}
public void setOrderItemId(int orderItemId)
{
this.orderItemId = orderItemId;
}
public int getOrderId()
{
return orderId;
}
public void setOrderId(int orderId)
{
this.orderId = orderId;
}
public int getProductId()
{
return productId;
}
public void setProductId(int productId)
{
this.productId = productId;
}
public Double getProductUnitPrice()
{
return productUnitPrice;
}
public void setProductUnitPrice(Double productUnitPrice)
{
this.productUnitPrice = productUnitPrice;
}
public int getNum()
{
return num;
}
public void setNum(int num)
{
this.num = num;
}
public String toString()
{
return (new StringBuilder()).append("WsOrderItemVo{orderItemId=").append(orderItemId).append(", orderId=").append(orderId).append(", productId=").append(productId).append(", productUnitPrice=").append(productUnitPrice).append(", num=").append(num).append('}').toString();
}
}
5.3流量分析及经营状况分析
5.3.1流量分析
WsSysDataResultVo.java定义了流量分析数据。实现了Serializable接口。它包含了以下属性：
reportTime（String类型）：时间维度
pageView（int类型）：浏览量
userView（int类型）：访客数
visitView（int类型）：访问次数
visitDepth（float类型）：平均访问深度
package hbase.resultVo;
import java.io.Serializable;
public class WsSysDataResultVo implements Serializable{
private static final long serialVersionUID = 1L;
//时间维度
private String reportTime;
//浏览量
private int pageView;
//访客数
private int userView;
//访问次数
private int visitView;
//平均访问深度
private float visitDepth;
public String getReportTime() {
return reportTime;
}
public void setReportTime(String reportTime) {
this.reportTime = reportTime;
}
public int getPageView() {
return pageView;
}
public void setPageView(int pageView) {
this.pageView = pageView;
}
public int getUserView() {
return userView;
}
public void setUserView(int userView) {
this.userView = userView;
}
public int getVisitView() {
return visitView;
}
public void setVisitView(int visitView) {
this.visitView = visitView;
}
public float getVisitDepth() {
return visitDepth;
}
public void setVisitDepth(float visitDepth) {
this.visitDepth = visitDepth;
}
}
5.3.2经营状况分析数据
OrderVo.java定义经营状况分析数据，实现了Serializable接口。它包含了以下属性：
reportTime（String类型）：时间维度
orderPriceTotal（Double类型）：下单金额
orderProductCount（int类型）：下单商品件数
orderCount（int类型）：下单数量
orderUserCount（int类型）：下单客户数
orderSalePrice（Double类型）：客单价
package hbase.resultVo;
import java.io.Serializable;
public class OrderVo implements Serializable{
private static final long serialVersionUID = 1L;
private String reportTime; //时间维度
private Double orderPriceTotal; //下单金额
private int orderProductCount; //下单商品件数
private int orderCount; //下单数量
private int orderUserCount; //下单客户数
private Double orderSalePrice; //客单价
public String getReportTime() {
return reportTime;
}
public void setReportTime(String reportTime) {
this.reportTime = reportTime;
}
public Double getOrderPriceTotal() {
return orderPriceTotal;
}
public void setOrderPriceTotal(Double orderPriceTotal) {
this.orderPriceTotal = orderPriceTotal;
}
public int getOrderProductCount() {
return orderProductCount;
}
public void setOrderProductCount(int orderProductCount) {
this.orderProductCount = orderProductCount;
}
public int getOrderCount() {
return orderCount;
}
public void setOrderCount(int orderCount) {
this.orderCount = orderCount;
}
public int getOrderUserCount() {
return orderUserCount;
}
public void setOrderUserCount(int orderUserCount) {
this.orderUserCount = orderUserCount;
}
public Double getOrderSalePrice() {
return orderSalePrice;
}
public void setOrderSalePrice(Double orderSalePrice) {
this.orderSalePrice = orderSalePrice;
}
}
5.4数据存储接口
HBaseDAO.java
package hbase.dao;
import java.util.List;
import receiveVo.WsOrderItemVo;
import receiveVo.WsOrderVo;
import receiveVo.WsSysAgentAccessVo;
public interface HBaseDAO {
//存储流量数据
public void saveAgentAccessData(List<WsSysAgentAccessVo>list) ;
//存储订单数据及订单项数据
public void saveOrderVo(WsOrderVo orderVo, List<WsOrderItemVo>list);
}
HBaseDAOImp.java
package hbase.dao.imp;
import hbase.dao.HBaseDAO;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import receiveVo.WsOrderItemVo;
import receiveVo.WsOrderVo;
import receiveVo.WsSysAgentAccessVo;
import utils.ObjectAndByte;
public class HBaseDAOImp implements HBaseDAO{
HConnection hTbalePool = null ;
public HBaseDAOImp()
{
Configuration conf = new Configuration();
String zk_list = "172.30.0.10,172.30.0.11,172.30.0.12" ;
conf.set("hbase.zookeeper.quorum",zk_list) ;
try {
hTbalePool = HConnectionManager.createConnection(conf) ;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override
public void saveAgentAccessData(List<WsSysAgentAccessVo> list) {
// TODO Auto-generated method stub
HTableInterface table = null ;
try {
table = hTbalePool.getTable("tb_data") ;
for (WsSysAgentAccessVo vo : list) {
//行健 ：用户类型+当前时间+用户标识
//用户类型，目前只分析pc端
String userTypeString = "1"; //1=PC流量，2=WAP流量，3=微信流量
//当前时间
Date now = new Date() ;
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss") ;//可以方便地修改日期格式
String timeStr = dateFormat.format( now ) ;
String rowkey = userTypeString+"-"+timeStr+"-"+System.currentTimeMillis() ;
System.out.println("rowkey:"+rowkey) ;
Put p = new Put(rowkey.getBytes()) ;
ObjectAndByte objToByte = new ObjectAndByte() ;
System.out.println(objToByte.toByteArray(vo)) ;
p.add("cf".getBytes(), "wsSysAgentAccessVo".getBytes(), objToByte.toByteArray(vo)) ;
table.put(p) ;
}
}catch (Exception e){
e.printStackTrace() ;
}finally{
try {
table.close() ;
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Override
public void saveOrderVo(WsOrderVo orderVo, List<WsOrderItemVo>list) {
// TODO Auto-generated method stub
HTableInterface table = null ;
try {
int orderProductCount = 0; //订单商品件数
for (WsOrderItemVo vo1 : list) {
orderProductCount = orderProductCount + vo1.getNum();
}
table = hTbalePool.getTable("tb_order") ;
//orderStatus 1 货到付款 0 在线支付 只有这两种把
//当前时间
Date now = new Date() ;
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss") ;//可以方便地修改日期格式
String timeStr = dateFormat.format( now ) ;
String rowkey = timeStr+"-"+System.currentTimeMillis() ;
System.out.println("rowkey:"+rowkey) ;
Put p = new Put(rowkey.getBytes()) ;
p.add("cf".getBytes(), "orderProductCount".getBytes(), String.valueOf(orderProductCount).getBytes());
ObjectAndByte objToByte = new ObjectAndByte() ;
p.add("cf".getBytes(), "wsOrderVo".getBytes(), objToByte.toByteArray(orderVo)) ;
table.put(p);
}catch (Exception e){
e.printStackTrace() ;
}finally{
try {
table.close() ;
} catch (IOException e) {
e.printStackTrace() ;
}
}
try {
table = hTbalePool.getTable("tb_orderItem") ;
int i = list.size();
for (WsOrderItemVo vo : list) {
String rowkey = String.valueOf(vo.getOrderId()) + "_" + i;
System.out.println("rowkey:"+rowkey) ;
Put p = new Put(rowkey.getBytes()) ;
ObjectAndByte objToByte = new ObjectAndByte() ;
p.add("cf".getBytes(), "wsOrderItemVo".getBytes(), objToByte.toByteArray(vo)) ;
i++;
table.put(p);
}
}catch (Exception e){
e.printStackTrace() ;
}finally{
try {
table.close() ;
} catch (IOException e) {
e.printStackTrace() ;
}
}
}
public static void main(String[] args) {
HBaseDAO dao = new HBaseDAOImp();
//测试存储流量数据
List<WsSysAgentAccessVo> list = new ArrayList<WsSysAgentAccessVo>() ;
int i = 20 ;
int type = 1 ;
while (i > 0) {
WsSysAgentAccessVo vo = new WsSysAgentAccessVo() ;
vo.setSessionId("000000"+i) ;
vo.setUserId(1867500+i) ;
vo.setTrackUid("111100"+i);
vo.setUserAgent(String.valueOf(type)) ;
vo.setReferer("www.baidu.com"+i);
i--;
list.add(vo);
}
dao.saveAgentAccessData(list) ;
//测试存储订单数据
// List<WsOrderVo> list1 = new ArrayList<WsOrderVo>() ;
// int i = 20 ;
// while (i > 0) {
// WsOrderVo vo = new WsOrderVo() ;
// vo.setOrderId(20170829);
// vo.setUserId(1867500) ;
// vo.setOrderNum("111100"+i) ;
// vo.setPayment(18.50) ;
// vo.setProductTotalAmount(128.6) ;
// vo.setIsCod("1") ;
// vo.setOrderStatus(String.valueOf(i%3)) ;
// i-- ;
// list.add(vo) ;
// }
// dao.saveOrderData(list) ;
//测试存储订单项数据
// List<WsOrderItemVo> list = new ArrayList<WsOrderItemVo>() ;
// int i = 10;
// while(i > 0){
// WsOrderItemVo vo = new WsOrderItemVo() ;
// vo.setOrderId(i);
// vo.setNum(10+i);
// vo.setOrderItemId(100+i);
// vo.setProductId(100000+i);
// vo.setProductUnitPrice(99.0);
// i-- ;
// list.add(vo) ;
// }
// dao.saveOrderItemData(list);
System.err.println("save successful !");
}
}
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
