项目实施-12 （数据抽取贰）

最新推荐文章于 2024-10-13 18:17:44 发布

纯净天空7

最新推荐文章于 2024-10-13 18:17:44 发布

阅读量326

点赞数

分类专栏：数据综合分析系统

本文链接：https://blog.csdn.net/asymuxue/article/details/105421616

版权

本文是项目实施的第二部分，主要讲述数据抽取过程，包括验证数据的策略和使用流计算引擎进行数据仓库ETL操作。同时，提到了几种评估方法如余弦相似度、欧式距离和球面距离，并提供了项目的相关链接。

摘要由CSDN通过智能技术生成

概述

本篇承接项目实施-11 https://blog.csdn.net/ASYMUXUE/article/details/105353601继续介绍 风险评估数据 的抽取。

本篇涉及部分简单算法，请参考https://blog.csdn.net/ASYMUXUE/category_9862606.html

抽取验证数据

思路总结：我们通过 从登录的日志中 获取到用户登录的数据：

①对此数据进行实体类封装

②建立评估因子模型，确定要对哪些数据进行评估

③创建评估报告实体类，作为传递最终判断结果数据的封装模型

④采用责任链设计模式，对评估因子一一进行评估判断

⑤判定每个评估因子的评估结果，从此次需要验证的数据中获取与之历史数据进行对比

⑥将每个因子的判定结果放入 评估报告 ,进行报告结果数据的向下传递

①`创建验证数据的实体类`

package com.baizhi.entities;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors;

import java.io.Serializable;

@NoArgsConstructor
@AllArgsConstructor
@Data
@Accessors(chain = true)
public class EvaluateData implements Serializable {
   
    private long evaluateTime;
    //应用名
    private String applicationName;
    //用户的唯一标识
    private String userIdentify;
    //登录序列号
    private String loginSequence;
    private String ordernessPassword;
    private String cityName;
    //地理位置经纬度，其对象的创建在 第 11 篇中。
    private GeoPoint geoPoint;
    //输入特征 （输入时长）
    private Double[] inputFeatures;
    //设备信息
    private String deviceInformation;
}
/*
* //INFO 2020-03-31 10:12:00 QQ EVALUATE [张三] 6ebaf4ac780f40f486359f3ea6934620 "12355421" Beijing "116.4,39.5"
//[1200,15000,2100] "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
* */

②`创建评估因子`

package com.baizhi.entities;

/**
 * 声明所有的风险元素
 */
public enum  RiskFactor {
   
    AREA("area"), //地址
    DEVICE("device"), //设备
    TOTAL("total"), //次数
    TIMESLOT("timeslot"), //访问时段
    SIMILARITY("similarity"),//密码相似度
    INPUTFEATURE("inputfeature"),//输入特征
    SPEED("speed");//位移速度

    private String name;
    //提供构造
    RiskFactor(String name){
   
       this.name=name;
    }
}

③`创建评估报告`

package com.baizhi.entities;

import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;

//评估报告
public class EvaluateReport implements Serializable {
   
    private String applicationName;
    private String userIdentify;
    private String loginSequence;
    private long evaluateTime;
    private String cityName;
    private GeoPoint geoPoint;

    //map集合，存储所有的风险因子属性
    private Map<RiskFactor,Boolean> metrics=new HashMap<RiskFactor,Boolean>();

    //定下特定风险因子的报告（改变属性的特定值）
    public void signReport(RiskFactor riskFactor,boolean flag){
   
        metrics.put(riskFactor,flag);
    }

    public EvaluateReport(String applicationName, String userIdentify, String loginSequence, long evaluateTime, String cityName, GeoPoint geoPoint) {
   
        this.applicationName = applicationName;
        this.userIdentify = userIdentify;
        this.loginSequence = loginSequence;
        this.evaluateTime = evaluateTime;
        this.cityName = cityName;
        this.geoPoint = geoPoint;

        //初始化所有风险因子都是false
        metrics.put(RiskFactor.AREA,false);
        metrics.put(RiskFactor.DEVICE,false);
        metrics.put(RiskFactor.SIMILARITY,false);
        metrics.put(RiskFactor.SPEED,false);
        metrics.put(RiskFactor.TIMESLOT,false);
        metrics.put(RiskFactor.INPUTFEATURE,false);
        metrics.put(RiskFactor.TOTAL,false);
    }

    /**
     *                                                  AREA DEVICE INPUTFEATURE SIMILARITY SPEED TIMESLOT TOTAL
     * QQ zhangsan 001 1585640451510 Beijing 116.4,39.5 true false    false        false    true   false     false
     * @return
     */
     @Override
     public String toString(){
   
         String report=metrics.keySet()
                 .stream()
                 .sorted((RiskFactor o1, RiskFactor o2) -> o1.name().compareTo(o2.name()))
                 .map(riskFactor -> metrics.get(riskFactor)+"")
                 .reduce((v1,v2)->v1+" "+v2)
                 .get();


          return applicationName+" "+userIdentify+" "+loginSequence+" "+evaluateTime+" "+cityName+ " "+geoPoint.getLongtitude()+","+geoPoint.getLatitude()+" "+report;

     }

    public static void main(String[] args) {
   
        System.out.println( new EvaluateReport("QQ","zhangsan","001",new Date().getTime(),"Beijing",new GeoPoint(116.4,39.5)));
    }

    public String getApplicationName() {
   
        return applicationName;
    }

    public String getUserIdentify() {
   
        return userIdentify;
    }
}

④`创建评估链`

package com.baizhi.evaluate;

import com.baizhi.entities.EvaluateData;
import com.baizhi.entities.EvaluateReport;
import com.baizhi.entities.HistoryData;

import java.util.List;

public class EvaluateChain {
   
    //验证位置
    private int position=0;
    //评估链（责任链）
    private List<Evaluate> evaluates;

    //构造
    public EvaluateChain(List<Evaluate> evaluates) {
   
        this.evaluates = evaluates;
    }

    //向下执行
    public void doChain(EvaluateData evaluateData, HistoryData historyData, EvaluateReport evaluateReport){
   
        if(position < evaluates.size()){
   
            //获取一个责任
            Evaluate evaluate = evaluates.get(position);
            position +=1;
            evaluate.eval(evaluateData,historyData,evaluateReport,this);
        }
    }
}

⑤`创建验证的抽象类`（后续每个因子的评估类都将继承此抽象类）

package com.baizhi.evaluate;

import com.baizhi.entities.EvaluateData;
import com.baizhi.entities.EvaluateReport;
import com.baizhi.entities.HistoryData;
import com.baizhi.entities.RiskFactor;

//验证抽象类
public abstract class Evaluate {
   
    //风险因子
    private RiskFactor riskFactor;

    //构造
    public Evaluate(RiskFactor riskFactor) {
   
        this.riskFactor = riskFactor;
    }

    //获取验证的当前风险
    public RiskFactor getRiskFactor() {
   
        return riskFactor;
    }

    /**
     *验证方法
     * @param evaluateData
     * @param historyData
     * @param evaluateReport
     * @param evaluateChain:驱动下一个Evaluate实例
     */
    public abstract  void eval(EvaluateData evaluateData, HistoryData historyData, EvaluateReport evaluateReport,
                               EvaluateChain evaluateChain);
}

⑥`继承抽象类`

登录城市验证

package com.baizhi.evaluate.impl;

import com.baizhi.entities.EvaluateData;
import com.baizhi.entities.EvaluateReport;
import com.baizhi.entities.HistoryData;
import com.baizhi.entities.RiskFactor;
import com.baizhi.evaluate.Evaluate;
import com.baizhi.evaluate.EvaluateChain;

import java.util.Set;

public class AreaEvaluate extends Evaluate {
   
    public AreaEvaluate() {
   
        super(RiskFactor.AREA);
    }

    @Override
    public void eval(EvaluateData evaluateData, HistoryData historyData, EvaluateReport evaluateReport, EvaluateChain evaluateChain) {
   
        evaluateReport.signReport(getRiskFactor(),doEval(evaluateData.getCityName(),historyData.getHistoryCities()))