springboot+mongodb+python+echarts数据分析与预测

零：概要

目标：数据监测与数据预测
系统：centos7 docker发布
展示方式：使用echarts展示，数据实时监测制作模型，指标分析模型，回归模型
方法：1、从mongodb取值，制作数据模型，
2、调用python算法进行数据的计算及展示

一、前端输出

1、实时展示

2、指标模型

3、标准化OLS

二、构建过程

1、pom.xml配置，回载模块

	<dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-mongodb</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-thymeleaf</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

2、配置数据库

spring:
  data:
    mongodb:
      uri: mongodb://username:password@localhost:27017/databases
  thymeleaf:
    cache: false
server:
  port: 8081

3、构建接口调用mongodb数据

public class MongoRealTime {

    @Autowired
    MongoTemplate mongoTemplate;


    public AggregateIterable<Document> getdata(String collection, String order,long timeval) {

        String collectionName=collection;
        String orderNum=order;
        Integer depth=24; //查看数据量24小时
        MongoCollection<Document> coll = null;
        String col=null;

        Date start_time=new Date(timeval+1000*60*60*8);
        Date end_time=new Date(timeval+1000*60*60*(depth+8));

        Document match = new Document();
        match.put("MachineNum", new Document("$exists", true));
        match.put("JobNum", new Document("$exists", true));
        Document match_time=new Document();
        match_time.put("$exists", true);
        match_time.put("$gt", start_time);
        match_time.put("$lt",end_time);
        match.put("CreateTime",match_time);

        Document project = new Document();
        project.put("_id", 0);
        project.put("MachineNum", 1);
        project.put("JobNum", 1);
        project.put("CreateTime", 1);

        if (collectionName.equals( "OutWater")) {
            match.put("FullDrain", new Document("$exists", true));
            match.put("HalfDrain", new Document("$exists", true));
            match.put("DrainOrder", orderNum);

            project.put("FullDrain", 1);
            project.put("HalfDrain", 1);
            col="WaterValve";

        } else if (collectionName.equals("SmartCover")) {
            match.put("SeatTime", new Document("$exists", true));
            match.put("CoverTime", new Document("$exists", true));
            match.put("Order", orderNum);

            project.put("SeatTime", 1);
            project.put("CoverTime", 1);
            col="SmartCover";

        } else if (collectionName.equals("InterWater")) {
            match.put("RealWaterLevel", new Document("$exists", true));
            match.put("InletOrder", orderNum);
            project.put("RealWaterLevel", 1);
            col="WaterValve";
        }

        Document group_match = new Document("$match", match);
        Document group_project = new Document("$project", project);
        List<Document> aggregateList = new ArrayList<Document>();
        aggregateList.add(group_match);
        aggregateList.add(group_project);
        AggregateIterable<Document> docs=mongoTemplate.getCollection(col).aggregate(aggregateList);
        return docs;
    }

    
}

4、api

public class MongoContoller {
	@Autowired
    MongoRealTime mongoRealTime;
    
	@ResponseBody
    @RequestMapping("/realtimelist")
    public HashMap realtimelist(
            String conn
            , String order
            ,long timeval
    ){return mongoRealTime.getrealtime(conn,order,timeval); }

}

5、html展示

<div class="data_bodey">
    <div th:replace="commons/reportheadbar::#reportheadMenu(activerUri='r3')"></div>

    <div class="inner" style="height: 90%;">
        <div class="dataAllBorder01">
            <div class="dataAllBorder01">
                <p class="data_chartall" id="realtimeid" >数据加载中。。。。。</p>
            </div>
        </div>
    </div>

</div>

6、模型渲染

function labrealtime(conn, order, date) {

    var myChart = echarts.init(document.getElementById('realtimeid'));
    var option;

    $.getJSON('/realtimelist', {"conn": conn, "order": order, "timeval": date.getTime()}, function (_rawData) {
        var MachineNums = _rawData['key'][0];
        var datasetWithFilters = [];
        var seriesList = [];
        var yname = null;
        var ychange = null; //转换/100
        var yname100 = null;

        echarts.util.each(MachineNums, function (MachineNum) {

            var ylist = [];

            if (conn == "SmartCover") {
                ylist.push("SeatTime");
                ylist.push("CoverTime");
                yname = '单位：秒';
                yname100 = 'ms';
                ychange = 100;
            } else if (conn == "OutWater") {
                ylist.push("FullDrain");
                ylist.push("HalfDrain");
                yname = '单位：mm';
                ychange = 10;
                yname100 = 'mm';
            } else {
                ylist.push("RealWaterLevel");
                yname = '单位：mm';
                ychange = 10;
                yname100 = 'mm';
            }

            for (let i = 0; i < ylist.length; i++) {
                datasetWithFilters.push({
                        id: 'dataset_' + MachineNum + '_' + ylist[i],
                        fromDatasetId: 'dataset_raw',
                        transform: {
                            type: 'filter',
                            config: {
                                and: [
                                    {dimension: ylist[i], gt: 0},
                                    {dimension: 'MachineNum', '=': MachineNum}
                                ]
                            }
                        }
                    },
                );
                seriesList.push({
                        type: 'line',
                        datasetId: 'dataset_' + MachineNum + '_' + ylist[i],
                        showSymbol: false,
                        name: MachineNum,
                        endLabel: {
                            show: true,
                            formatter: function (params) {
                                return ylist[i] + params.value[0] + ': ' + params.value[2 + i] + yname100;
                            },
                            color: '#fbff24',
                        },
                        labelLayout: {
                            moveOverlap: 'shiftY'
                        },
                        emphasis: {
                            focus: 'series'
                        },
                        encode: {
                            x: 'CreateTime',
                            y: ylist[i],
                            label: ['MachineNum', ylist[i]],
                            tooltip: ['MachineNum', ylist[i]],
                        }
                    },
                );
            }
        });
        option = {
            animationDuration: 5000,
            dataset: [{
                id: 'dataset_raw',
                source: _rawData['data']
            }].concat(datasetWithFilters),
            title: {
                text: '机台数据实时监测',
                textStyle: {
                    color: '#38f3ff'
                }
            },
            tooltip: {
                order: 'valueDesc',
                trigger: 'axis'
            },
            xAxis: {
                // type: 'category',
                type: 'time',
                nameLocation: 'middle',
                axisLabel: {
                    color: '#38f3ff',
                    fontWeigth: 'bold',
                    fontSize: 20,
                }
            },
            yAxis: {
                name: yname,
                nameTextStyle: {
                    fontSize: 12,
                    color: 'rgb(255,255,255,0.8)',
                },
                axisLabel: {
                    fontSize: 20,
                    fontWeight: 'bold',
                    color: function (value, index) {
                        return ychange === 100 && value <= 2000 && value >= 300 ? 'green' : 'red';
                    },
                    formatter: function (value, index) {
                        return value / ychange;
                    },
                },
                splitLine: {
                    show: true,
                    lineStyle: {
                        color: 'rgb(18,100,254,0.3)'
                    }
                },

            },
            grid: {
                right: 180,
            },
            series: seriesList,
        };
        myChart.clear();
        myChart.setOption(option);
    });
    option && myChart.setOption(option);

}

7、python建模（使用java调用python进行建模，结果集用于数据的展示）

class labModel:

    def __init__(self):
        labmongodb = MongoClient("mongodb://localhost:27017")
        self.servercoll = labmongodb["machine"]
        self.anaysiscoll = self.servercoll['AnalysisDB']

    def get_data(self, docslist):
        obj = docslist.get('ObjectName')

        if obj == "CoverTime" or obj == "SeatTime":
            databasename = "SmartCover"
            ordername = "Order"
        elif obj == "FullDrain" or obj == "HalfDrain":
            databasename = "WaterValve"
            ordername = "DrainOrder"
        else:
            databasename = "WaterValve"
            ordername = "InletOrder"

        coll = self.servercoll[databasename]
        docs = coll.aggregate([
            {'$match': {ordername: docslist.get('Order'),
                        'MachineNum': docslist.get('MachineNum'),
                        'JobNum': int(docslist.get('JobNum')),
                        docslist.get('ObjectName'):{'$gt':0}
                        }
             },
            {'$project': {'CreateTime': 1, '_id': 0, obj: 1}},
            {'$sort': {'CreateTime': 1}}
        ])
        df = pd.DataFrame(list(docs))
        return df

    def df_OLS(self, df, obj):

        # 抽样
        df_train = df.sample(300)

        features = df_train[obj].index.values.reshape(-1, 1)
        target = df_train[obj]
        train_x, test_x, train_y, test_y = train_test_split(features, target, test_size=0.3, random_state=42)
        test_x.sort(axis=0)
        train_x = sm.add_constant(train_x)
        test_x1 = sm.add_constant(test_x)

        est = sm.OLS(train_y, train_x)
        est2 = est.fit()

        predict_est = est2.predict(test_x1) - test_y  # 残差
        predict_est_mean = np.mean(predict_est)  # 残差均值
        predict_mean_std = (predict_est - predict_est_mean) / np.std(predict_est)  # 标准化残差

        fit_v=pd.DataFrame(train_x, columns=['b', 'x'])
        fit_v.drop('b',axis=1,inplace=True)
        fit_v['y']=est2.fittedvalues.values.tolist()
        fit_v['resid_pearson']=est2.resid_pearson.tolist()
        fittedvalues_v=np.append((fit_v.columns.to_numpy()).reshape((1,-1)),fit_v.to_numpy(),axis=0)

        test_v=pd.DataFrame(test_x,columns=['x'])
        test_v['y']=test_y.tolist()
        test_v['predict_y']=est2.predict(test_x1).tolist()
        test_v['ZRESID']=predict_mean_std.tolist()
        # predict_v=np.insert(test_v.to_numpy(),0,['x','y','predict_y','ZRESID'],axis=0)
        predict_v=np.append((test_v.columns.to_numpy()).reshape((1,-1)),test_v.to_numpy(),axis=0)

        ols_document = {
            'fittedvalues':fittedvalues_v.tolist(),
            'predict':predict_v.tolist(),
            'model': {
                'r2': est2.rsquared,  # 方程R方
                'p': est2.f_pvalue,  # 方程p
                'params': est2.params.to_dict(),  # 参数
                'stdbse': est2.bse.to_dict(),  # 参数估计标准误
                'pvalues': est2.pvalues.to_dict(),  # 参数p值
            }
        }

        return ols_document

    def df_save(self, docs):
        # coll=self.colltestdb #localhost mongodb
        coll = self.anaysiscoll  # serverhost mongodb
        document_exists = {'Type': docs.get('Type'), 'Order': docs.get('Order'), 'MachineNum': docs.get('MachineNum'),
                           'JobNum': docs.get('JobNum'),
                           'ObjectName': docs.get('ObjectName')}
        coll.replace_one(document_exists, docs, upsert=True)


if __name__ == '__main__':
    try:
        docs = {}
        docs['Type'] = 2
        docs['ObjectName'] = sys.argv[1]  # 项目进水，排水，慢落
        docs['Order'] = sys.argv[2]  # 订单号
        docs['MachineNum'] = sys.argv[3]  # 机台
        docs['JobNum'] = sys.argv[4]  # 工位号
        docs['RunDateTime'] = datetime.now()

        try:
            lab = labModel()
            data = lab.get_data(docs)
            if len(data) > 100:
                olsdocs = lab.df_OLS(data, docs.get('ObjectName'))
                docs.update(olsdocs)
                lab.df_save(docs)
                print('ok')
        except:
            print('err')
    except IndexError:
        print('err')