4.9 nodejs操作多种数据库

LetsStudy

已于 2022-09-22 17:26:15 修改

阅读量958

点赞数

分类专栏： Node自动化测试文章标签： node mysql elastchsearch mongoose node redis

于 2022-09-13 14:44:59 首次发布

本文链接：https://blog.csdn.net/LetsStudy/article/details/126833010

版权

Node自动化测试专栏收录该内容

54 篇文章 0 订阅

订阅专栏

该博客详细介绍了如何从MySQL数据库中读取数据，然后分别写入Redis、MongoDB和Elasticsearch。过程包括配置数据库连接，使用Node.js操作数据库，以及设置数据映射和存储结构。这是一个多步骤的数据处理和存储流程，适用于需要实时数据同步的场景。

摘要由CSDN通过智能技术生成

1、目录结构

2、从mysql中取数据

npm install mysql

config/mysql_config.js

module.exports = {
    options: {
        host: '192.168.3.74',
        user: 'root',
        password: '123456',
        database: 'testdata'
    },
    sql: 'select * from testdata_check',
}

mysql/read_mysql.js

const mysql = require('mysql')

let connect = (options) => {
    let connection = mysql.createConnection(options)
    return new Promise((resolve, reject) => {
        connection.connect(function (err) {
            if (err) {
                console.error('error connecting: ' + err.stack);
                return;
            }
            console.log('mysql connected as id ' + connection.threadId);
            resolve(connection)
        });
    })
}

let read = (connection, sql) => {
    return new Promise(((resolve, reject) => {
        connection.query(sql, function (err, results, filed) {
            resolve(results)
        })
    }))
}

module.exports = {
    connect,
    read
}

2、写入redis，存储为hash

npm install redis

redis/operate_redis.js

const redis = require('redis')

let con = (url) => {
    return new Promise((resolve, reject) => {
        let client = redis.createClient({url: url})
        client.on('error', (err) => console.log('Redis Client Error', err));
        client.on('connect', () => console.log('Redis Client Connected'))
        client.connect().then(r => {
        })
        resolve(client)
    })
}
let write = (con, arraylist) => {
    return new Promise((resolve, reject) => {
        for (let i in arraylist) {
            if (arraylist.hasOwnProperty(i)) {
                if (arraylist[i]['id'] && arraylist[i]['task_instance_id']) {
                    con.hSet(arraylist[i]['id'], arraylist[i]['task_instance_id'], arraylist[i]['crawl_content']);
                }
            }
        }
        resolve(arraylist.length)
    })
}
module.exports = {
    con,
    write
}

run.js

const {read, connect} = require('../mysql/read_mysql');
const {write, con} = require('./operate_redis');
const mysql_config = require('../config/mysql_config');
const redis_config = require('../config/redis_config');

(async () => {
    let c = await connect(mysql_config.options)
    let array = await read(c, mysql_config.sql)
    let redis_client = await con(redis_config.redis_url)
    await write(redis_client, array)
    await c.end()
    await redis_client.disconnect()
})()

config/redis_config.js

module.exports = {
    redis_url: 'redis://10.7.128.15:6379'
}

3、写入mongodb

npm install mongoose

mongodb/write.js

const mongoose = require('mongoose')

let con = (url) => {
    return new Promise(((resolve, reject) => {
        mongoose.connect(url).then(r => {
        })
        let db = mongoose.connection;
        db.on('error', console.error.bind(console, 'connection error:'));
        db.once('open', function () {
            console.log('mongodb connected')
        });
        resolve(mongoose)
    }))
}

let create_schema = (obj) => {
    return new Promise(((resolve, reject) => {
        let test_schema = mongoose.Schema(obj)
        resolve(test_schema)
    }))
}

let create_model = (schema) => {
    return new Promise(((resolve, reject) => {
        let TestModel = mongoose.model('TestModel', schema)
        resolve(TestModel)
    }))
}

let save = (model, array) => {
    return new Promise(((resolve, reject) => {
        for (let i in array) {
            if (array.hasOwnProperty(i)) {
                let m = new model(array[i])
                m.save(function (err, m) {
                    if (err) return console.log(err)
                    else console.log('success')
                })
            }
        }
    }))
}

module.exports = {
    con,
    create_schema,
    create_model,
    save
}

mongodb/run.js

const {con, create_model, create_schema, save} = require('./write');
const mongo_config = require('../config/mongo_config');
const mysql_config = require('../config/mysql_config');
const {connect, read} = require('../mysql/read_mysql');
(async () => {
    let c = await connect(mysql_config.options)
    let arr = await read(c, mysql_config.sql)
    let db = await con(mongo_config.mongo_url)
    let obj = {
        id: String,
        crawl_cfg_id: String,
        detection_type: String,
        crawl_file_info_id: String,
        task_instance_id: String,
        crawl_content: String,
        col6: String,
        col7: String,
        col8: String
    }
    let test_schema = await create_schema(obj)
    let TestModel = await create_model(test_schema)
    await save(TestModel, arr)
    await c.end()
    db.disconnect()
})()

config/mongo_config.js

module.exports = {
    mongo_url: 'mongodb://10.7.128.16/testdata'
}

4、写入elastchsearch

client方式

npm install @elastic/elasticsearch

const {Client } = require('@elastic/elasticsearch');

(async ()=>{
    const client = new Client({node:'http://10.7.128.15:9200'})
    const result = await client.search({
        index:'index',
        type:'testdata'
    })
    await console.log(result.body)
})()

rest方式

文档地址：https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/search_examples.html

elastchsearch/write.js

const axios = require('axios');
const mysql_config = require('../config/mysql_config');
const es_config = require('../config/elastchsearch_config')
const {connect, read} = require('../mysql/read_mysql');

let options = {
    mappings: {}
}
options.mappings[`${es_config.entity_name}`] = {
    properties: {
        id: {
            type: "text"
        },
        crawl_cfg_id: {
            type: "text"
        },
        detection_type: {
            type: "text"
        },
        crawl_file_info_id: {
            type: "text"
        },
        task_instance_id: {
            type: "text"
        },
        crawl_content: {
            type: "text"
        },
        col6: {
            type: "text"
        },
        col7: {
            type: "text"
        },
        col8: {
            type: "text"
        },
    }
}
let config = {
    method: 'put',
    url: '',
    headers: {},
    body: null
};
let createIndex = (config) => {
    return new Promise(((resolve, reject) => {
        config.body = options
        config.url = `${es_config.url}/${es_config.index_name}`
        axios(config).then().catch(err => {
            console.log('Index already exist')
        })
        resolve()
    }))
}

let save = (config, array) => {
    return new Promise((resolve, reject) => {
        for (let i in array) {
            if (array.hasOwnProperty(i)) {
                config.url = `${es_config.url}/${es_config.index_name}/${es_config.entity_name}/${Number.parseInt(i) + 1}`
                config.body = array[i]
                config.headers['content-type'] = 'application/json; charset=UTF-8'
                axios.put(config.url, config.body, config.headers).then()
            }
        }
        resolve()
    })
};
(async () => {
    let c = await connect(mysql_config.options)
    let r = await read(c, mysql_config.sql)
    await createIndex(config)
    await save(config, r)
    c.end()
})()

config/elastchsearch_config.js

module.exports = {
    url:'http://10.7.128.15:9200',
    index_name:'index',
    entity_name:'testdata'
}