[数据库]数据库迁移DEMO

const MongoClient = require('mongodb').MongoClient;
const async = require('async');
const http = require('http');
const https = require("https")
const NET_RESPONSE_TIMEOUT = 2000; //  Response time out.
const NET_RESPONSE_EXCEED_MAX_SIZE = 2001; //  Reponse data is too large.
const NET_HTTP_REQUEST_ERROR = 2002;   //  Error occured when using http/https to send a request.
const NET_RESPONSE_FORMAT_ERROR = 2003;
let defaultArea = "us";
let cfg = {
    "oldDB": {
        "uri": "mongodb://172.20.9.16:27017",
        "user": "app",
        "password": "Mnm3FQt8v2DykM42",
        "dbName": "photo"
    },
    "shardDB": {
        "uri": "mongodb://172.20.222.189:27117",
        "user": "app",
        "password": "8sj5jc2y877e",
        "dbName": "photo"
    }
}
// let cfg = {
//     "oldDB": {
//         "uri": "mongodb://127.0.0.1:27017",
//         "dbName": "photo"
//     },
//     "shardDB": {
//         "uri": "mongodb://127.0.0.1:27017",
//         "dbName": "photo_new"
//     }
// }
let registerCfg = {
    "dim": {
        "https": "false",
        "host": "172.20.222.170",
        "port": 8029,
        "path": "/devmanager.do",
        "userid": "device_log",
        "password": "vJ7j5yeLPs7J9zNe"
    },
    "users": {
        "https": "false",
        "host": "172.20.222.169",
        "port": "8001",
        "path": "/user.do",
        "userid": "device_log",
        "password": "h6GkRWTxgyvMY5Tx"
    }
}
// let registerCfg = {
//     "dim": {
//         "https": "false",
//         "host": "172.20.222.170",
//         "port": 8029,
//         "path": "/devmanager.do",
//         "userid": "device_log",
//         "password": "vJ7j5yeLPs7J9zNe"
//     },
//     "users": {
//         "https": "true",
//         "host": "portal-ww.ecouser.net/api/users",
//         "port": "443",
//         "path": "/user.do",
//         "userid": "device_log",
//         "password": "h6GkRWTxgyvMY5Tx"
//     }
// }
let successCount = 0;
let failCount=0;
let totalCount=0;
let timer = setInterval(() => {
    console.log({ "totalCount": totalCount, "successCount": successCount, "failCount": failCount })
}, 1000 * 5)
let registerTables = ["photo","user_current_usage"]
let schema = {
    "photo": [
        [{ uid: 1, ts: 1, shardArea: 1, isDelete: 1}],
        [{ uid: 1, type: 1, shardArea: 1, isDelete: 1}],
        [{ uid: 1, shardArea: 1 }],
        [{ _id: 1 }]
    ],
    "user_current_usage": [
        [{ uid: 1, shardArea: 1},{ unique: true }]
    ],
}
async.parallel([
    (callback) => {
        setupMongo(cfg.oldDB, (err, conn) => {
            callback(err, conn)
        })
    },
    (callback) => {
        setupMongo(cfg.shardDB, (err, conn) => {
            callback(err, conn)
        })
    }
], (err, conns) => {
    if (err) {
        console.log("setup db fail: ", JSON.stringify(err));
        return
    }
    let oldClient = conns[0]
    let shardClient = conns[1]
    async.eachSeries(registerTables, (table, callback) => {
        console.log(table)
        let colColl = oldClient.db(cfg.oldDB.dbName).collection(table);
        let shardColl = shardClient.db(cfg.shardDB.dbName).collection(table);

        createMongoIndex(shardClient, cfg.shardDB.dbName, table, (err) => {
            if (err) {
                return callback(err)
            }
            let cursor = colColl.find();

            async.eachLimit(cursor, 20, (document, cb) => {
                migrage(shardColl, document, (err, res) => {
                    cb(null)
                });
            }, (err, res)=>{
                callback(null)
            })
        })


    }, (err) => {
        if (err) {
            return console.log("migarge series interrupt", err)
        }
        console.log("migarge series finish")
        oldClient.close();
        shardClient.close();
        clearInterval(timer);

    })
})

async function migrage(coll, document, cb) {
    totalCount++;
    let shardArea;
    let uid;
    if (document.uid) {
        uid = document.uid
    } else {
        if (document.did ) {
            let did = document.did 
            let result = await getDeviceBindedUsers(did)
            if (result.err) {
                console.log("get userid fail: " + JSON.stringify(result))
                shardArea = defaultArea
            } else {
                uid = result.uid
            }
        }
    }

    if (!uid) {
        shardArea = defaultArea
    } else {
        let result = await getUserRegistArea(uid)
        if (result.err) {
            console.log("get area fail: " + JSON.stringify(result))
            shardArea = defaultArea
        } else {
            shardArea = result.shardArea;
        }
    }
    document.shardArea = shardArea || defaultArea
    try {
        await coll.insertOne(document)
        successCount++;
    } catch (error) {
        // if (error.code != 11000) {
        //     console.log("insert fail: ", document._id)
        // }
        console.log("insert fail: ", document._id,error)
        failCount++;
    }
    cb(null)
}
function setupMongo(opts, cb) {
    let url = opts.uri
    let options = {
        useNewUrlParser: true,
        autoReconnect: true,
        reconnectInterval: 3000,
        reconnectTries: Number.MAX_SAFE_INTEGER
        maxPoolSize: 30
    }
    if (opts.user && (opts.password || opts.password === "")) {
        options.auth = {
            user: opts.user,
            password: opts.password
        }
    }

    MongoClient.connect(url, options, (err, client) => {
        if (err) {
            return cb(err)
        }
        cb(err, client)
    })
}

function createMongoIndex(client, db, table, cb) {
    let schemas = schema[table]
    async.eachSeries(schemas, function (schemaa, eachCb) {
        let filed = schemaa[0], option = schemaa[1]
        client.db(db).collection(table).createIndex(filed, option, (err, indexName) => {
            eachCb(err)
        })
    }, function (err) {
        cb(err)
    })
}
function getDeviceBindedUsers(did) {
    return new Promise((resolve, reject) => {
        let data = {
            "td": "GetDeviceBindedUsers",
            "auth": {
                "with": "password",
                "userid": registerCfg.dim.userid,
                "password": registerCfg.dim.password
            },
            "did": did
        }
        SimpleJsonPost(registerCfg.dim.https == true ? https : http, registerCfg.dim.host, registerCfg.dim.port, registerCfg.dim.path, data, 1024 * 1024 * 10, 8000, (err, resp) => {
            if (err) {
                return resolve({ err })
            }
            let uid;
            if (resp.ret == "ok" && resp.users) {
                for (let key in resp.users) {
                    if (key == resp.users[key]["authFrom"]) {
                        uid = key;
                        break;
                    }
                }
                if (uid) {
                    resolve({ uid })
                } else {
                    resolve({ err: resp })

                }
            } else {
                return resolve({ err: resp });

            }
        })
    })

}
function getUserRegistArea(uid) {
    return new Promise((resolve, reject) => {
        let data = {
            "auth": {
                "with": "password",
                "userid": registerCfg.users.userid,
                "password": registerCfg.users.password
            },
            "todo": "GetUserRegistArea",
            "userid": uid
        }
        SimpleJsonPost(registerCfg.users.https == true ? https : http, registerCfg.users.host, registerCfg.users.port, registerCfg.users.path, data, 1024 * 1024 * 10, 8000, (err, resp) => {
            if (err) {
                return resolve(err);
            }
            if (resp.result == "ok" && resp.userRegistArea) {
                return resolve({ shardArea: resp.userRegistArea.toLowerCase() });
            } else {
                return resolve({ err: resp })
            }

        })
    })

}
function SimplePost(http, hostname, port, path, headers, data, respMaxSize, timeoutValue, cb) {
    var options = {
        hostname: hostname,
        port: port,
        path: path,
        method: 'POST',
        headers: headers,
        rejectUnauthorized: false,
        cbd: false,
    };

    var req = http.request(options, function (res) {
        var body = "";

        res.on('data', function (chunk) {
            body += chunk;
            if (body.length > respMaxSize) {
                req.abort();
                cb({ errno: NET_RESPONSE_EXCEED_MAX_SIZE, error: "response too large" + body });
                options.cbd = true;
            }
        });
        res.on('end', function () {
            if (!options.cbd) {
                cb(null, body);
                options.cbd = true;
            }
        });
    }).on('error', function (e) {
        if (!options.cbd) {
            cb({ errno: NET_HTTP_REQUEST_ERROR, error: e.toString() });
            options.cbd = true;
            // stderr("SimplePost error");
        }
    });
    req.on('socket', function (socket) {
        socket.setTimeout(timeoutValue);
        socket.on('timeout', function () {
            req.abort();
            cb({ errno: NET_RESPONSE_TIMEOUT, error: "request timed out" });
            options.cbd = true;
            // stderr("SimplePost timeout");
        });
    });
    req.write(data);
    req.end();
}
function SimpleJsonPost(http, hostname, port, path, data, respMaxSize, timeoutValue, cb) {
    data = JSON.stringify(data);
    if (!data) {
        cb({ errno: 500, error: "request not in JSON format" }, null);
        return;
    }
    SimplePost(http, hostname,
        port,
        path,
        {
            "Content-Type": "application/json",
            "Content-Length": data.length
        },
        data,
        respMaxSize,
        timeoutValue,
        function (err, resp) {
            if (err) {
                cb(err, null);
                return;
            }
            var respJ;
            try {
                respJ = JSON.parse(resp);
            } catch (err) {
                respJ = null;
            }
            if (!respJ) {
                cb({ errno: NET_RESPONSE_FORMAT_ERROR, error: "response not in JSON format: " + resp }, null);
                return;
            }
            cb(null, respJ);
        });
}