const MongoClient = require('mongodb').MongoClient;
const async = require('async');
const http = require('http');
const https = require("https")
const NET_RESPONSE_TIMEOUT = 2000; // Response time out.
const NET_RESPONSE_EXCEED_MAX_SIZE = 2001; // Reponse data is too large.
const NET_HTTP_REQUEST_ERROR = 2002; // Error occured when using http/https to send a request.
const NET_RESPONSE_FORMAT_ERROR = 2003;
let defaultArea = "us";
let cfg = {
"oldDB": {
"uri": "mongodb://172.20.9.16:27017",
"user": "app",
"password": "Mnm3FQt8v2DykM42",
"dbName": "photo"
},
"shardDB": {
"uri": "mongodb://172.20.222.189:27117",
"user": "app",
"password": "8sj5jc2y877e",
"dbName": "photo"
}
}
// let cfg = {
// "oldDB": {
// "uri": "mongodb://127.0.0.1:27017",
// "dbName": "photo"
// },
// "shardDB": {
// "uri": "mongodb://127.0.0.1:27017",
// "dbName": "photo_new"
// }
// }
let registerCfg = {
"dim": {
"https": "false",
"host": "172.20.222.170",
"port": 8029,
"path": "/devmanager.do",
"userid": "device_log",
"password": "vJ7j5yeLPs7J9zNe"
},
"users": {
"https": "false",
"host": "172.20.222.169",
"port": "8001",
"path": "/user.do",
"userid": "device_log",
"password": "h6GkRWTxgyvMY5Tx"
}
}
// let registerCfg = {
// "dim": {
// "https": "false",
// "host": "172.20.222.170",
// "port": 8029,
// "path": "/devmanager.do",
// "userid": "device_log",
// "password": "vJ7j5yeLPs7J9zNe"
// },
// "users": {
// "https": "true",
// "host": "portal-ww.ecouser.net/api/users",
// "port": "443",
// "path": "/user.do",
// "userid": "device_log",
// "password": "h6GkRWTxgyvMY5Tx"
// }
// }
let successCount = 0;
let failCount=0;
let totalCount=0;
let timer = setInterval(() => {
console.log({ "totalCount": totalCount, "successCount": successCount, "failCount": failCount })
}, 1000 * 5)
let registerTables = ["photo","user_current_usage"]
let schema = {
"photo": [
[{ uid: 1, ts: 1, shardArea: 1, isDelete: 1}],
[{ uid: 1, type: 1, shardArea: 1, isDelete: 1}],
[{ uid: 1, shardArea: 1 }],
[{ _id: 1 }]
],
"user_current_usage": [
[{ uid: 1, shardArea: 1},{ unique: true }]
],
}
async.parallel([
(callback) => {
setupMongo(cfg.oldDB, (err, conn) => {
callback(err, conn)
})
},
(callback) => {
setupMongo(cfg.shardDB, (err, conn) => {
callback(err, conn)
})
}
], (err, conns) => {
if (err) {
console.log("setup db fail: ", JSON.stringify(err));
return
}
let oldClient = conns[0]
let shardClient = conns[1]
async.eachSeries(registerTables, (table, callback) => {
console.log(table)
let colColl = oldClient.db(cfg.oldDB.dbName).collection(table);
let shardColl = shardClient.db(cfg.shardDB.dbName).collection(table);
createMongoIndex(shardClient, cfg.shardDB.dbName, table, (err) => {
if (err) {
return callback(err)
}
let cursor = colColl.find();
async.eachLimit(cursor, 20, (document, cb) => {
migrage(shardColl, document, (err, res) => {
cb(null)
});
}, (err, res)=>{
callback(null)
})
})
}, (err) => {
if (err) {
return console.log("migarge series interrupt", err)
}
console.log("migarge series finish")
oldClient.close();
shardClient.close();
clearInterval(timer);
})
})
async function migrage(coll, document, cb) {
totalCount++;
let shardArea;
let uid;
if (document.uid) {
uid = document.uid
} else {
if (document.did ) {
let did = document.did
let result = await getDeviceBindedUsers(did)
if (result.err) {
console.log("get userid fail: " + JSON.stringify(result))
shardArea = defaultArea
} else {
uid = result.uid
}
}
}
if (!uid) {
shardArea = defaultArea
} else {
let result = await getUserRegistArea(uid)
if (result.err) {
console.log("get area fail: " + JSON.stringify(result))
shardArea = defaultArea
} else {
shardArea = result.shardArea;
}
}
document.shardArea = shardArea || defaultArea
try {
await coll.insertOne(document)
successCount++;
} catch (error) {
// if (error.code != 11000) {
// console.log("insert fail: ", document._id)
// }
console.log("insert fail: ", document._id,error)
failCount++;
}
cb(null)
}
function setupMongo(opts, cb) {
let url = opts.uri
let options = {
useNewUrlParser: true,
autoReconnect: true,
reconnectInterval: 3000,
reconnectTries: Number.MAX_SAFE_INTEGER
maxPoolSize: 30
}
if (opts.user && (opts.password || opts.password === "")) {
options.auth = {
user: opts.user,
password: opts.password
}
}
MongoClient.connect(url, options, (err, client) => {
if (err) {
return cb(err)
}
cb(err, client)
})
}
function createMongoIndex(client, db, table, cb) {
let schemas = schema[table]
async.eachSeries(schemas, function (schemaa, eachCb) {
let filed = schemaa[0], option = schemaa[1]
client.db(db).collection(table).createIndex(filed, option, (err, indexName) => {
eachCb(err)
})
}, function (err) {
cb(err)
})
}
function getDeviceBindedUsers(did) {
return new Promise((resolve, reject) => {
let data = {
"td": "GetDeviceBindedUsers",
"auth": {
"with": "password",
"userid": registerCfg.dim.userid,
"password": registerCfg.dim.password
},
"did": did
}
SimpleJsonPost(registerCfg.dim.https == true ? https : http, registerCfg.dim.host, registerCfg.dim.port, registerCfg.dim.path, data, 1024 * 1024 * 10, 8000, (err, resp) => {
if (err) {
return resolve({ err })
}
let uid;
if (resp.ret == "ok" && resp.users) {
for (let key in resp.users) {
if (key == resp.users[key]["authFrom"]) {
uid = key;
break;
}
}
if (uid) {
resolve({ uid })
} else {
resolve({ err: resp })
}
} else {
return resolve({ err: resp });
}
})
})
}
function getUserRegistArea(uid) {
return new Promise((resolve, reject) => {
let data = {
"auth": {
"with": "password",
"userid": registerCfg.users.userid,
"password": registerCfg.users.password
},
"todo": "GetUserRegistArea",
"userid": uid
}
SimpleJsonPost(registerCfg.users.https == true ? https : http, registerCfg.users.host, registerCfg.users.port, registerCfg.users.path, data, 1024 * 1024 * 10, 8000, (err, resp) => {
if (err) {
return resolve(err);
}
if (resp.result == "ok" && resp.userRegistArea) {
return resolve({ shardArea: resp.userRegistArea.toLowerCase() });
} else {
return resolve({ err: resp })
}
})
})
}
function SimplePost(http, hostname, port, path, headers, data, respMaxSize, timeoutValue, cb) {
var options = {
hostname: hostname,
port: port,
path: path,
method: 'POST',
headers: headers,
rejectUnauthorized: false,
cbd: false,
};
var req = http.request(options, function (res) {
var body = "";
res.on('data', function (chunk) {
body += chunk;
if (body.length > respMaxSize) {
req.abort();
cb({ errno: NET_RESPONSE_EXCEED_MAX_SIZE, error: "response too large" + body });
options.cbd = true;
}
});
res.on('end', function () {
if (!options.cbd) {
cb(null, body);
options.cbd = true;
}
});
}).on('error', function (e) {
if (!options.cbd) {
cb({ errno: NET_HTTP_REQUEST_ERROR, error: e.toString() });
options.cbd = true;
// stderr("SimplePost error");
}
});
req.on('socket', function (socket) {
socket.setTimeout(timeoutValue);
socket.on('timeout', function () {
req.abort();
cb({ errno: NET_RESPONSE_TIMEOUT, error: "request timed out" });
options.cbd = true;
// stderr("SimplePost timeout");
});
});
req.write(data);
req.end();
}
function SimpleJsonPost(http, hostname, port, path, data, respMaxSize, timeoutValue, cb) {
data = JSON.stringify(data);
if (!data) {
cb({ errno: 500, error: "request not in JSON format" }, null);
return;
}
SimplePost(http, hostname,
port,
path,
{
"Content-Type": "application/json",
"Content-Length": data.length
},
data,
respMaxSize,
timeoutValue,
function (err, resp) {
if (err) {
cb(err, null);
return;
}
var respJ;
try {
respJ = JSON.parse(resp);
} catch (err) {
respJ = null;
}
if (!respJ) {
cb({ errno: NET_RESPONSE_FORMAT_ERROR, error: "response not in JSON format: " + resp }, null);
return;
}
cb(null, respJ);
});
}
[数据库]数据库迁移DEMO
最新推荐文章于 2024-09-11 23:37:29 发布