MongoDB数据迁移MySQL

1.安装MongoDB

https://blog.csdn.net/qq_45664055/article/details/119794071

http://bbs.itying.com/topic/5bd675d80e525017c449479d

1.1 创建repo文件，MongoDB数据库不在yum的官方仓库，通过配置如下文件，可以通过yum下载MongoDB

vi /etc/yum.repos.d/mongodb-org-4.4.repo

[mongodb-org-4.4]
name=MongoDB Repository
baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.4/x86_64/
gpgcheck=0
enabled=1
gpgkey=https://www.mongodb.org/static/pgp/server-4.4.asc

1.2 安装MongoDB

yum install -y mongodb-org

 1.3 卸载MongoDB

 systemctl stop mongod  关闭
yum erase $(rpm -qa | grep mongodb-org)  卸载
#删除数据及日志
rm -rf /var/log/mongodb
rm -rf /var/lib/mongo

 1.4 修改MongoDB配置文件，允许远程连接

vi /etc/mongod.conf

net:
  port: 27017  #数据库端口，如果要远程连接，服务器记得开放端口
  bindIp: 0.0.0.0  # 默认是 127.0.0.1 只限于本机连接，如果要远程连接修改为：0.0.0.0

 1.5 启动MongoDB

service mongod start      centos6的写法
systemctl start mongod    centos7的写法

 1.6 MongoDB的基本操作

客户端进入MongoDB：mongo 
创建数据库       ：use demo
创建表，并且数据 : db.w_demo.insert({"name":"王明","age":"10","grade":"小学"})

2.MongoDB可视化工具

3.创建表、添加测试数据

6685758046e0fb0001dad8e8,340030000B47363438383733,8C780D32F900260383493808CC96,2024-07-04 00:00:00 055
6685758046e0fb0001dad8e9,340030000B47363438383733,8C79A06C39EE65FC81D828307124,2024-07-04 00:00:00 055
6685758046e0fb0001dad8ea,340030000B47363438383733,8C79A06C39EE632C2C12766ABC7D,2024-07-04 00:00:00 055
6685758046e0fb0001dad8eb,340030000B47363438383733,8C780D32381A65EEB9D6ACD107E7,2024-07-04 00:00:00 055
6685758046e0fb0001dad8ec,340030000B47363438383733,8C79A06C39EE65FC83D8242B91FC,2024-07-04 00:00:00 055
6685758046e0fb0001dadb53,180025000847363438383733,02818334223D7A,2024-07-04 00:00:00 125
6685758046e0fb0001dadb54,180025000847363438383733,8C7813B93818F058371851BB46ED,2024-07-04 00:00:00 125
6685758046e0fb0001dadb55,180025000847363438383733,A8001BAF809CEF25E00492C097AD,2024-07-04 00:00:00 125
6685758046e0fb0001dadb56,180025000847363438383733,8D78046A990C8E9DF09019F5FFD9,2024-07-04 00:00:00 125
6685758046e0fb0001dadb57,180025000847363438383733,02C18CB2F5ACA1,2024-07-04 00:00:00 125
6685758046e0fb0001dadb58,180025000847363438383733,200016303DA8AC,2024-07-04 00:00:00 125
6685758046e0fb0001dadb59,180025000847363438383733,02C18CB2F5ACA1,2024-07-04 00:00:00 125
6685758046e0fb0001dadb5a,180025000847363438383733,02C189B8C3FFB4,2024-07-04 00:00:00 125
6685758046e0fb0001dadb5b,180025000847363438383733,8D89805E584FE2AC38F4F65130D7,2024-07-04 00:00:00 125
6685758046e0fb0001dadb5c,180025000847363438383733,02A185BA442656,2024-07-04 00:00:00 125
6685758046e0fb0001dadb5d,180025000847363438383733,8D7805AF9909180C18041613AFAB,2024-07-04 00:00:00 125
6685758046e0fb0001dadb5e,180025000847363438383733,02E18D1AB8F754,2024-07-04 00:00:00 125
6685758046e0fb0001dadb5f,180025000847363438383733,02A184B1B5AC11,2024-07-04 00:00:00 125
6685758046e0fb0001dadb60,180025000847363438383733,80618193580D32DD1EC5D965CAAF,2024-07-04 00:00:00 125
6685758046e0fb0001dadb61,180025000847363438383733,A000019389C80030A40000B08473,2024-07-04 00:00:00 125
6685758046e0fb0001dadb62,180025000847363438383733,A8001235FF731F13FFF453FB3E9D,2024-07-04 00:00:00 125
6685758046e0fb0001dadb63,180025000847363438383733,A00015BDC2980030A400000C9499,2024-07-04 00:00:00 125
6685758046e0fb0001dadb64,180025000847363438383733,02A18639AEDAAD,2024-07-04 00:00:00 125
6685758046e0fb0001dadb65,180025000847363438383733,8D780E409908D120F0482094F4EF,2024-07-04 00:00:00 125
6685758046e0fb0001dadb66,180025000847363438383733,5D75021BAFC19A,2024-07-04 00:00:00 125
6685758046e0fb0001dadb67,180025000847363438383733,02C18930C484A8,2024-07-04 00:00:00 125
6685758046e0fb0001dadb68,180025000847363438383733,A00015BDFFD9F93B2004E186573A,2024-07-04 00:00:00 125

pip install pymongo==4.4

from pymongo import MongoClient

# 创建MongoDB连接
client = MongoClient('hadoop13', 27017)

# 选择数据库，如果不存在则会自动创建
db = client['demo']

# 选择集合，如果不存在则会自动创建
collection = db['y_demo']

# 插入数据
#rawDataContent,revTime,deviceCode

with open('测试数据','r') as file1:
    for line in file1:
        arr = line.split(',')
        print(arr)
        dict = {"rawDataContent": arr[2], "revTime": arr[3].rstrip('\n'), "deviceCode": arr[1]}
        print(dict)
        collection.insert_one(dict)

4.Datax - mongodb reader

https://github.com/alibaba/DataX/blob/master/mongodbreader/doc/mongodbreader.md

https://www.cnblogs.com/LIAOBO/p/13666845.html

1. 源码修改

5.修改datax源码打包

对datax的所有模块进行打包，时间比较长

mvn -U clean package assembly:assembly '-Dmaven.test.skip=true'

指定mongodbreader模块 以及 它所依赖的模块进行打包

mvn -U clean package -pl mongodbreader -am  assembly:assembly '-Dmaven.test.skip=true'

将idea中打的jar包上传到datax的mongodbreader下，替换原本的插件jar包

hutool工具类jar包上传到datax的mongodbreader的libs目录下

6.自己写的项目 已经打jar包

1.添加依赖

2.自定义函数，解析字段，生成系统时间

3. 项目打jar包上传到/opt/installs/datax/plugin/reader/mongodbreader/libs/目录下

7.mysql建表

create table y_demo(
        device_code varchar(100),
        rev_time  varchar(100),
        msg_content  varchar(100),
        icao  varchar(100),
        msg_type  varchar(100),
        handle_time  varchar(100)
)

8.编写datax的json文件,并且测试运行

{
    "job": {
        "content": [
            {
                "reader": {
                    "name": "mongodbreader",
                    "parameter": {
                        "address": ["hadoop13:27017"],
                        "collectionName": "y_demo",
                        "column": [
                         {
                         "name":"deviceCode",
                         "type":"string"
                         },
                         {
                         "name":"revTime",
                         "type":"string"
                         },
                         {
                         "name":"rawDataContent",
                         "type":"string"
                         },
                         {
                         "name":"rawDataContent",
                         "type":"string",
                                                 "functionClassName":"com.demo.ParseContent",
                                                 "functionMethodName":"m1"
                         },
                         {
                         "name":"rawDataContent",
                         "type":"string",
                                                 "functionClassName":"com.demo.ParseContent",
                                                 "functionMethodName":"m2"
                         },
                         {
                         "type":"string",
                                                 "functionClassName":"com.demo.ParseContent",
                                                 "functionMethodName":"m3"
                         }
                        ],
                        "dbName": "demo",
                    }
                },
                "writer": {
                    "name": "mysqlwriter",
                    "parameter": {
                        "column": ["device_code","rev_time","msg_content","icao","msg_type","handle_time"],
                        "connection": [
                            {
                                "jdbcUrl": "jdbc:mysql://hadoop11:3306/test1",
                                "table": ["y_demo"]
                            }
                        ],
                        "password": "123456",
                        "username": "root",
                        "writeMode": "insert"
                    }
                }
            }
        ],
        "setting": {
            "speed": {
                "channel": "1"
            }
        }
    }
}

python bin/datax.py job/xxx.json