Flink MongoDBSink

最新推荐文章于 2024-08-07 09:35:05 发布

A_ChunUnique

最新推荐文章于 2024-08-07 09:35:05 发布

阅读量3.7k

点赞数

分类专栏： Flink

本文链接：https://blog.csdn.net/Gavin_chun/article/details/85234411

版权

Flink 专栏收录该内容

2 篇文章 0 订阅

订阅专栏

MongoUtils

package com.soul.utils;

import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;

import java.util.ArrayList;
import java.util.List;

/**
 * @author soulChun
 * @create 2018-12-24-14:08
 */
public class MongoUtils {

    public static MongoClient getConnect(){
        ServerAddress serverAddress = new ServerAddress("localhost", 27017);
        List<MongoCredential> credential = new ArrayList<>();
        //MongoCredential.createScramSha1Credential()三个参数分别为 用户名 数据库名称 密码
        MongoCredential mongoCredential1 = MongoCredential.createScramSha1Credential("root", "soul_db", "root".toCharArray());
        credential.add(mongoCredential1);
        //通过连接认证获取MongoDB连接
        MongoClient mongoClient = new MongoClient(serverAddress, credential);
        return mongoClient;
    }

}

MongoDBSink

package com.soul.kafka;

import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.soul.utils.MongoUtils;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.bson.Document;

import java.util.ArrayList;
import java.util.List;

/**
 * @author soulChun
 * @create 2018-12-24-10:05
 */
public class MongoDBSink extends RichSinkFunction<Tuple5<String, String, String, String, String>> {
    private static final long serialVersionUID = 1L;
    MongoClient mongoClient = null;

    public void invoke(Tuple5<String, String, String, String, String> value) {
        try {
            if (mongoClient != null) {
                mongoClient = MongoUtils.getConnect();
                MongoDatabase db = mongoClient.getDatabase("soul_db");
                MongoCollection collection = db.getCollection("kafka");
                List<Document> list = new ArrayList<>();
                Document doc = new Document();
                doc.append("IP", value.f0);
                doc.append("TIME", value.f1);
                doc.append("CourseID", value.f2);
                doc.append("Status_Code", value.f3);
                doc.append("Referer", value.f4);
                list.add(doc);
                System.out.println("Insert Starting");
                collection.insertMany(list);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void open(Configuration parms) throws Exception {
        super.open(parms);
        mongoClient = MongoUtils.getConnect();
    }

    public void close() throws Exception {
        if (mongoClient != null) {
            mongoClient.close();
        }
    }
}

可以使用此Sink替换Flink清洗Kafka数据存入MySQL测试的MySQLSink