Spark JDBC操作mysql

最新推荐文章于 2023-07-27 17:45:00 发布

一只小菜鸟(*￣︶￣)

最新推荐文章于 2023-07-27 17:45:00 发布

阅读量746

点赞数 1

分类专栏： Spark 文章标签： Spark Spark JDBC Spark JDBC操作mysql

本文链接：https://blog.csdn.net/mao502010435/article/details/89792041

版权

Spark 专栏收录该内容

20 篇文章 0 订阅

订阅专栏

package com.mao.scala.java;

import org.apache.spark.SparkConf;
import org.apache.spark.sql.*;

import java.util.Properties;

/**
 * spark jdbc操作mysql
 */
public class SQLJDBCJava {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf();
        conf.setMaster("local") ;
        conf.setAppName("SQLJava");
        SparkSession sess = SparkSession.builder()
                            .appName("SQLJava")
                            .config("spark.master","local")
                            .getOrCreate();
        String url = "jdbc:mysql://192.168.174.1:3306/mybatis" ;
        String table = "persons" ;

        //查询数据库
        Dataset<Row> df = sess.read()
                .format("jdbc")
                .option("url", url)
                .option("dbtable", table)
                .option("user", "root")
                .option("password", "root")
                .option("driver", "com.mysql.jdbc.Driver")
                .load();
        df.show();

        //投影查询
        Dataset<Row> df2 = df.select(new Column("name"),new Column("phone"));
        //过滤
        df2 = df2.where("phone like '135%'");
        //去重
        df2 = df2.distinct();


        Properties prop = new Properties();
        prop.put("user", "root");
        prop.put("password", "root");
        prop.put("driver", "com.mysql.jdbc.Driver");

        //写入
        df2.write().jdbc(url,"subpersons",prop);
        df2.show();

    }
}