Spark大数据学习之路六 -- RDD的方法两大类转换和行动2.0--filter

 一、pom文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>Spark01</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.5.2</version>
        </dependency>
    </dependencies>

</project>

二、代码示例：

package com.care.spark01.operate;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import java.util.Arrays;
import java.util.List;

public class Spark_Operate_Transform_Filter {
    public static void main(String[] args) {
        final SparkConf conf = new SparkConf();
        conf.setMaster("local");
        conf.setAppName("Spark_Operate_Transform_Filter");
        final JavaSparkContext jsc = new JavaSparkContext(conf);
        final List<Integer> list = Arrays.asList(1, 2, 3, 4);
        final JavaRDD<Integer> rdd = jsc.parallelize(list, 2);

        // TODO RDD的转换方法： filter（过滤）
        // RDD可以根据指定的过滤规则对数据源中的数据进行筛选过滤
        // 如果满足规则（返回结果true），那么数据保留。如果不满足规则(返回结果为false)，那么数据就会丢弃
//        final JavaRDD<Integer> filter = rdd.filter(new Function<Integer, Boolean>() {
//            @Override
//            public Boolean call(Integer num) throws Exception {
//                return num % 2 == 0;
//            }
//        });

        final JavaRDD<Integer> filter = rdd.filter(
                num -> num % 2 == 0
        );

        filter.collect().forEach(System.out::println);

        jsc.close();
    }
}

三、运行结果：

**************************************************************************************************************

  一、pom文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>Spark01</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.5.2</version>
        </dependency>
    </dependencies>

</project>

二、代码示例：

package com.care.spark01.operate;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import java.util.Arrays;

public class Spark_Operate_Transform_FlatMap02 {
    public static void main(String[] args) {
        final SparkConf conf = new SparkConf();
        conf.setMaster("local");
        conf.setAppName("Spark_Operate_Transform_FlatMap02");
        JavaSparkContext jsc = new JavaSparkContext(conf);
        jsc.setLogLevel("ERROR");

        final JavaRDD<String> rdd = jsc.textFile("C:\\Users\\74794\\IdeaProjects\\Spark01\\data\\test.txt");
        // TODO map方法只负责转换数据（A -> B[B1, B3, B3]）,不能将数据拆分后独立使用
        // line => hadoop hive flume
        // string[] => [hadoop, hive, flume]
        // flatmap方法可以将数据拆分后使用（A -> B1, B3, B3）
        final JavaRDD<String> rdd1 = rdd.flatMap(
                line -> Arrays.asList(line.split(" ")).iterator()
        );
        rdd1.collect().forEach(System.out::println);
        jsc.close();
    }
}

三、运行结果：