Spark读写数据（文本、Object、SequenceFile、Hbase）

VIP文章无声---

已于 2022-11-09 15:14:39 修改

阅读量515

点赞数

分类专栏： spark 文章标签： spark 大数据

于 2020-12-04 19:59:46 首次发布

本文链接：https://blog.csdn.net/qq_43605617/article/details/110671854

版权

package com.atguigu.sparkTest.readandwrite

import java.sql.{
   DriverManager, ResultSet}

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{
   FileSystem, Path}
import org.apache.hadoop.hbase.{
   Cell, CellUtil, HBaseConfiguration}
import org.apache.hadoop.hbase.client.{
   Put, Result}
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.rdd.{
   JdbcRDD, NewHadoopRDD, RDD}
import org.apache.spark.{
   SparkConf, SparkContext}
import org.junit.{
   After, Before, Test}
case class Region(id:String,regionName:String)
case class Student(name:String,age:Int,address: String)
class TextTest {
   
  val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("My app")
  val sparkContext = new SparkContext(conf)

  @After
  def close(): Unit = {
   
    sparkContext.stop()
  }

  @Before
  def start(): Unit = {
   
    //本地文件系统
    val fileSystem: FileSystem = FileSystem.get(new Configuration())
    // 输出目录
    val path = new Path("output")
    if (fileSystem.exists(path)) {
   
      fileSystem.delete(path, true)
    }
    fileSystem.close()
  }
  /**
   * spark读Text文件
   */