spark 、 spark SQL 读取HDFS中的文件 处理数据

spark 、 spark SQL 读取HDFS中的文件 处理数据

在IDEA中:

导入 spark-core_2.11的依赖包 版本和安装spark的版本相同

Object ConnTest{
	def main(args: Array[String]): Unit = {
		val conf = SparkConf().setMaster("local[2]").setAppName("app")
		val sc = SparkContext.getOrCreate(conf)
		val RDDText = sc.textFile("hdfs://192.168.56.100:9000/20200106/customers.csv")
		val startTime = System.currentTimeMillis()
		RDDText.map(_.replaceAll("\"","").split(",")(2)).map((_,1)).reduceByKey(_+_).sortBy(_._2).take(5)
		val endTime = System.currentTimeMillis()
		println(endTime-startTime)
	}
}

再多导入 spark-hive_2.11、spark-sql——2.11的依赖包

Object ConnTest2{
	val spark = SparkSession.builder().master("local[2]").appName("app").enableHiveSupport().getOrCreate()
	val rdd = spark.read.format("csv").option("header","true").load("hdfs://192.168.56.100:9000/20200106/customers.csv")
	//这里去除文件中的文字表头
	println(rdd.take(5))
	spark.stop()
}

在Linux中:

读取HDFS中的文件

scala> case class Customers(custid:String.lname:String,fname:String,cardno:String,addr:String,area:String,city:String,language:String,score:String)
scala> val custs = sc.textFile("hdfs://192.168.56.100:9000/20200106/customers.csv").map(_.replaceAll("\"","").split(",")).map(x=>Customers(x(0),x(1),x(2),x(3),x(4),x(5),x(6),x(7),x(8)))
scala> custs.toDS.take(5)
val ords = sc.textFile("hdfs://192.168.56.100:9000/20200107/orders.csv").map(_.replaceAll("\"","").split(",")).map(x=>Orders(x(0),x(1),x(2),x(3))).toDS
val items = sc.textFile("hdfs://192.168.56.100:9000/20200107/order_items.csv").map(_.replaceAll("\"","").split(",")).map(x=>OrderItem(x(0),x(1),x(2),x(3),x(4),x(5))).toDS
items.groupBy("ordid").agg(sum($"cp").as("countPrice")).where("ordid=2").show
val tabitem = items.groupBy("ordid").agg(sum($"cp").as("countPrice"))
val tabord = tabitem.join(ords,tabitem("ordid")===ords("ordid")).drop(ords("ordid"))
tabord.where("ordid=1").show
tabord.orderBy(desc("countPrice")).limit(1).show

读取虚拟机本地文件

scala> val rdd = sc.textFile("file:///opt/soft/data/customers.csv")
scala> rdd.flatMap(_.split(",")).map((_,1)).reduceByKey(_+_).collect
//词频统计
scala> a.flatMap(_.split(" ")).map(_,1).groupBy(_._1).mapValue(x=>x.map(y=>y._2).sum.foreach(println(_)))
scala> a.flatMap(_.split(" ")).map(_,1).reduceByKey(_+_).collect