jps
cd /apps/hadoop/sbin
./start-all.sh
mkdir -p /data/spark5
cd /data/spark5
wget http://192.168.1.150:60000/allfiles/spark5/orders
wget http://192.168.1.150:60000/allfiles/spark5/order_items
hadoop fs -mkdir /myspark5
hadoop fs -put /data/spark5/orders /myspark5
hadoop fs -put /data/spark5/order_items /myspark5
spark-shell
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import sqlContext.implicits._
case class Orders(order_id:String,order_number:String,buyer_id:String,create_dt:String)
val dforders = sc.textFile("/myspark5/orders").map(_.split('\t')).map(line=>Orders(line(0),line(1),line(2),line(3))).toDF()
dforders.registerTempTable("orders")
sqlContext.sql("show tables").map(t=>"tableName is:"+t(1)).collect().foreach(println)
sqlContext.sql("select order_id,buyer_id from orders").collect
cd /data/spark5/
vim CheckSpark1
import org.apache.spark.sql._
import org.apache.spark.sql.types._
val rddorder_items = sc.textFile("/myspark5/order_items")
val roworder_items = rddorder_items.map(_.split("\t")).map( p=>Row(p(0),p(1),p(2) ) )
val schemaorder_items = "item_id order_id goods_id"
val schema = StructType(schemaorder_items.split(" ").map(fieldName=>StructField(fieldName,StringType,true)) )
val dforder_items = sqlContext.applySchema(roworder_items, schema)
dforder_items.registerTempTable("order_items")
sqlContext.sql("show tables").map(t=>"tableName is:"+t(1)).collect().foreach(println)
sqlContext.sql("select order_id,goods_id from order_items ").collect
sqlContext.sql("select orders.buyer_id, order_items.goods_id from order_items join orders on order_items.order_id=orders.order_id ").collect
cd /data/spark5/
vim CheckSpark2
spark-sql
create table orders (order_id string,order_number string,buyer_id string,create_dt string)
row format delimited fields terminated by '\t' stored as textfile;
create table order_items(item_id string,order_id string,goods_id string)
row format delimited fields terminated by '\t' stored as textfile;
load data local inpath '/data/spark5/orders' into table orders;
load data local inpath '/data/spark5/order_items' into table order_items;
select * from orders;
select * from order_items;
select orders.buyer_id, order_items.goods_id from order_items join orders on order_items.order_id=orders.order_id;
cd /data/spark5/
spark-sql --num-executors 100 -e "show tables" > results.txt
cat /data/spark5/results.txt
spark
于 2024-05-07 08:53:19 首次发布