使用spark-shell 通过groupByKey方法将行专列。
一、需求
在HDFS或本地目录有一份text格式数据(数据内容为英文逗号分隔,字段内容为uid,value,key),现在要求将这些数据按照uid分组,最后保存的格式为uid,uidValue|key1,value1|key2,value2|… 。具体参见下方:
//输入数据
cat /home/hadoop/data/source.csv
8888880674736793701,1003422322c6c937c63af250637fb07e89012,a_zmxy_bindid
8888880674736793701,82a5d0ff83cdd46f8aed902a02105e9f,id_no
8888880674736793701,098dcdb655a715d3816ee8d4854f5db9,regist_mobile
8888880674736793701,c4de399c388097b9a3909bb94cfd51a1,travel_emerg_mobile
8888880674736793702,82a5d0ff83cdd46f8aed902a02105e93,id_no
8888880674736793702,1003422322c6c937c63af250637fb07e89013,b_zmxy_bindid
8888880674736793702,098dcdb655a715d3816ee8d4854f5db3,regist_mobile
8888880674736793702,c4de399c388097b9a3909bb94cfd51a3,travel_emerg_mobile
8888880674736793704,1003422322c6c937c63af250637fb07e89014,a_zmxy_bindid
8888880674736793704,82a5d0ff83cdd46f8aed902a02105e94,id_no
8888880674736793704,098dcdb655a715d3816ee8d4854f5db4,regist_mobile
8888880674736793704,c4de399c388097b9a3909bb94cfd51a4,travel_emerg_mobile
8888880674736793704,c4de399c388097b9a3909bb94cfd51h4,home_emerg_mobile
8888880674736793965,6fab6c3dd609da82acc4b4a48713bdff,regist_mobile
8888880674736793965,c397e3055a5a60566d4a2670979ee0b4,wx_id
8888880674736793965,ws0gnv-ec:01:ee:38:c4:fa,work_wifimac
8888880674736793965,196985b4d3922813b15fe9790c689b1e15dc6,a_zmxy_bindid
8888880674736793965,f4f954dbe1be7bb14643041099a0ed2d,id_no
8888880674736793965,863264039353592,travel_imei
8888880674736793965,ws0gnv-ec:01:ee:38:c4:fa,rest_wifimac
8888880674736793965,7hfL8qWJQGmlH4H-W2Jgbg,travel_oid
//期待输出数据
8888880674736793965,uid|6fab6c3dd609da82acc4b4a48713bdff,regist_mobile|c397e3055a5a60566d4a2670979ee0b4,wx_id|ws0gnv-ec:01:ee:38:c4:fa,work_wifimac|196985b4d3922813b15fe9790c689b1e15dc6,a_zmxy_bindid|f4f954dbe1be7bb14643041099a0ed2d,id_no|863264039353592,travel_imei|ws0gnv-ec:01:ee:38:c4:fa,rest_wifimac|7hfL8qWJQGmlH4H-W2Jgbg,travel_oid
8888880674736793702,uid|1003422322c6c937c63af250637fb07e89013,a_zmxy_bindid|82a5d0ff83cdd46f8aed902a02105e93,id_no|1003422322c6c937c63af250637fb07e89013,b_zmxy_bindid|098dcdb655a715d3816ee8d4854f5db3,regist_mobile|c4de399c388097b9a3909bb94cfd51a3,travel_emerg_mobile
8888880674736793704,uid|1003422322c6c937c63af250637fb07e89014,a_zmxy_bindid|82a5d0ff83cdd46f8aed902a02105e94,id_no|1003422322c6c937c63af250637fb07e89014,b_zmxy_bindid|098dcdb655a715d3816ee8d4854f5db4,regist_mobile|c4de399c388097b9a3909bb94cfd51a4,travel_emerg_mobile|c4de399c388097b9a3909bb94cfd51h4,home_emerg_mobile
8888880674736793701,uid|1003422322c6c937c63af250637fb07e89012,a_zmxy_bindid|82a5d0ff83cdd46f8aed902a02105e9f,id_no|1003422322c6c937c63af250637fb07e89012,b_zmxy_bindid|098dcdb655a715d3816ee8d4854f5db9,regist_mobile|c4de399c388097b9a3909bb94cfd51a1,travel_emerg_mobile
二、spark-shell scala 实现
#原始本地数据 /home/hadoop/data/source.csv
$SPARK_HOME/bin/spark-shell
//spark 列转行
//(uid),(uid,value,key)
scala> val lines = sc.textFile("/home/hadoop/data/source.csv").map(_.split(",")).keyBy(a => a(0)).cache()
scala> lines.collect
#打印的内容省略,后续不在强调
#将map中的value数组类型变成字符串类型,用英文逗号链接
scala> var rdd1= lines.mapValues(v =>v.mkString(","))
rdd1: org.apache.spark.rdd.RDD[(String, String)] = MapPartitionsRDD[39] at mapValues at <console>:28
scala> rdd1.collect
#groupByKey(uid) 按照key group by
scala> val rdd2 = rdd1.groupByKey();
rdd2: org.apache.spark.rdd.RDD[(String, Iterable[String])] = ShuffledRDD[40] at groupByKey at <console>:30
scala> rdd2.collect
#将分组后的数据改成可识别的形式,打印结果
rdd2.map{ /* 分组,最重要的就是这,同类的数据分组到一起,后面只需要计算V了 */
case (k, v) =>
var vt = "" /* 定义vt存数据的变量,恩,这很java,一般scala中很少见到var */
v.foreach { /* 遍历需要计算的V */
x =>
val i = x.indexOf(",")
vt ++= x.substring(i+1) + "|"
}
val s=vt.dropRight(1) /*删除最后一个字符串即|*/
s"$k,uid|$s" /* 拼字符串,返回数据 */
}
.foreach(println)
val rdd3 = rdd2.map{ /* 分组,最重要的就是这,同类的数据分组到一起,后面只需要计算V了 */
case (k, v) =>
var vt = "" /* 定义vt存数据的变量,恩,这很java,一般scala中很少见到var */
v.foreach { /* 遍历需要计算的V */
x =>
val i = x.indexOf(",")
vt ++= x.substring(i+1) + "|"
}
val s=vt.dropRight(1) /*删除最后一个字符串即|*/
s"$k,uid|$s" /* 拼字符串,返回数据 */
}
scala> rdd3.collect
#保存结果到本地
scala> rdd3.saveAsTextFile("/home/hadoop/data/adjacent_list.csv")
[hadoop@bigdata-k-01 adjacent_list.csv]$ pwd
/home/hadoop/janusgraph-data/adjacent_list.csv
三、验证结果
[hadoop@bigdata-k-01 adjacent_list.csv]$ pwd
/home/hadoop/data/adjacent_list.csv
[hadoop@bigdata-k-01 adjacent_list.csv]$ ll
总用量 8
-rw-r--r-- 1 hadoop hadoop 912 3月 23 18:06 part-00000
-rw-r--r-- 1 hadoop hadoop 263 3月 23 18:06 part-00001
-rw-r--r-- 1 hadoop hadoop 0 3月 23 18:06 _SUCCESS
[hadoop@bigdata-k-01 adjacent_list.csv]$ cat *
8888880674736793965,uid|6fab6c3dd609da82acc4b4a48713bdff,regist_mobile|c397e3055a5a60566d4a2670979ee0b4,wx_id|ws0gnv-ec:01:ee:38:c4:fa,work_wifimac|196985b4d3922813b15fe9790c689b1e15dc6,a_zmxy_bindid|f4f954dbe1be7bb14643041099a0ed2d,id_no|863264039353592,travel_imei|ws0gnv-ec:01:ee:38:c4:fa,rest_wifimac|7hfL8qWJQGmlH4H-W2Jgbg,travel_oid
8888880674736793702,uid|1003422322c6c937c63af250637fb07e89013,a_zmxy_bindid|82a5d0ff83cdd46f8aed902a02105e93,id_no|1003422322c6c937c63af250637fb07e89013,b_zmxy_bindid|098dcdb655a715d3816ee8d4854f5db3,regist_mobile|c4de399c388097b9a3909bb94cfd51a3,travel_emerg_mobile
8888880674736793704,uid|1003422322c6c937c63af250637fb07e89014,a_zmxy_bindid|82a5d0ff83cdd46f8aed902a02105e94,id_no|1003422322c6c937c63af250637fb07e89014,b_zmxy_bindid|098dcdb655a715d3816ee8d4854f5db4,regist_mobile|c4de399c388097b9a3909bb94cfd51a4,travel_emerg_mobile|c4de399c388097b9a3909bb94cfd51h4,home_emerg_mobile
8888880674736793701,uid|1003422322c6c937c63af250637fb07e89012,a_zmxy_bindid|82a5d0ff83cdd46f8aed902a02105e9f,id_no|1003422322c6c937c63af250637fb07e89012,b_zmxy_bindid|098dcdb655a715d3816ee8d4854f5db9,regist_mobile|c4de399c388097b9a3909bb94cfd51a1,travel_emerg_mobile