实体剥离标签加ID

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.{LongType, StringType, StructField, StructType}

object MakeLabel {
def main(args: Array[String]): Unit = {
val session = SparkSession.builder().master(“local[1]”).appName(“make_label”).getOrCreate()
import session.implicits._
// 读取csv文件
val csvFile = session.read.option(“header”,“true”).csv("./data/ownthink_v2.csv")
csvFile.printSchema()
val csvRDD = csvFile.rdd.repartition(1)
var num1 = 0L
// 处理第一列
val res1 = csvRDD.filter(x => {
!(x.get(0) == null)
}).map(x => {x.get(0).toString}).distinct().map(words => {
num1 = num1 + 1
(“ID_”+num1,words,“ENTITY”)
})
var num2 = res1.count()
// 处理第三列
val res2 = csvRDD.filter(x => {
!(x.get(2) == null)
}).map(x => {x.get(2).toString}).distinct().map(words => {
num2 = num2 + 1
(“ID_”+num2,words,“ENTITY1”)
})

val dataFrame1 = res1.toDF(":ID","name",":LABLE")
val dataFrame2 = res2.toDF(":ID","name",":LABLE")
dataFrame1.write.option("header", "true").csv("./data/res1")
dataFrame2.write.option("header", "true").csv("./data/res2")

}
}