spark业务开发-列值替换
输入数据
order_number,order_date,purchaser,quantity,product_id,remark
10001,2016-01-16,1001,1,102,机q器w记e录r
10003,2016-01-17,1002,2,105,人工记录
10002,2016-01-19,1002,3,106,人工补录
10004,2016-02-21,1003,4,107,自然交易
10001,2016-01-16,1001,1,102,机器记录
输出数据
+------------+-------------------+---------+--------+----------+------------+
|order_number| order_date|purchaser|quantity|product_id| remark|
+------------+-------------------+---------+--------+----------+------------+
| 10001|2016-01-16 00:00:00| 1001| 1| 102|机q器w记e录r|
| 10003|2016-01-17 00:00:00| 1002| 2| 105| 人工记录|
| 10002|2016-01-19 00:00:00| 1002| 3| 106| 人工补录|
| 10004|2016-02-21 00:00:00| 1003| 4| 107| 自然交易|
| 10001|2016-01-16 00:00:00| 1001| 1| 102| 计算机记录|
+------------+-------------------+---------+--------+----------+------------+
程序代码
package com.cch.bigdata.spark.process.replace
import com.cch.bigdata.spark.process.AbstractTransform
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.functions.{col, expr, when}
class ValueReplacer extends AbstractTransform{
case class ReplaceColumn(name:String,strategy:Int,old_value:Any,new_value:Any)
private val column = Array[String]("remark")
private val strategy = Array[String]("1")
private val old_value = Array[Any]("机器记录")
private val new_value = Array[Any]("计算机记录")
override def process(): Unit = {
if(column.isEmpty){
throw new RuntimeException("替换列配置不能为空")
}
if(old_value==null){
throw new RuntimeException("替换列被替换值配置不能为空")
}
if(new_value==null){
throw new RuntimeException("替换列新值配置不能为空")
}
var df: DataFrame = loadCsv("src/main/resources/csv/orders.csv",spark)
var index = 0
column.foreach(c=>{
val replaceColumn: ReplaceColumn = ReplaceColumn(c,strategy(index).toInt,old_value(index),new_value(index))
replaceColumn.strategy match {
case 1 =>{
df = valueReplace(df,replaceColumn.name,replaceColumn.old_value, replaceColumn.new_value)
}
case 2 =>{
df = regexpReplace(df,replaceColumn.name,replaceColumn.old_value.toString, replaceColumn.new_value)
}
}
index+=1
})
df.show()
}
def valueReplace(df:DataFrame,columnName:String, oldValue:Any,newValue:Any):DataFrame={
df.withColumn(columnName, when(col(columnName) === oldValue,newValue)
.otherwise(col(columnName)))
}
def regexpReplace(df:DataFrame,columnName:String,regexp:String,newValue:Any):DataFrame={
val exprString:String = "regexp_replace("+columnName+",'"+regexp+"','"+newValue+"')"
df.withColumn(columnName,expr(exprString).alias(columnName))
}
override def getAppName(): String = "值替换"
}
object ValueReplacer{
def main(args: Array[String]): Unit = {
new ValueReplacer().process()
}
}
参数解释
- columns:需要进行值替换的列,字符串数组
- strategy:替换策略,(1:字符串或值替换 2->使用正则表达式替换)
- old_value:需要被替换的值
- new_value:替换后的新值