Pyspark:dropDuplicates去重获取每个人最新的一条数据

#/bin/python3 
from pyspark.sql import Row
from pyspark.sql import functions as F

#按时间time取每个人name的最近的一条数据
def main(sparkSession):
   
	df = sc.parallelize([\
    Row(name='A',time='20200221',age='18'),\
    Row(name='A',time='20200221',age='18'),\
    Row(name='A',time='20200223',age='28'),\
    Row(name='A',time='20200223',age='30'),\
    Row(name='A',time='20200330',age='30'),\
    Row(name='B',time='20200225',age='21'),\
    Row(name='B',time='20200226',age='21'),\
    Row(name='C',time='20200228',age='21'),\
    Row(name='C',time='20200226',age='21')\
    ]).toDF()
    df.dropDuplicates().show()
    ddf=df.dropDuplicates(['name','time'])
    ddf.show()
    ddf0=ddf.orderBy(F.col('time').desc())
    ddf1= ddf0.dropDuplicates(['name'])
    ddf1.show()

#/bin/python3 
from pyspark.sql import Row
from pyspark.sql import functions as F

#按时间time取每个人name的最近的一条数据
def main(sparkSession):
    
    df = sc.parallelize([\
    Row(name='A',time='20200221',age='18'),\
    Row(name='A',time='20200221',age='18'),\
    Row(name='A',time='20200223',age='28'),\
    Row(name='A',time='20200223',age='30'),\
    Row(name='A',time='20200330',age='30'),\
    Row(name='B',time='20200225',age='21'),\
    Row(name='B',time='20200226',age='21'),\
    Row(name='C',time='20200228',age='21'),\
    Row(name='C',time='20200226',age='21')\
    ]).toDF()
	#step1:按整列entire去重 保留遇到第一列
    df.dropDuplicates().show()
	[2021-04-18 09:01:59.459] [INFO] - +---+----+--------+
	|age|name|    time|
	+---+----+--------+
	| 30|   A|20200330|
	| 28|   A|20200223|
	| 21|   B|20200226|
	| 21|   B|20200225|
	| 21|   C|20200228|
	| 21|   C|20200226|
	| 30|   A|20200223|
	| 18|   A|20200221|
	+---+----+--------+
	#step2:按指定列去重
    ddf=df.dropDuplicates(['name','time'])
    ddf.show()
	[2021-04-18 09:02:00.507] [INFO] - +---+----+--------+
	|age|name|    time|
	+---+----+--------+
	| 21|   C|20200226|
	| 21|   B|20200226|
	| 21|   C|20200228|
	| 18|   A|20200221|
	| 30|   A|20200330|
	| 21|   B|20200225|
	| 28|   A|20200223|
	+---+----+--------+
    #step3:按时间降序排序，去重保留遇到的第一列，即获取最近的日期那一条
    ddf0=ddf.orderBy(F.col('time').desc())
    ddf1= ddf0.dropDuplicates(['name'])
    ddf1.show()
	[2021-04-18 09:02:04.534] [INFO] - +---+----+--------+
	|age|name|    time|
	+---+----+--------+
	| 21|   B|20200226|
	| 21|   C|20200228|
	| 30|   A|20200330|
	+---+----+--------+
```