2021-07-20

spark
https://github.com/databricks/koalas
https://github.com/sparklingpandas/sparklingpandas
https://github.com/LearningJournal/Spark-Programming-In-Python
https://github.com/hanhanwu/Hanhan-Spark-Python
from pyspark.sql import SparkSession
https://blog.csdn.net/lrxcmwy2/article/details/82313149?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522162678913216780262528108%2522%252C%2522scm%2522%253A%252220140713.130102334…%2522%257D&request_id=162678913216780262528108&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2_alltop_positive~default-3-82313149.first_rank_v2_pc_rank_v29&utm_term=spark&spm=1018.2226.3001.4187

from lib.logger import Log4j

if name == “main”:
spark = SparkSession
.builder
.appName(“Shuffle Join Demo”)
.master(“local[3]”)
.getOrCreate()

logger = Log4j(spark)

flight_time_df1 = spark.read.json("data/d1/")
flight_time_df2 = spark.read.json("data/d2/")

spark.conf.set("spark.sql.shuffle.partitions", 3)

join_expr = flight_time_df1.id == flight_time_df2.id
join_df = flight_time_df1.join(flight_time_df2, join_expr, "inner")

join_df.collect()
input("press a key to stop...")