python线程池ThreadPoolExecutor使用案例

 

# -*- coding: utf-8 -*-
from concurrent.futures import wait
from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor

from pyspark import SparkConf, SparkContext
from pyspark.sql import HiveContext
conf = SparkConf()
conf.set("spark.yarn.queue", "queue_name") \
    .set("spark.app.name", "application_name") \
    .set("spark.driver.memory", "20g") \
    .set("spark.executor.cores", "1") \
    .set("spark.executor.memory", "10g") \
    .set("spark.executor.memoryOverhead", "5g") \
    .set("spark.executor.instances", "400") \
    .set("spark.dynamicAllocation.enabled", "true") \
    .set("spark.dynamicAllocation.maxExecutors", "500") \
    .set("spark.rdd.compress", "true") \
    .set("spark.shuffle.compress", "true")

sc = SparkContext(conf=conf)
sp