ConfigParser模块在python中用来读取配置文件,配置文件的格式跟windows下的ini配置文件相似,可以包含一个或多个节(section), 每个节可以有多个参数(键=值)。使用的配置文件的好处就是不用在程序员写死,可以使程序更灵活。
注意:在python 3 中ConfigParser
模块名已更名为configparser
conf文件:
[pyspark_conf]
spark.executor.memory=5g
spark.driver.memory=10g
spark.executor.cores=2
spark.dynamicAllocation.maxExecutors=5
spark.driver.maxResultSize=0
spark.dynamicAllocation.enabled=true
spark.shuffle.service.enabled=true
spark.shuffle.service.port=7338
# -*- conding:utf-8 -*-
import configparser
import os
path=os.getcwd()
config = configparser.ConfigParser()
conf_path=path+"/pyspark_conf.conf"
config.read(conf_path)
def read_conf(section="pyspark_conf"):
return config.items(section=section)
def write_conf(section="pyspark_conf",options={}):
try:
for name,value in options.items():
config.set(section,name,str(value))
config.write(open(conf_path, "w"))
except configparser.DuplicateSectionError as Error:
raise Error
def add_conf(section,options={}):
try:
config.add_section(section)
for name,value in options.items():
config.set(section,name,str(value))
config.write(open(conf_path, "w"))
except configparser.DuplicateSectionError as Error:
raise Error
def delete_conf(section,option_list=[]):
try:
for option in option_list:
config.has_option(section,option)
config.remove_option(section,option)
config.write(open(conf_path,'w'))
except configparser.DuplicateSectionError as Error:
raise Error