分享一个Python配置管理方法 – 综合配置文件，argparse，传参

本文链接：https://blog.csdn.net/qq_30504647/article/details/131351450

分享一个Python配置管理方法 – 综合配置文件，argparse，传参

最近在写Python代码的时候发现自己的配置项非常混乱，有来自配置文件的，有来自argparse的，有来自函数传参的还有来自全局变量的。
混乱的配置严重影响写代码的效率。于是想把配置项统一管理起来。

需要实现的目标：

支持读取配置文件；
支持使用argparse；
支持函数传参；
要有代码提示（让编辑器识别出变量名字与类型）；
简单好用；

分析后发现，我们可以根据配置文件生成一个类和argparse配置项，写入Python代码文件。
编辑器可以跟据这个类提供的信息生成代码提示，方便开发。

由于配置文件可能会被修改，我们加入文件监视。当配置文件有改动的时候，自动生成目标脚本。

代码实现

需要的库

yaml
jinja2
watchdog

实现的代码如下：

#! python

import argparse
import os
import time

import yaml
from jinja2 import Template
from watchdog.events import FileSystemEvent, FileSystemEventHandler
from watchdog.observers import Observer

_arg = argparse.ArgumentParser("build config")
_arg.add_argument("config_path", type=str, help="The path to config description yaml file.")
_arg.add_argument("-L", "--listen", action="store_true", help="Listen to the config_path. If config is changed, the target py file will be update automatically.")
args = _arg.parse_args()

LIST_SPLIT_CHAR = ", "

def get_type_val(src_type: str, src_val):

    if src_type is None:
        if src_val is None:
            return "Any", None, "str", "\"\""
        src_type = type(src_val).__name__
    
    lwsrc_type = src_type.lower()

    if lwsrc_type == "integer":
        src_type = "int"
    elif lwsrc_type == "string":
        src_type = "str"
    elif lwsrc_type == "boolean":
        src_type = "bool"

    if src_type == "int":
        return "int", src_val, "int", src_val
    if src_type == "float":
        return "float", src_val, "float", src_val
    elif src_type == "str":
        return "str", src_val, "str", src_val
    elif src_type == "list":
        # add a super comple
        src_str = LIST_SPLIT_CHAR.join([str(ss) for ss in src_val])
        return "list", "field(default_factory=list)", "_type_arg_list", f'"{src_str}"'
    elif src_type == "bool":
        return "bool", src_val, "_type_arg_bool", src_val
    else:
        return src_type, src_val, "str", str(src_val)

class MetaVar:
    def __init__(self, name: str, other: dict) -> None:
        odefault = other.get("default", None)
        ohelp = other.get("help", "")
        otype = other.get("type", "Any")

        self.name = name

        self.help = "" if ohelp is None else ohelp

        self.default = odefault

        self.type, self.val, self.arg_type, self.arg_val = get_type_val(otype, odefault)

# Using a dash (-) after the opening tag ({%-) or before the closing tag (-%}) of a block, variable, or comment to indicate that the whitespace before or after that tag should be stripped.

# _arg.add_argument("--ddd", type=str, default="", help="")
out_template = Template("""r\"\"\"
WARN: DO NOT EDIT!!!
WARN: DO NOT EDIT!!!

This file is generated from {{name}}:
    ./build_config.py "{{config_path}}"

WARN: DO NOT EDIT!!!
WARN: DO NOT EDIT!!!
\"\"\"

from dataclasses import dataclass, field
from typing import Any
import argparse

@dataclass
class Config:
    {%- for meta in vardef -%}
        {%- if meta.val is none %}
    {{ meta.name }} : {{ meta.type }} = None # {{meta.help}}
        {%- elif meta.type == "str" %}
    {{ meta.name }} : {{ meta.type }} = "{{ meta.val }}" # {{meta.help}}
        {%- else %}
    {{ meta.name }} : {{ meta.type }} = {{ meta.val }} # {{meta.help}}
        {%- endif -%}
    {% endfor %}
    {% if has_post_init > 0 %}
    def __post_init__(self):
        {% for meta in post_init -%}
        self.{{meta.name}} = {{meta.default}}
        {% endfor %}
    {% endif %}

def _type_arg_list(src: str):
    out = []
    for ss in src.split("{{LIST_SPLIT_CHAR}}"):
        try: 
            xx = float(ss)
            if xx.is_integer():
                out.append(int(xx))
            else:
                out.append(xx)
        except ValueError:
            out.append(ss)
    return out

def _type_arg_bool(src: str):
    if src in ["T", "True", "true", "1"]:
        return True
    elif src in ["F", 'f', "False", "false", "0"]:
        return False
    else:
        raise ValueError(f"{src} is not a boolean value.")

def new_config(**kwargs) -> Config:
    conf = Config()
    for k, v in kwargs.items():
        conf.__dict__[k] = v
    return conf

def new_config_argparse(**kwargs) -> Config:
    _args = argparse.ArgumentParser()
    {%- for meta in vardef -%}
        {%- if meta.arg_val is none %}
    _args.add_argument("--{{meta.name}}", type={{meta.arg_type}}, default=None, help="{{meta.help}}")
        {%- elif meta.type == "str" %}
    _args.add_argument("--{{meta.name}}", type={{meta.arg_type}}, default="{{meta.arg_val}}", help="{{meta.help}}")
        {%- else %}
    _args.add_argument("--{{meta.name}}", type={{meta.arg_type}}, default={{meta.arg_val}}, help="{{meta.help}}")
        {%- endif -%}
    {% endfor %}
    args = _args.parse_args()

    conf = Config()

    for k, v in kwargs.items():
        conf.__dict__[k] = v

    for k, v in vars(args).items():
        conf.__dict__[k] = v
    
    return conf
""")

def write_to_file(conf_path: str):
    if not conf_path.endswith(".yaml"):
        raise ValueError("Only support YAML file. The suffix must be .yaml")

    basename = os.path.basename(conf_path)
    out_name = "config_" + basename.removesuffix(".yaml") + ".py"
    metavars = []
    post_init = []
    with open(conf_path, "r") as stream:
        try:
            YAML_DATA = yaml.safe_load(stream) # type: dict
            for k, v in YAML_DATA.items():
                meta = MetaVar(k, v)
                metavars.append(meta)
                if not meta.arg_type in ["int", "str"]:
                    post_init.append(meta)
        except Exception as exc:
            print(exc)
            return

    try:
        render_output = out_template.render({
            "name": basename,
            "config_path": conf_path,
            "vardef": metavars,
            "LIST_SPLIT_CHAR": LIST_SPLIT_CHAR,
            "post_init": post_init,
            "has_post_init": len(post_init),
        })
    except Exception as e:
        print(e)
        return

    with open(out_name, "w") as out_file:
        out_file.write(render_output)

    print(" Sync config from", conf_path, "to", out_name)

class ConfHandler(FileSystemEventHandler):
    def on_modified(self, event: FileSystemEvent):
        if event.is_directory:
            return
        print("File modified", event.src_path)
        write_to_file(event.src_path)

if args.listen:
    event_handler = ConfHandler()
    observer = Observer()
    observer.schedule(event_handler, path=args.config_path, recursive=False)
    observer.daemon = True
    observer.start()

    write_to_file(args.config_path)
    print(f"Listen on {args.config_path}. Press Ctrl+C to exit.")
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()
    observer.join()
else:
    write_to_file(args.config_path)

效果展示

配置文件

test_var1:
    type: "str"
    default: "test var1"
    help: test string variable.
test_var2:
    type: 
    default: "test var2"
    help: test string variable.
test_var3:
    type: 
    default: 3
    help: test int variable.
test_var4:
    type: 
    default: [1.5,2,3]
    help: test list variable.
test_var5:
    type: 
    default: False
    help: test boolean variable.

运行程序

python build_config.py config_yaml/demo.yaml
# or 
python build_config.py -L config_yaml/demo.yaml

生成的目标脚本文件

r"""
WARN: DO NOT EDIT!!!
WARN: DO NOT EDIT!!!

This file is generated from demo.yaml:
    ./build_config.py "config_yaml/demo.yaml"

WARN: DO NOT EDIT!!!
WARN: DO NOT EDIT!!!
"""

from dataclasses import dataclass, field
from typing import Any
import argparse

@dataclass
class Config:
    test_var1 : str = "test var1" # test string variable.
    test_var2 : str = "test var2" # test string variable.
    test_var3 : int = 3 # test int variable.
    test_var4 : list = field(default_factory=list) # test list variable.
    test_var5 : bool = False # test boolean variable.
    
    def __post_init__(self):
        self.test_var4 = [1.5, 2, 3]
        self.test_var5 = False
        
    

def _type_arg_list(src: str):
    out = []
    for ss in src.split(", "):
        try: 
            xx = float(ss)
            if xx.is_integer():
                out.append(int(xx))
            else:
                out.append(xx)
        except ValueError:
            out.append(ss)
    return out

def _type_arg_bool(src: str):
    if src in ["T", "True", "true", "1"]:
        return True
    elif src in ["F", 'f', "False", "false", "0"]:
        return False
    else:
        raise ValueError(f"{src} is not a boolean value.")

def load_configure(**kwargs) -> Config:
    _args = argparse.ArgumentParser()
    _args.add_argument("--test_var1", type=str, default="test var1", help="test string variable.")
    _args.add_argument("--test_var2", type=str, default="test var2", help="test string variable.")
    _args.add_argument("--test_var3", type=int, default=3, help="test int variable.")
    _args.add_argument("--test_var4", type=_type_arg_list, default="1.5, 2, 3", help="test list variable.")
    _args.add_argument("--test_var5", type=_type_arg_bool, default=False, help="test boolean variable.")
    args = _args.parse_args()

    conf = Config()

    for k, v in kwargs.items():
        conf.__dict__[k] = v

    for k, v in vars(args).items():
        conf.__dict__[k] = v
    
    return conf

使用样例

import config_demo

config = config_demo.load_configure(test_var1="new test var 1")

print(config)
# run 
# python apply_config_demo.py
# python apply_config_demo.py --test_var4 "2, 3"

后记

感谢大家，欢迎讨论交流。

更新说明

20230919:
- 增加函数： def new_config(**kwargs) -> Config:
- 重命名函数：def load_configure(**kwargs) -> Config: => def new_config_argparse(**kwargs) -> Config: