BOT代码解读（Buffer of Thoughts: Thought-Augmented Reasoning with Large Language Models）

注释后的代码仓库见lengyanju8/buffer-of-thought-llmhttps://gitee.com/lengyanju8/buffer-of-thought-llm

bot_pipeline.py文件 也是文章的核心文件了

import transformers
import torch
import http.client
import json
from accelerate import infer_auto_device_map
from meta_buffer_utilis import meta_distiller_prompt,extract_and_execute_code
from test_templates import game24,checkmate,word_sorting
# 加载模型用
class Pipeline:
    def __init__(self,model_id,api_key=None):
        self.api = False
        self.local = False
        self.model_id = model_id
        # 没提供api
        if api_key is None:
            self.local = True
            # 使用transformer加载
            self.pipeline = transformers.pipeline(
        "text-generation",
        model=self.model_id,
        model_kwargs={"torch_dtype": torch.bfloat16},
        device_map = 'auto'
        )
        else:
            self.api = True
            self.api_key = api_key
    # 获取回复 传入meta_prompt和user_prompt,meta_prompt是作为"system"角色的。
    def get_respond(self,meta_prompt,user_prompt):
        if self.api: # 提供了api
            # 调用api得到结果
            conn = http.client.HTTPSConnection("api.openai.com")
            payload = json.dumps({
            "model": self.model_id,
            "messages": [
                {
                    "role": 'assistant',
                    "content": meta_prompt
                },
                {
                    "role": 'user',
                    "content": user_prompt
                },
            ]
            })
            headers = {
            'Accept': 'application/json',
            'Authorization': f'Bearer {self.api_key}',
            'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
            'Content-Type': 'application/json'
            }
            conn.request("POST", "/v1/chat/completions", payload, headers)
            res = conn.getresponse()
            data = res.read()
            text = data.decode("utf-8")
            dict_object = json.loads(text)
            respond = dict_object['choices'][0]['message']['content']
            return respond
        # 没提供api，那就是本地模型了。
        else:
            messages = [
            {"role": "system", "content": meta_prompt},
            {"role": "user", "content": user_prompt},
        ]
            # 使用huggingface的tokenizer模板来格式化输入的信息。
            prompt = self.pipeline.tokenizer.apply_chat_template(
                    messages, 
                    tokenize=False, 
                    add_generation_prompt=True
            )
            # 这个字符串指定了要使用的pipeline类型。在这个例子中，它是文本生成，意味着模型将会根据给定的输入文本继续生成后续的文本。

            terminators = [
                self.pipeline.tokenizer.eos_token_id,
                self.pipeline.tokenizer.convert_tokens_to_ids("<|eot_id|>")
            ]
            # 第一个元素是从tokenizer中获取的默认结束符ID（eos_token_id），第二个元素是通过将特殊标记"<|eot_id|>"转换成对应的token ID。
            # 发送参数。
            outputs = self.pipeline(
                prompt,
                max_new_tokens=2048,
                eos_token_id=terminators, # 设置了终止生成的条件，可以是单个ID或ID列表，这里使用了前面定义的terminators列表。
                do_sample=True,
                temperature=0.4,
                top_p=0.9,
            )
            # 获得response
            respond = outputs[0]["generated_text"][len(prompt):]
            # outputs[0]访问的是第一个批次的结果（通常pipeline只处理一个prompt时只有一个批次），然后从"generated_text"字段获取完整生成的文本。由于我们只对# prompt之后的部分感兴趣，所以使用切片操作[len(prompt):]去除prompt本身的内容，最后返回这部分生成的文本作为响应。
            return respond
            

        
class BoT:
    # 初始化
    def __init__(self, user_input,problem_id,api_key=None,model_id=None):
        self.api_key = api_key
        self.model_id = model_id
        self.pipeline = Pipeline(self.model_id,self.api_key)
        self.user_input = user_input
        # Only for test use, stay tuned for our update
        self.problem_id = problem_id 
        
    def problem_distillation(self):
        print(f'User prompt:{self.user_input}')
        # 传入的meta_distiller_prompt是作为system角色的
        self.distilled_information = self.pipeline.get_respond(meta_distiller_prompt, self.user_input)
        print(f'Distilled information:{self.distilled_information}')
        
    # 这里就是根据问题id来选择模板
    def buffer_retrieve(self):
        # For initial test use, we will later update the embedding retrieval version to support more 
        if self.problem_id == 0:
            self.thought_template = game24
        elif self.problem_id == 1:
            self.thought_template = checkmate
        elif self.problem_id == 2:
            self.thought_template = word_sorting
            
    def reasoner_instantiation(self):
        # Temporay using selection method to select answer extract method
        problem_id_list = [0,1,2]
        self.instantiation_instruct = """
You are an expert in problem analysis and can apply previous problem-solving approaches to new issues. The user will provide a specific task description and a thought template. Your goal is to analyze the user's task and generate a specific solution based on the thought template. If the instantiated solution involves Python code, only provide the code and let the compiler handle it. If the solution does not involve code, provide a final answer that is easy to extract from the text.
        """

        """
        你是问题分析方面的专家，可以将以前解决问题的方法应用于新问题。用户将提供具体的任务描述和思想模板。您的目标是分析用户的任务，并根据思维模板生成特定的解决方案。如果实例化的解决方案涉及Python代码，则只提供代码并让编译器处理。如果解决方案不涉及代码，则提供易于从文本中提取的最终答案。
        """
        # 这里输入了蒸馏问题，还有思维的模板 {self.distilled_information} 用来做系统角色，{self.thought_template}具体问题的模板。
        self.formated_input = f"""
Distilled information:
{self.distilled_information}
Thought template:
{self.thought_template}

Instantiated Solution:
Please analyze the above user task description and thought template, and generate a specific, detailed solution. If the solution involves Python code, only provide the code. If not, provide a clear and extractable final answer.        
        """
        """
        请分析以上用户任务描述和思想模板，并生成具体、详细的解决方案。如果解决方案涉及Python代码，则仅提供代码。如果没有，请提供一个清晰可提取的最终答案。
        """
        # 写入提示词，然后拿到结果。
        self.result = self.pipeline.get_respond(self.instantiation_instruct,self.formated_input)
        print(f'Instantiated reasoning result: {self.result}')
        # 如果问题id在列表中，则执行代码。有可可能有python代码。
        if self.problem_id in problem_id_list:
            self.final_result, code_str = extract_and_execute_code(self.result)
            print(f'The result of code execution: {self.final_result}')
        else:
            self.final_result = self.result 

    
    def bot_run(self):
        # 执行问题蒸馏，缓冲，
        self.problem_distillation()
        self.buffer_retrieve()
        self.reasoner_instantiation()
        return self.final_result
    
    
    
        
           
            
            
        

meta_buffer_utilis.py文件

import re
import io
import sys

meta_distiller_prompt = """

As a highly professional and intelligent expert in information distillation, you excel at extracting essential information to solve problems from user input queries. You adeptly transform this extracted information into a suitable format based on the respective type of the issue. If the problem can be generalized to a higher level to solve multiple issues, further analysis and explanation will be provided upon your next response.
Please categorize and extract the crucial information required to solve the problem from the user's input query. Combining these two elements will generate distilled information. Subsequently, deliver this distilled information, based on the problem type, to your downstream meta planner. The problem type should belong to one of the six categories mentioned above, and the distilled information should include:

1. Values and information of key variables extracted from user input, which will be handed over to the respective expert for task resolution, ensuring all essential information required to solve the problem is provided.
2. The objective of the problem and corresponding constraints.
3. Extend the problem based on 1 and 2, propose a meta problem that can address the user query and handle more input and output variations. Incorporate the real-world scenario of the extended problem along with the types of key variables and information constraints from the original problem to restrict the key variables in the extended problem. After that, use the user query input key information as input to solve the problem as an example.
4. Try to transform the problem into a python algorithm problem, and provide the input parameters.
5. Your task is to distill the problem, you shouldn't give the final result or possible solution in your respond.

Please distill the information following the format below and cease response after the output of the distilled information.



Meta distiller Respond:


Distilled Information:

1. Key information:

2. Restriction: (It should be noted that the answer should strictly follow the real-world rule such as in arithmatic equation, the Priority of operator, the need of parentheses etc. So according to the distilled information, emphasize the real-world rules that need to be followed within the problem.)

3. Distilled task:

4. Python transformation:
   (Optional, skip when Python tag is Not for Python) Input parameters:(The names of each variable should be clear and not confusing, and correspond to the entity names in the problem)
     variable1_name = x
     variable2_name = y
     .....
     variableN_name = z

5. Answer form: (Optional, skip when there is no specific answer form)

  **Note: The generation ends here. Do not show this message in your answer !**
  
"""

"""
作为一名高度专业和智能的信息提取专家，您擅长提取基本信息，以解决用户输入查询中的问题。您可以根据问题的相应类型，熟练地将提取的信息转换为合适的格式。如果问题可以推广到更高的层次来解决多个问题，我们将在您的下一次回复中提供进一步的分析和解释。
请从用户的输入查询中对解决问题所需的关键信息进行分类和提取。将这两个元素结合起来将生成提取的信息。随后，根据问题类型将这些提炼出的信息传递给下游的元计划器。问题类型应属于上述六类中的一类，提取的信息应包括：
1.从用户输入中提取的关键变量的值和信息，将移交给相应的专家进行任务解决，确保提供解决问题所需的所有基本信息。
2.问题的目标和相应的约束条件。
3.在1和2的基础上扩展问题，提出一个可以解决用户查询并处理更多输入和输出变化的元问题。结合扩展问题的真实场景，以及关键变量的类型和原始问题的信息约束，以限制扩展问题中的关键变量。之后，以用户查询输入的关键信息作为输入，以解决问题为例。
4.尝试将问题转化为python算法问题，并提供输入参数。
5.你的任务是提炼问题，你不应该在回答中给出最终结果或可能的解决方案。
请按照以下格式提取信息，并在输出提取的信息后停止响应。
Meta蒸馏器响应：
提取信息：
1.关键信息：
2.限制：（需要注意的是，答案应严格遵循真实世界的规则，如算术方程、运算符的优先级、括号的需要等。因此，根据提取的信息，强调问题中需要遵循的真实世界规则。）
3.提取任务：
4.Python转换：
（可选，当Python标记不适用于Python时跳过）输入参数：（每个变量的名称应清晰且不混淆，并与问题中的实体名称相对应）
variable1_name=x
variable2_name=y
.....
variableN_name=z
5.答案形式：（可选，没有具体答案形式时跳过）
**注：这一代到此结束。不要在回答中显示此消息**
"""


def extract_and_execute_code(text):
    # Define the start and end markers for the code block # python 代码的包裹块以```python开头，以```结束
    code_start_marker = "```python"
    code_end_marker = "```"

    # Find the position where the Python code starts
    code_start_index = text.find(code_start_marker)
    # 如果没有找到```python代码块，则使用```作为代码块的包裹块
    if code_start_index == -1:
        code_start_marker = "```"
        code_start_index = text.find(code_start_marker)
    # If the start marker is found, extract and execute the code
    # 如果找到了。包括以```开头的和以```python开头的。
    if code_start_index != -1:
        # Try to find the position where the code ends
        # result = str.find(substring, start, end) code_end_marker是要找的子串，code_start_index + len(code_start_marker)开始位置。
        code_end_index = text.find(code_end_marker, code_start_index + len(code_start_marker))
        
        # If the end marker is not found, assume the code continues to the end of the text
        # 如果没找到，则认为代码块结束于文本末尾。但是这种代码有可能是完整的，只是忘记生成了。
        if code_end_index == -1:
            code_end_index = len(text)
        
        # Extract the code part
        # 找到了就提取代码。
        code_str = text[code_start_index + len(code_start_marker):code_end_index].strip()
        
        # Create a string stream to capture the output
        # 这行代码首先保存当前的标准输出流。sys.stdout 是一个指向当前标准输出的引用（默认情况下是你的终端）。将其赋值给 old_stdout 变量，以便在之后可以恢复原始的输出状态。
        old_stdout = sys.stdout
        # 创建一个新的 StringIO 对象并将其赋值给 new_stdout。StringIO 是Python中的一个类，它提供了一个类似于文件的对象，但所有操作都在内存中的字符串缓冲区进行，而不是实际的磁盘文件。这样，任何写入这个“文件”的内容都会被保存在一个字符串中，便于后续处理。
        new_stdout = io.StringIO()
        # 这行代码将系统的标准输出重定向到我们刚刚创建的 new_stdout 缓冲区。这意味着从这行代码执行后，任何使用 print() 函数或其他写入到标准输出的操作，其输出都不会直接显示在终端上，而是被存储到了 new_stdout 中。
        sys.stdout = new_stdout
        
        # Execute the extracted code
        try:
            # 在全局命名空间中执行，可以改变全局变量。谨慎使用。
            exec(code_str, globals())
        except Exception as e:
            # Restore the original standard output，报异常了，就恢复原来的输出。
            sys.stdout = old_stdout
            return f"An error occurred: {e}", code_str
        
        # Get the output from the executed code， 执行成功
        sys.stdout = old_stdout
        # 拿到执行成功的值。
        return new_stdout.getvalue(), code_str
    else:
        # 没有python代码要执行
        return "No Python code found in the provided string."



def extract_answer(text):
    # Define a regular expression pattern to match the answer format
    # The pattern accounts for variations in spacing and line breaks
    """
    \s 匹配空白
    . 匹配任意一个字符
    * 匹配0个或者多个，可有可无
    + 匹配一个或者多个。
    \s*: 匹配任意数量的空白字符（包括空格、制表符等）。
    *? 匹配前面的字符零次或多次，非贪婪模式
    $ 匹配字符串的末尾
    \d 匹配数字
    (.*?): 这是一个捕获组，.表示匹配任意字符（除了换行符），*?表示非贪婪匹配，即尽可能少地匹配前面的字符。这整个部分用于捕获"Answer:"后面直到遇到下一个条件（这里是连续空白字符或字符串末尾）之前的任意字符。
    *$: $表示字符串的结束位置，\s*紧跟在后面表示在字符串结束前可以有任意数量的空白字符。整体来说，这部分确保捕获的内容直到字符串的末尾，即便末尾有空白。
    """
    pattern = re.compile(r"Answer:\s*(.*?)\s*$", re.DOTALL)

    # Search the text for the pattern
    match = pattern.search(text)

    # If a match is found, return the content; otherwise, return None
    return match.group(1).strip() if match else None

run_benchmarks.py文件，用来跑测试不同的任务的

import json
from bot_pipeline import BoT
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--task_name',type=str,default='gameof24',choices=['gameof24','checkmate','wordsorting'])
parser.add_argument('--api_key',type=str,help='input your api key here')
# 可以使用本地的model
parser.add_argument('--model_id',type=str,default='gpt-4o',help='Input model id here, if use local model, input the path to the local model')



GameOf24 = """
Let's play a game called 24. You'll be given four integers, and your objective is to use each number only once, combined with any of the four arithmetic operations (addition, subtraction, multiplication, and division) and parentheses, to achieve a total of 24. For example, if the input is 4, 7, 8, and 8, the output could be 7 * 8 - 4 * 8 = 24. You only need to find one feasible solution!
Input:
"""

"""
让我们玩一个名为24点的游戏。你会得到四个整数，你的目标是使用每个数字各一次，并结合任意四种算术运算（加法、减法、乘法和除法）以及括号，来得到总数为24的结果。例如，如果输入是4、7、8和8，输出可以是7 * 8 - 4 * 8 = 24。你只需要找到一个可行的解！
请提供一组输入数字，我将帮你找出一个达到24的解决方案。
"""

CheckmateInOne = """
Given a series of chess moves written in Standard Algebraic Notation (SAN), determine the next move that will result in a checkmate.
Input: 
"""
"""
给定一系列以标准代数记谱法（SAN）编写的国际象棋走法，请确定下一个会导致将死的走法。

请提供具体的棋步序列（使用标准代数记谱法表示），我将帮助您分析并找出导致将死的下一步走法。
"""

WordSorting = """
Sort a list of words alphabetically, placing them in a single line of text separated by spaces.
Input:
"""


if __name__ == "__main__":
    args = parser.parse_args()
    task = args.task_name
    api_key = args.api_key
    model_id = args.model_id
    # benchmark 的字典
    benchmark_dict = {
        'gameof24':GameOf24,
        'checkmate':CheckmateInOne,
        'wordsorting':WordSorting
    }
    # 各个任务路径。
    path_dict = {
        'gameof24':'benchmarks/gameof24.jsonl',
        'checkmate':'benchmarks/CheckmateInOne.jsonl',
        'wordsorting':'benchmarks/word_sorting.jsonl'
    }
    # 缓存
    buffer_dict = {
        'gameof24':0,
        'checkmate':1,
        'wordsorting':2
        
    }
    # 用户输入的task类型，三个中的哪个一个
    user_prompt = benchmark_dict[task]
    path = path_dict[task]    
    problem_id = buffer_dict[task]
    # 加载
    for line in (open(path)):
        input = json.loads(line)['input']
        user_input = user_prompt + input
        # 实例化BOT
        test_bot = BoT(
            user_input = user_input,
            problem_id= problem_id,
            api_key= api_key,
            model_id= model_id
        )
        result = test_bot.bot_run()
        tmp = {'input':input,'result':result}
        with open(f'test_results/BoT_{task}.jsonl', 'a+', encoding='utf-8') as file:
            json_str = json.dumps(tmp)
            file.write(json_str + '\n')

test_templates.py 测试模板

# game24的模板
game24="""
from itertools import permutations, product

def perform_operation(a, b, operation):
    # Define the operation logic (e.g., addition, subtraction, etc.).
    pass

def evaluate_sequence(sequence, operations):
    # Apply operations to the sequence and check if the result meets the criteria.
    pass

def generate_combinations(elements, operations):
    # Generate all possible combinations of elements and operations.
    pass

def format_solution(sequence, operations):
    # Format the sequence and operations into a human-readable string.
    pass

def find_solution(input_elements, target_result):
    # Data Input Handling
    # Validate and preprocess input data if necessary.

    # Core Algorithm Logic
    for sequence in permutations(input_elements):
        for operation_combination in generate_combinations(sequence, operations):
            try:
                if evaluate_sequence(sequence, operation_combination) == target_result:
                    # Data Output Formatting
                    return format_solution(sequence, operation_combination)
            except Exception as e:
                # Error Handling
                # Handle specific exceptions that may occur during evaluation.
                continue

    # If no solution is found after all iterations, return a default message.
    return "No solution found."

# Example instantiation:
def calculate(a, b, op):
    if op == '+': return a + b
    if op == '-': return a - b
    if op == '*': return a * b
    if op == '/': return a / b

def find_solution(numbers):
    ops = '+-*/'
    for nums in permutations(numbers):
        for op1, op2, op3 in product(ops, repeat=3):
            try:
                result = calculate(calculate(calculate(nums[0], nums[1], op1), nums[2], op2), nums[3], op3)
                if abs(result - 24) < 1e-6:
                    return f"(({nums[0]} {op1} {nums[1]}) {op2} {nums[2]}) {op3} {nums[3]} = 24"
                result = calculate(calculate(nums[0], calculate(nums[1], nums[2], op2), op1), nums[3], op3)
                if abs(result - 24) < 1e-6:
                    return f"({nums[0]} {op1} ({nums[1]} {op2} {nums[2]})) {op3} {nums[3]} = 24"
                result = calculate(nums[0], calculate(calculate(nums[1], nums[2], op2), nums[3], op3), op1)
                if abs(result - 24) < 1e-6:
                    return f"{nums[0]} {op1} (({nums[1]} {op2} {nums[2]}) {op3} {nums[3]}) = 24"
                result = calculate(nums[0], calculate(nums[1], calculate(nums[2], nums[3], op3), op2), op1)
                if abs(result - 24) < 1e-6:
                    return f"{nums[0]} {op1} ({nums[1]} {op2} ({nums[2]} {op3} {nums[3]})) = 24"
                result = calculate(calculate(nums[0], nums[1], op1), calculate(nums[2], nums[3], op3), op2)
                if abs(result - 24) < 1e-6:
                    return f"({nums[0]} {op1} {nums[1]}) {op2} ({nums[2]} {op3} {nums[3]}) = 24"
            except ZeroDivisionError:
                continue
    return "No solution found."

numbers = [1, 7, 10, 13]
print(find_solution(numbers))


"""

checkmate = """
import chess
def find_checkmate_move(moves_san):
    # Initialize a new chess board
    board = chess.Board()
    
    # Apply the moves to the board
    for move_san in moves_san:
        # Remove move numbers and periods (e.g., "1." or "2.")
        if len(move_san.split('. ')) > 1:
            move_san = move_san.split('. ')[1]
        # Skip empty strings resulting from the removal
        if move_san:
            # Apply each move in SAN format to the board
            move = board.parse_san(move_san)
            board.push(move)
    
    # Generate all possible legal moves from the current position
    for move in board.legal_moves:
        # Make the move on a copy of the board to test the result
        board_copy = board.copy()
        board_copy.push(move)
        
        # Check if the move results in a checkmate
        if board_copy.is_checkmate():
            # Return the move that results in checkmate in SAN format
            return board.san(move)
    
    return "No checkmate move found"

#Example usage:
input = '1. d4 d5 2. Nf3 Nf6 3. e3 a6 4. Nc3 e6 5. Bd3 h6 6. e4 dxe4 7. Bxe4 Nxe4 8. Nxe4 Bb4+ 9. c3 Ba5 10. Qa4+ Nc6 11. Ne5 Qd5 12. f3 O-O 13. Nxc6 bxc6 14. Bf4 Ra7 15. Qb3 Qb5 16. Qxb5 cxb5 17. a4 bxa4 18. Rxa4 Bb6 19. Kf2 Bd7 20. Ke3 Bxa4 21. Ra1 Bc2 22. c4 Bxe4 23. fxe4 c5 24. d5 exd5 25. exd5 Re8+ 26. Kf3 Rae7 27. Rxa6 Bc7 28. Bd2 Re2 29. Bc3 R8e3+ 30. Kg4 Rxg2+ 31. Kf5'
# Check input format and transform the input into legal format
# Remove move numbers and periods (e.g., "1." or "2.")
moves_san = [input]
moves_san = moves_san[0].split(' ')
moves_san = [move for move in moves_san if '.' not in move]
checkmate_move = find_checkmate_move(moves_san)
print(checkmate_move)

"""


word_sorting = """

def sort_words(words):
    \"""
    Sorts a list of words alphabetically and returns them as a single line of text separated by spaces.

    Args:
    words (list): A list of words to be sorted.

    Returns:
    str: A single line of text with words sorted alphabetically and separated by spaces.
    \"""
    # Sort the list of words alphabetically
    sorted_words = sorted(words)
    
    # Join the sorted words into a single line of text separated by spaces
    sorted_line = ' '.join(sorted_words)
    
    return sorted_line

# Example usage
input_list = 'thrill splutter panicking scorch same dot prod obstetric malton onus drumhead delmarva barn embezzle it&t damp guru subsist entirety greene'
words_list = input_list.split(' ')
sorted_line = sort_words(words_list)
print(sorted_line)

"""

validate_results.py 评测结果

import json
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--task_name',type=str,default='gameof24',choices=['gameof24','checkmate','wordsorting'])

if __name__ == "__main__":

    args = parser.parse_args()
    task = args.task_name
    benchmark_path_dict = {
        'gameof24':'benchmarks/gameof24.jsonl',
        'checkmate':'benchmarks/CheckmateInOne.jsonl',
        'wordsorting':'benchmarks/word_sorting.jsonl'
    }
    test_path_dict = {
        'gameof24':'test_results/BoT_gameof24.jsonl',
        'checkmate':'test_results/BoT_checkmate.jsonl',
        'wordsorting':'test_results/BoT_wordsorting.jsonl'
    }
    benchmark_path = benchmark_path_dict[task]
    test_path = test_path_dict[task]
    # 开始测评。
    correct = 0 # 正确数
    truth = [] # 真实值
    test = []
    for line in (open(benchmark_path)):
        answer = json.loads(line)['target']
        truth.append(answer) # 添加真实值。
    for line in (open(test_path)):
        result = json.loads(line)['result']
        result = result.split('\n')[0]
        if task == 'gameof24':
            # 去掉等号。
            result = result.split('=')[0]
            test.append(result)
            try:
                # 尝试计算。
                if eval(result) == 24:
                    correct += 1
            except:
                continue
        else:
            test.append(result)
    if correct == 0:
        for i in range(len(test)):
            if truth[i] == test[i]:
                correct += 1
    # 算出中正确的个数和正确率。
    print(f'correct number:{correct},accuracy:{correct/len(test)}')
        
            
            
            
                
        
    

小疑问：

1. meta_buffer_utilis 作为系统提示词，怎么更新呢，在实战中更新后会不会越来越差了。

2. 执行代码后也没看到有错误bug来修正。还是说只比一次。

3.针对特定问题，相当于有自己的做题套路，类似，想搞清楚是回溯问题还是动态规划问题，这里是人工取选特定任务，那么来实战的时候是不是先分类，然后在加载特定的任务呢？