通过redfish协议实现服务器固件升级、从虚拟光驱启动自检盘并等待完成,最后截图保存

新开发的PCIE设备在做服务器适配时,有时需要服务器厂家更新BMC或BIOS固件。同时,我们也希望对PCIE设备做一些检测,最后收集一些信息存档。如果需要处理的服务器很多,通过BMC的界面进行人工操作就会比较麻烦。以下提供了一个脚本,供参考。

主要思路:

• 采用haneWIN NFS Server搭建一个NFS服务,目录为nfs,里面存放着boot.iso(设备检测镜像)
• 通过redfish协议登录BMC,获取PCIE设备信息,服务器信息,升级固件,重启服务器,挂iso,设置启动方式
• 截屏获取KVM的内容,通过图片相似度的方法判断ISO里的检测程序是否运行完成.

版本信息

属性	值
NFS服务器	haneWIN NFS Server
服务器型号	NF5270M6

代码

# -*- coding: utf-8 -*-

from queue import Queue
from skimage.metrics import structural_similarity
import traceback
import cv2
import codecs
import csv
import argparse
import shutil
import json
import time
import redfish
import sys
import os
import uuid
import threading
import warnings
warnings.filterwarnings("ignore")
import logging

parser = argparse.ArgumentParser()
parser.add_argument('-server_list', type=str,
                    required=True, help="server_list")
parser.add_argument('-nfs_server', type=str, required=True, help="nfs server")
parser.add_argument('-threads', type=int, required=True, help="nfs server")
parser.add_argument('-checkonly', type=int, required=True, help="check only")

args = parser.parse_args()


class TimeSpan:
    """统计代码段的耗时
    """    
    def __init__(self,logger,prefix=""):
        self.prefix = prefix
        self.logger=logger

    def __enter__(self):
        self.end = None
        self.start = time.time()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.end = time.time()
        interval = self.end - self.start
        unit = "sec"
        if interval > 60:
            unit = "min"
            interval = interval/60
        self.logger.info('%-64s:%.3f(%s)' % (self.prefix, interval, unit))

def isSimilarity(filename):
    """判断自检程序是否运行完成

    Args:
        filename ([string]): [截屏图片路径]

    Returns:
        [bool]: [是否完成]
    """    
    last_image = cv2.imread('target_image.jpg', cv2.IMREAD_GRAYSCALE)
    img = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)
    cell_h, cell_w = last_image.shape
    hoff = 280
    h, w = img.shape
    while hoff < 320:
        img2 = img[hoff:hoff+cell_h, 0:cell_w]
        ssim = structural_similarity(last_image, img2)
        if ssim > 0.5:
            # cv2.imwrite("{}_{}.jpg".format(hoff,int(ssim)),img2)
            return True
        hoff += 3
    return False

class RedFishProxy:
    def __init__(self,handle,retry_count=3):
        self.handle=handle
        self.retry_count=retry_count

    def post(self, path, args=None, body=None, headers=None):
        count=0
        while True:
            response=self.handle.post(path,args,body,headers)
            if response._status == 500 and count<self.retry_count:
                time.sleep(2)
                count+=1
                continue
            else:
                return response            

    def get(self, path, args=None, headers=None):
        count=0
        while True:
            response=self.handle.get(path,args,headers)
            if response._status == 500 and count<self.retry_count:
                time.sleep(2)
                count+=1
                continue
            else:
                return response   
            
    def delete(self,path, args=None, headers=None):
        count=0
        while True:
            response=self.handle.delete(path,args,headers)
            if response._status == 500 and count<self.retry_count:
                time.sleep(2)
                count+=1
                continue
            else:
                return response   

    def patch(self, path, args=None, body=None, headers=None):
        count=0
        while True:
            response=self.handle.patch(path,args,body,headers)
            if response._status == 500 and count<self.retry_count:
                time.sleep(2)
                count+=1
                continue
            else:
                return response        

class InspurVA1Query:
    def __init__(self,logger,index,bmc_host, username, password, nfs_server, try_count):
        self.logger=logger
        self.nfs_server = nfs_server
        self.username = username
        self.password = password
        self.try_count = try_count
        self.bmc_host = bmc_host
        self.seq = 0
        self.token=None
        self.index=index
        self.redfish_client=RedFishProxy(redfish.redfish_client(base_url=self.bmc_host, username=self.username, password=self.password))

    def Login(self):
        url = '/redfish/v1/SessionService/Sessions'
        req_body = {"UserName": self.username,
                    "Password": self.password, "SessionTimeOut": 300}
        req_headers = {"Content-Type": "application/json"}
        response = self.redfish_client.post(
            url, headers=req_headers, body=req_body)
        if response._status == 201:
            session = json.loads(response._read.decode())
            self.token = session["Oem"]['Public']['X-Auth-Token']
            self.Id = session["Id"]
            return True
        else:
            self.logger.error("Thermal:{}".format(response))
        return False

    def Logout(self):
        if self.token:
            url = '/redfish/v1/SessionService/Sessions/{}'.format(self.Id)
            req_headers = {"X-Auth-Token": self.token}
            response = self.redfish_client.delete(url, headers=req_headers)
            self.token=None
            if response._status == 200:
                return True
            self.logger.error("Logout:{}".format(response))
            return False
        else:
            return False
            
    def QueryMedia(self):
        url = '/redfish/v1/Managers/1/VirtualMedia/CD'
        req_headers = {"X-Auth-Token": self.token}
        response = self.redfish_client.get(url, headers=req_headers)
        self.logger.info("QueryMedia:{}".format(response))
        return True

    def IsMounted(self):
        """查询是否已经加载
        """        
        url = '/redfish/v1/Managers/1/VirtualMedia/CD'
        req_headers = {"X-Auth-Token": self.token}
        response = self.redfish_client.get(url, headers=req_headers)
        if response._status == 200:
            body = json.loads(response._read.decode())
            if 'Inserted' in body:
                return body['Inserted']
            return False
        self.logger.error("IsMounted:{}".format(response))
        return False

    def InsertMedia(self):
        url = '/redfish/v1/Managers/1/VirtualMedia/CD/Actions/VirtualMedia.InsertMedia'
        req_headers = {"X-Auth-Token": self.token}
        req_body = {"TransferProtocolType": 'NFS',
                    "Image": '{}/nfs/boot.iso'.format(self.nfs_server)}
        response = self.redfish_client.post(
            url, headers=req_headers, body=req_body)
        if response._status == 200:
            while True:
                if self.IsMounted():
                    return True
        self.logger.error("InsertMedia:{}".format(response))
        return False
   
    def EnableVirtualCDBoot(self,timeout=400):
        url = '/redfish/v1/Systems/1/Bios'
        req_headers = {"X-Auth-Token": self.token}
        response = self.redfish_client.get(url, headers=req_headers)
        if response._status == 200:
            ETag = response.getheader("ETag")  #获取 ETag
            body = json.loads(response._read.decode())
            # with open("{}_{}.json".format(self.bmc_host,uuid.uuid4().hex[:8]),"w") as f:
            #     f.write(json.dumps(body))

            bootOrderName=['UefiBootOrder1','UefiBootOrder2','UefiBootOrder3','UefiBootOrder4']

            cureOrder=""
            for name in bootOrderName:
                if name in body['Attributes'] and body['Attributes'][name].find('CD/DVD')>=0:
                    cureOrder=name
                    break

            if body['Attributes']['FixedBootOrderEn']=='Disabled' and cureOrder!="":
                self.logger.info("CDBoot Already Enable:{} {} {}".format(body['Attributes']['FixedBootOrderEn'],cureOrder,body['Attributes'][cureOrder]))
                return True
            
            url = '/redfish/v1/Systems/1/Bios/Settings'
            req_headers = {"X-Auth-Token": self.token,"If-Match":ETag}
            req_body = {}
            req_body['Attributes']={}
            req_body['Attributes']['FixedBootOrderEn']='Disabled'
            if cureOrder=="":
                req_body['Attributes']['UefiBootOrder4']="CD/DVD:UEFI: AMI Virtual CDROM0 1.00"  #最后一个启动项设置为CD启动,不影响正常的启动
            response = self.redfish_client.patch(url, headers=req_headers, body=req_body)
            if response._status in [200]:
                if not self.ComputerSystemReset("ForceRestart"):
                    return False
                beg=time.time()
                ii=0
                while True:
                    time.sleep(2)
                    cur=time.time()
                    if cur-beg>timeout:
                        self.logger.error("EnableVirtualCDBoot Timeout:{}".format(cur-beg))
                        return False
                    
                    # url = '/redfish/v1/Systems/1/Bios/Settings'
                    # req_headers = {"X-Auth-Token": self.token}
                    # response = self.redfish_client.get(url, headers=req_headers)
                    # if response._status == 200:
                    #     body = json.loads(response._read.decode())
                    #     print(body)
                        # if 'Attributes' not in body:
                        #     return True
                    url = '/redfish/v1/Systems/1/Bios'
                    req_headers = {"X-Auth-Token": self.token}
                    response = self.redfish_client.get(url, headers=req_headers)
                    if response._status == 200:
                        body = json.loads(response._read.decode())
                        cureOrder=""
                        for name in bootOrderName:
                            if name in body['Attributes'] and body['Attributes'][name].find('CD/DVD')>=0:
                                cureOrder=name
                                break                    
                        if body['Attributes']['FixedBootOrderEn']=='Disabled' and cureOrder!="":
                            self.logger.info("EnableVirtualCDBoot Finished:{} {} {}".format(cur-beg,cureOrder,body['Attributes'][cureOrder]))
                            return True                        
                    else:
                        self.logger.error("QueryBiosSetting2:{}".format(response))
            else:
                self.logger.error("BiosSetting:{}".format(response))
        else:
            self.logger.error("QueryBiosSetting1:{}".format(response))
            
        return False

    def SetBootOrder(self):
        url = '/redfish/v1/Systems/1'
        req_headers = {"X-Auth-Token": self.token}
        response = self.redfish_client.get(url, headers=req_headers)
        if response._status == 200:
            ETag = response.getheader("ETag")
            body = json.loads(response._read.decode())
            AssetTag = body['AssetTag']
            IndicatorLED = body['IndicatorLED']
            HostName = body['HostName']
            req_headers = {"X-Auth-Token": self.token, "If-Match": ETag}
            req_body = {}
            # req_body['AssetTag']=AssetTag
            # req_body['IndicatorLED']=IndicatorLED
            # req_body['HostName']=HostName
            req_body['Boot'] = {}
            req_body['Boot']['BootSourceOverrideTarget'] = 'Cd'
            req_body['Boot']['BootSourceOverrideEnabled'] = 'Once'
            req_body['Boot']['BootSourceOverrideMode'] = 'UEFI'
            response = self.redfish_client.patch(
                url, headers=req_headers, body=req_body)
            if response._status == 200:         
                for i in range(10):
                    url = '/redfish/v1/Systems/1'
                    req_headers = {"X-Auth-Token": self.token}
                    response = self.redfish_client.get(url, headers=req_headers)
                    if response._status == 200:
                        body = json.loads(response._read.decode())
                        BootSourceOverrideTarget=body['Boot']['BootSourceOverrideTarget']
                        if BootSourceOverrideTarget=="Cd":
                            return True
            self.logger.error("SetBootOrder:{}".format(response))
        else:
            self.logger.error("SetBootOrder:{}".format(response))

        return False


    def SetBootOrderLegacy(self):
        url = '/redfish/v1/Systems/1'
        req_headers = {"X-Auth-Token": self.token}
        response = self.redfish_client.get(url, headers=req_headers)
        if response._status == 200:
            ETag = response.getheader("ETag")
            body = json.loads(response._read.decode())
            AssetTag = body['AssetTag']
            IndicatorLED = body['IndicatorLED']
            HostName = body['HostName']
            req_headers = {"X-Auth-Token": self.token, "If-Match": ETag}
            req_body = {}
            # req_body['AssetTag']=AssetTag
            # req_body['IndicatorLED']=IndicatorLED
            # req_body['HostName']=HostName
            req_body['Boot'] = {}
            req_body['Boot']['BootSourceOverrideTarget'] = 'Hdd'
            req_body['Boot']['BootSourceOverrideEnabled'] = 'Continuous'
            req_body['Boot']['BootSourceOverrideMode'] = 'UEFI' #UEFI Legacy
            response = self.redfish_client.patch(
                url, headers=req_headers, body=req_body)
            if response._status == 200:  
                body = json.loads(response._read.decode())
                print(body)
                for i in range(10):
                    url = '/redfish/v1/Systems/1'
                    req_headers = {"X-Auth-Token": self.token}
                    response = self.redfish_client.get(url, headers=req_headers)
                    if response._status == 200:
                        body = json.loads(response._read.decode())
                        BootSourceOverrideMode=body['Boot']['BootSourceOverrideMode']
                        print("BootSourceOverrideMode:",BootSourceOverrideMode)
                        if BootSourceOverrideMode=="UEFI":
                            return True                            
            else:
                self.logger.error("SetBootOrder:{}".format(response))
        else:
            self.logger.error("SetBootOrder:{}".format(response))

        return False

    def EjectMedia(self):
        if self.IsMounted():
            url = '/redfish/v1/Managers/1/VirtualMedia/CD/Actions/VirtualMedia.EjectMedia'
            req_headers = {"X-Auth-Token": self.token}
            req_body = {"TransferProtocolType": 'NFS', "ImageName": 'boot.iso'}
            response = self.redfish_client.post(
                url, headers=req_headers, body=req_body)
            if response._status in [200, 500]:
                while True:
                    if not self.IsMounted():
                        return True
            self.logger.error("EjectMedia:{}".format(response))
            return False
        return True

    def ComputerSystemReset(self, ResetType):
        url = '/redfish/v1/Systems/1/Actions/ComputerSystem.Reset'
        req_headers = {"X-Auth-Token": self.token}
        req_body = {"ResetType": ResetType}
        response = self.redfish_client.post(url, headers=req_headers, body=req_body)
        if response._status == 200:
            body = json.loads(response._read.decode())
            return True
        self.logger.error("ComputerSystemReset:{}".format(response))
        return False

    def ChassisReset(self, ResetType,retry_count=3):
        for i in range(retry_count):
            url = '/redfish/v1/Chassis/1/Actions/Chassis.Reset'
            req_headers = {"X-Auth-Token": self.token}
            req_body = {"ResetType": ResetType}
            response = self.redfish_client.post(url, headers=req_headers, body=req_body)
            if response._status == 200:
                body = json.loads(response._read.decode())
                return True
            self.logger.error("ChassisReset[{}-{}]:{}".format(i,ResetType,response))
            time.sleep(2)
        return False

    def WaitFinished(self,timeout=5*60):
        beg = time.time()
        snap_count=0
        while True:
            time.sleep(3)  #频率不宜太快,否则容易导致系统出问题
            cur = time.time()
            if cur-beg > timeout:
                self.logger.error("WaitFinished,Timeout")
                return False
            url = '/redfish/v1/Managers/1/Actions/Oem/Public/KVM/Screenshot'
            req_headers = {"X-Auth-Token": self.token}
            snap_count+=1
            response = self.redfish_client.post(url, headers=req_headers)
            if response._status == 200:
                retry_count=10  #如果正在生成,不要再触发抓图,等待抓屏完成,否则会导致黑屏
                while retry_count>0:
                    cur = time.time()
                    if cur-beg > timeout:
                        self.logger.error("WaitFinished,Timeout")
                        return False
                    url = '/redfish/v1/Managers/1/Actions/Oem/Public/KVM/ScreenshotDownload'
                    req_headers = {"X-Auth-Token": self.token}
                    req_body = {"PictureAttributes": 'manual'}
                    response = self.redfish_client.post(url, headers=req_headers, body=req_body)                    
                    #if response._status != 200:
                    #    print("{}-{} {}:{}".format(snap_count,retry_count,response._status,json.loads(response._read.decode("utf-8","ignore"))))
                    if response._status == 404:  # The file is being generated
                        time.sleep(2)
                        retry_count-=1
                        continue
                    elif response._status == 500:  # There are no manual pictures at present
                        break
                    elif response._status == 401:  # Invalid Authentication
                        break
                    elif response._status == 200:
                        image_path = "{}-{}-{}-ing.jpg".format(self.index,self.bmc_host, self.seq)
                        with open(image_path, "wb") as f:
                            f.write(response.read)
                        if isSimilarity(image_path):
                            shutil.move(image_path, self.result_image)
                            return True
                        break
                    else:
                        self.logger.error("ScreenshotDownload:{}".format(response))
                        break
            else:
                self.logger.error("Screenshot:{}".format(response))

    def PCIEDeviceSummary(self,target_dev_count=3):
        """获取PCIE链路信息
        """

        url = '/redfish/v1/Systems/1'
        req_headers = {"X-Auth-Token": self.token}
        response = self.redfish_client.get(url, headers=req_headers)
        if response._status != 200:
            self.logger.error("Systems:{}".format(response))
            return False

        url = '/redfish/v1/Systems/1/Bios'
        req_headers = {"X-Auth-Token": self.token}
        response = self.redfish_client.get(url, headers=req_headers)
        if response._status != 200:
            self.logger.error("Bios:{}".format(response))
            return False
        
        url = '/redfish/v1/Chassis/1/PCIeDevices'
        req_headers = {"X-Auth-Token": self.token}
        response = self.redfish_client.get(url, headers=req_headers)
        if response._status == 200:
            body = json.loads(response._read.decode())
            count=body['Members@odata.count']
            #如果没开机,则跳过
            if count==0:
                self.logger.info("{} PowerStatus=Off".format(self.bmc_host))
                return True
            #如果发现掉卡,返回失败
            if count<target_dev_count:
                self.logger.error("VA1 Lost,Current:{}".format(count))
                return False
            for i in range(count):
                for _ in range(30):
                    url = '/redfish/v1/Chassis/1/PCIeDevices/{}'.format(i)
                    req_headers = {"X-Auth-Token": self.token}
                    response = self.redfish_client.get(url, headers=req_headers)
                    if response._status == 200:
                        body = json.loads(response._read.decode())
                        State=body['Status']['State']
                        Health=body['Status']['Health']
                        SlotNumber=body['Oem']['Public']['SlotNumber']
                        Manufacturer=body['Manufacturer']
                        if Manufacturer!="NVIDIA":
                            continue
                        url = '/redfish/v1/Chassis/1/PCIeDevices/{}/PCIeFunctions/1'.format(i)
                        req_headers = {"X-Auth-Token": self.token}
                        response = self.redfish_client.get(url, headers=req_headers)
                        if response._status == 200:
                            body = json.loads(response._read.decode())
                            LinkWidth=body['Oem']['Public']['LinkWidth']
                            LinkSpeed=body['Oem']['Public']['LinkSpeed']
                            self.logger.info("{} {} {} {} {} {} {}".format(i,Manufacturer,State,Health,SlotNumber,LinkWidth,LinkSpeed))
                            break
                        else:
                            time.sleep(1)
                    else:
                        time.sleep(1)
        else:
            self.logger.error("PCIeDevices1:{}".format(response))
            return False
        return True

    def run(self):
        """测试序列
        """
        # 测试次数
        for i in range(self.try_count):
            self.seq = i
            self.result_image = "{}-{}-{}-done.jpg".format(self.index,self.bmc_host, self.seq)

            #如果运行过,则跳过
            if os.path.exists(self.result_image):
                continue
            try:
                #统计总耗时
                with TimeSpan(self.logger,"{}-{} InspurVA1QueryE2E:".format(self.bmc_host, self.seq)):

                    #创建RedFish会话,获取token
                    with TimeSpan(self.logger,"*1.{}-{}-Login".format(self.bmc_host, self.seq)):
                        self.Logout()
                        if not self.Login():
                            continue
                    
                    #通过BMC查看卡数是否正常,如果掉卡直接返回失败
                    with TimeSpan(self.logger,"*2.{}-{}-PCIEDeviceSummary".format(self.bmc_host, self.seq)):
                        if not self.PCIEDeviceSummary():
                            self.logger.error("ERROR,{}".format(self.bmc_host))
                            continue                   
                    
                    #服务器下电,防止虚拟光驱被占用,导致后续加载失败
                    with TimeSpan(self.logger,"*3.{}-{}-PowerDown".format(self.bmc_host, self.seq)):
                        if not self.ChassisReset("ForceOff"):
                            continue
                    
                    #弹出虚拟光驱
                    with TimeSpan(self.logger,"*4.{}-{}-EjectMedia".format(self.bmc_host, self.seq)):
                        if not self.EjectMedia():
                            continue
                            
                    #设置虚拟光驱NFS挂载参数
                    with TimeSpan(self.logger,"*5.{}-{}-InsertMedia".format(self.bmc_host, self.seq)):
                        if not self.InsertMedia():
                            continue

                    #服务器上电
                    with TimeSpan(self.logger,"*6.{}-{}-PowerOn".format(self.bmc_host, self.seq)):
                        if not self.ChassisReset("On"):
                            continue

                    #确认并开启虚拟光驱启动功能
                    with TimeSpan(self.logger,"*7.{}-{}-EnableVirtualCDBoot".format(self.bmc_host, self.seq)):
                        if not self.EnableVirtualCDBoot():
                            continue

                    #设置下一次从虚拟光驱启动
                    with TimeSpan(self.logger,"*8.{}-{}-SetBootOrder".format(self.bmc_host, self.seq)):
                        if not self.SetBootOrder():
                            continue
                    
                    #重启,从光驱启动
                    with TimeSpan(self.logger,"*9.{}-{}-ForceRestart".format(self.bmc_host, self.seq)):               
                        if not self.ComputerSystemReset("ForceRestart"):
                            return False
                            
                    if i==0:
                        time.sleep(120)
                        continue
                        
                    #KVM循环截屏,ISO中的检测程序完后会打印"Please press Enter to activate this console"
                    #通过计算图像的SSIM,判断测图片中是否出现了以上打印
                    #5分钟如果没有检测到,则超时退出
                    with TimeSpan(self.logger,"*10.{}-{}-WaitFinished".format(self.bmc_host, self.seq)):
                        if not self.WaitFinished():
                            continue
                    
                    #弹出虚拟光驱
                    with TimeSpan(self.logger,"*11.{}-{}-EjectMedia".format(self.bmc_host, self.seq)):
                        if not self.EjectMedia():
                            continue
                    
                    #服务器下电再上电
                    with TimeSpan(self.logger,"*12.{}-{}-PowerCycle".format(self.bmc_host, self.seq)):
                        if not self.ChassisReset('PowerCycle'):
                            continue
                    
                    #注销RedFish会话
                    with TimeSpan(self.logger,"*13.{}-{}-Logout".format(self.bmc_host, self.seq)):
                        if not self.Logout():
                            continue
                            
                    return True
            except:
                self.logger.error("{}-{} Failed:".format(self.bmc_host, self.seq))
                traceback.print_exc(file=open('traceback_info.txt','a+'))

def FetchThread(checkonly,index,q):
    """自检任务线程
    """    

    logger=None

    while True:
        if q.empty():
            time.sleep(0.1)
            continue
        row = q.get()
        if row is None:
            break

        if logger is None:
            logger = logging.getLogger("FetchThread:{}".format(index))
            logger.setLevel(level = logging.INFO)
            handler = logging.FileHandler("nvidia_aic_check_inspur_{}.log".format(index))
            handler_ch=logging.StreamHandler()
            handler.setLevel(logging.INFO)
            handler_ch.setLevel(logging.INFO)
            formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
            handler.setFormatter(formatter)
            handler_ch.setFormatter(formatter)
            logger.addHandler(handler)
            logger.addHandler(handler_ch)
       
        index, bmc_addr, username, password, try_count = row
        logger.info(row)
        t = InspurVA1Query(logger,index,bmc_addr, username, password,
                           nfs_server, int(try_count))        
        if checkonly==1:
            if t.Login() and t.PCIEDeviceSummary():
                print("{} True".format(bmc_addr))
            else:
                print("{} False".format(bmc_addr))
        else:
            #t.run()
            t.Login()
            t.SetBootOrderLegacy()
            t.ComputerSystemReset("ForceRestart")
        
if __name__ == '__main__':

    nfs_server = args.nfs_server.strip()
    server_list = args.server_list.strip()
    threads = args.threads
    checkonly=args.checkonly
    
    request_queue = Queue(threads)
    tasks = []

    for i in range(threads):
        t = threading.Thread(target=FetchThread, args=(checkonly,i,request_queue, ))
        t.start()
        tasks.append(t)
    
    with codecs.open(server_list, "r", 'utf-8') as csvfile:
        csvreader = csv.reader(csvfile)
        next(csvreader)
        for row in csvreader:
            if row[0].startswith("#"):
                continue            
            request_queue.put(row)

    for i in range(threads):
        request_queue.put(None)

    for t in tasks:
        t.join()

'''
重启BMC
curl -X POST  https://192.168.1.100/redfish/v1/Systems/1/Actions/ComputerSystem.Reset -d '{"ResetType": "ForceRestart"}' -H "Content-Type: application/json" -k -u admin:admin
curl -X POST  https://192.168.1.100/redfish/v1/Managers/1/Actions/Manager.Reset -d '{"ResetType": "ForceRestart"}' -H "Content-Type: application/json" -k -u admin:admin
curl -X POST  https://192.168.1.100/redfish/v1/Managers/1/Actions/Manager.Reset -d '{"ResetType": "ForceRestart"}' -H "Content-Type: application/json" -k -u admin:admin
'''