可视化训练

可视化

1.TensorBoard

visualizer=dict(type='Visualizer', vis_backends=[dict(type='TensorboardVisBackend')]),

wandb

import wandb
import re
import time
from datetime import datetime
import argparse

PathLog = './out_yolox.txt'  # log日志文件路径
WandbConfig = dict(
        learning_rate=0.0025,
        momentum=0.2,
        architecture="yolox",
        dataset_id="coco_xu",
        infra="AWS",
    )
max_time = 10800  # 3h

def WandbTrain():
    wandb_train_parser = argparse.ArgumentParser(description='WandbTrain')
    wandb_train_parser.add_argument(
        'max_epoch',
        default=300,
        type=int, )
    wandb_train_parser.add_argument(
        'interval',
        default=10,
        type=int, )
    wandb_train_parser.add_argument(
        '--project',
        default=r'yolox', )
    wandb_train_parser.add_argument(
        '--name',
        default='train_1', )
    wandb_train_parser.add_argument(
        '--notes',
        default='tweak baseline', )
    wandb_train_parser.add_argument(
        '--tags',
        nargs='+',
        default=["baseline", "paper1"], )
    wandb_train_parser.add_argument(
        '--wandbconfig',
        default=WandbConfig,
        type=dict,)
    WandbArgs = wandb_train_parser.parse_args()
    return WandbArgs


def extend_list(input_list, length):
    last_element = input_list[-1] if input_list else 0
    while len(input_list) < length:
        input_list.append(last_element)
    return input_list


WandbArgs = WandbTrain()
wandb.init(
  project=WandbArgs.project,
  name=WandbArgs.name,
  notes=WandbArgs.notes,
  tags=WandbArgs.tags,
  config=WandbArgs.wandbconfig,
)

counts = 0
start_time = time.time()
while True:
    LossList, EpochList, EtaList, MapList, RecallList = [], [], [], [], []
    with open(PathLog)as log:
        for i, line in enumerate(log):

            match_loss = re.search(r'loss: (\d+\.\d+)', line)
            match_epoch = re.search(r'\[(\d+)\]', line)
            match_eta = re.search(r'eta: (\d+:\d{2}:\d{2})', line)
            match_map = re.search(r'Average Precision  \(AP\) @\[ IoU=0.50:0.95 \| area=   all \| maxDets=100 \] = (\d+\.\d+)', line)
            match_recall = re.search(r'Average Recall     \(AR\) @\[ IoU=0.50:0.95 \| area=   all \| maxDets=100 \] = (\d+\.\d+)', line)
            # match = re.search(r'=\s+(\d\.\d+)$', s)
            if match_loss and match_epoch and match_eta:
                loss = float(match_loss.group(1))  # match.group(1) 会获取第一对括号的匹配内容
                epoch = int(match_epoch.group(1))
                time_str = match_eta.group(1)
                time_obj = datetime.strptime(time_str, '%H:%M:%S')
                eta = time_obj.hour * 60 + time_obj.minute  # 计算分钟数
                LossList.append(loss)
                EpochList.append(epoch)
                EtaList.append(eta)

            if match_map:
                for i in range(WandbArgs.interval-1):
                    MapList.append(MapList[-1] if len(MapList) > 0 else 0)
                map_ = float(match_map.group(1))
                MapList.append(map_)
            if match_recall:
                for i in range(WandbArgs.interval-1):
                    RecallList.append(RecallList[-1] if len(RecallList) > 0 else 0.5)
                recall = float(match_recall.group(1))
                RecallList.append(recall)

        judge = len(RecallList)
        MapList = extend_list(MapList, len(EpochList))
        RecallList = extend_list(RecallList, len(EpochList))
        
        for i in range(len(EpochList)):
            if i <= counts:
                continue
            else:
                wandb.log({'loss': LossList[i],
                           'epoch': EpochList[i],
                           'eta': EtaList[i],
                           'map': MapList[i],
                           'recall': RecallList[i],
                           })
                counts = i
    end_time = time.time()
    run_time = end_time - start_time
    if judge >= WandbArgs.max_epoch or run_time > max_time:
        break
    time.sleep(30.0)