不平衡数据研究：分配权重 合并2个loader

 分配权重（基于实例分配，基于类分配）

import numpy as np
import torch
from torch.utils.data import DataLoader, WeightedRandomSampler, Dataset

# Mock dataset class
class MockDataset(Dataset):
    def __init__(self, data, targets):
        self.data = data
        self.targets = targets
        self.dr = targets  # mimicking the 'dr' attribute in the original dataset

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx], self.targets[idx]

# Mock get_sampling_probabilities function
def get_sampling_probabilities(class_count, mode='instance', ep=None, n_eps=None):
    if mode == 'instance':
        q = 0
    elif mode == 'class':
        q = 1
    elif mode == 'sqrt':
        q = 0.5
    elif mode == 'cbrt':
        q = 0.125
    elif mode == 'prog':
        assert ep is not None and n_eps is not None, 'progressive sampling requires to pass values for ep and n_eps'
        relative_freq_imbal = class_count ** 0 / (class_count ** 0).sum()
        relative_freq_bal = class_count ** 1 / (class_count ** 1).sum()
        sampling_probabilities_imbal = relative_freq_imbal ** (-1)
        sampling_probabilities_bal = relative_freq_bal ** (-1)
        return (1 - ep / (n_eps - 1)) * sampling_probabilities_imbal + (ep / (n_eps - 1)) * sampling_probabilities_bal
    else:
        sys.exit('not a valid mode')

    relative_freq = class_count ** q / (class_count ** q).sum()
    sampling_probabilities = relative_freq ** (-1)
    return sampling_probabilities

# Mock modify_loader function
def modify_loader(loader, mode, ep=None, n_eps=None):
    # 计算每个类别的样本数
    class_count = np.unique(loader.dataset.dr, return_counts=True)[1]
    print(f"class_count: {class_count}")

    # 获取每个类别的采样概率
    sampling_probs = get_sampling_probabilities(class_count, mode=mode, ep=ep, n_eps=n_eps)
    print(f"sampling_probs: {sampling_probs}")

    # 计算每个样本的权重
    sample_weights = sampling_probs[loader.dataset.dr]
    print(f"loader.dataset.dr:{loader.dataset.dr}")
    print(f"sample_weights: {sample_weights}")

    # 创建加权随机采样器
    mod_sampler = WeightedRandomSampler(weights=sample_weights, num_samples=len(sample_weights))
    # 创建新的数据加载器
    mod_loader = DataLoader(loader.dataset, batch_size=loader.batch_size, sampler=mod_sampler, num_workers=loader.num_workers)
    return mod_loader

# 模拟数据和标签
data = np.random.randn(100, 3, 32, 32)
targets = np.random.choice([0, 1, 2], size=100, p=[0.7, 0.2, 0.1])  # imbalanced classes

dataset = MockDataset(data, targets)
loader = DataLoader(dataset, batch_size=10, num_workers=1)

# 使用 'class' 模式进行均衡采样
modified_loader = modify_loader(loader, mode='class')

# 打印修改后的加载器中的一些样本
for i, (data, target) in enumerate(modified_loader):
    print(f"Batch {i}:")
    print(f"Data shape: {data.shape}, Targets: {target}")
    if i == 2:  # 仅打印前3个批次以简洁
        break

 合并2个loader

import numpy as np
import torch
from torch.utils.data import DataLoader, Dataset
import torch.nn.functional as F

# 定义一个简单的数据集类
class SimpleDataset(Dataset):
    def __init__(self, data, targets):
        self.data = data
        self.targets = targets

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx], self.targets[idx]

# 定义 ComboIter 类
class ComboIter(object):
    """An iterator."""
    def __init__(self, my_loader):
        self.my_loader = my_loader
        self.loader_iters = [iter(loader) for loader in self.my_loader.loaders]

    def __iter__(self):
        return self

    def __next__(self):
        # 获取每个 loader 的一个 batch
        batches = [loader_iter.__next__() for loader_iter in self.loader_iters]
        # 返回合并后的 batch
        return self.my_loader.combine_batch(batches)

    def __len__(self):
        return len(self.my_loader)

# 定义 ComboLoader 类
class ComboLoader(object):
    """This class wraps several pytorch DataLoader objects, allowing each time
    taking a batch from each of them and then combining these several batches
    into one. This class mimics the `for batch in loader:` interface of
    pytorch `DataLoader`.
    Args:
    loaders: a list or tuple of pytorch DataLoader objects
    """
    def __init__(self, loaders):
        self.loaders = loaders

    def __iter__(self):
        return ComboIter(self)

    def __len__(self):
        return min([len(loader) for loader in self.loaders])

    # 自定义合并批次的方法
    def combine_batch(self, batches):
        return batches

# 创建两个数据集和 DataLoader
data1 = torch.randn(100, 3, 32, 32)
targets1 = torch.randint(0, 2, (100,))
dataset1 = SimpleDataset(data1, targets1)
loader1 = DataLoader(dataset1, batch_size=10)

data2 = torch.randn(100, 3, 32, 32)
targets2 = torch.randint(0, 2, (100,))
dataset2 = SimpleDataset(data2, targets2)
loader2 = DataLoader(dataset2, batch_size=10)

# 使用 ComboLoader 合并两个 DataLoader
combo_loader = ComboLoader([loader1, loader2])

# 打印合并后的批次数据格式
print("\nCombined Loader batches:")
for i, batches in enumerate(combo_loader):
    print(f"Batch {i}:")
    for j, batch in enumerate(batches):
        print(f" Sub-Batch {j}: Data shape: {batch[0].shape}, Targets: {batch[1]}")
    if i == 2:  # 仅打印前3个批次以简洁
        break

 

import numpy as np
import torch
from torch.utils.data import DataLoader, Dataset

# 定义一个简单的数据集类
class SimpleDataset(Dataset):
    def __init__(self, data, targets):
        self.data = data
        self.targets = targets

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx], self.targets[idx]

# 定义 ComboIter 类
class ComboIter(object):
    """An iterator."""
    def __init__(self, my_loader):
        self.my_loader = my_loader
        self.loader_iters = [iter(loader) for loader in self.my_loader.loaders]

    def __iter__(self):
        return self

    def __next__(self):
        # 获取每个 loader 的一个 batch
        batches = [loader_iter.__next__() for loader_iter in self.loader_iters]
        # 返回合并后的 batch
        return self.my_loader.combine_batch(batches)

    def __len__(self):
        return len(self.my_loader)

# 定义 ComboLoader 类
class ComboLoader(object):
    """This class wraps several pytorch DataLoader objects, allowing each time
    taking a batch from each of them and then combining these several batches
    into one. This class mimics the `for batch in loader:` interface of
    pytorch `DataLoader`.
    Args:
    loaders: a list or tuple of pytorch DataLoader objects
    """
    def __init__(self, loaders):
        self.loaders = loaders

    def __iter__(self):
        return ComboIter(self)

    def __len__(self):
        return min([len(loader) for loader in self.loaders])

    # 自定义合并批次的方法
    def combine_batch(self, batches):
        combined_data = torch.cat([batch[0] for batch in batches], dim=0)
        combined_targets = torch.cat([batch[1] for batch in batches], dim=0)
        return combined_data, combined_targets

# 创建两个数据集和 DataLoader
data1 = torch.randn(100, 3, 32, 32)
targets1 = torch.randint(0, 2, (100,))
dataset1 = SimpleDataset(data1, targets1)
loader1 = DataLoader(dataset1, batch_size=10)

data2 = torch.randn(100, 3, 32, 32)
targets2 = torch.randint(0, 2, (100,))
dataset2 = SimpleDataset(data2, targets2)
loader2 = DataLoader(dataset2, batch_size=10)

# 打印每个加载器中的一些批次数据
print("Loader 1 batches:")
for i, (data, targets) in enumerate(loader1):
    print(f"Batch {i}:")
    print(f"Data shape: {data.shape}, Targets: {targets}")
    if i == 2:  # 仅打印前3个批次以简洁
        break

print("\nLoader 2 batches:")
for i, (data, targets) in enumerate(loader2):
    print(f"Batch {i}:")
    print(f"Data shape: {data.shape}, Targets: {targets}")
    if i == 2:  # 仅打印前3个批次以简洁
        break

# 使用 ComboLoader 合并两个 DataLoader
combo_loader = ComboLoader([loader1, loader2])

# 打印合并后的批次数据
print("\nCombined Loader batches:")
for i, (data, targets) in enumerate(combo_loader):
    print(f"Batch {i}:")
    print(f"Data shape: {data.shape}, Targets: {targets}")
    if i == 2:  # 仅打印前3个批次以简洁
        break