即使模型在训练和验证数据中显示出良好的准确性,也可能导致错误的预测

问题描述

我使用了Kaggle中的皮肤癌分类比赛数据。有4个标签,整个数据不平衡。我在10倍交叉验证拆分上运行resnet 18模型,以训练数据,每次折叠大约2个纪元。该代码已附在下面。 基本上,该模型给出了98.2%的准确度,其中火车数据的损失值为0.07,在验证数据中给出了98.1%的准确度,损失为0.06。因此,这看起来还不错。 但是问题是... prediction.py(下面附有代码)。当我尝试进行预测时,模型始终将结果设为[0]。即使是火车图像数据。

我的代码有问题吗?

预期结果: 如果图像是输入,则输出应为0、1,2或3

model.py(进行培训的地方)

import pandas as pd 
import numpy as np

import torch
import torch.nn as nn

import os

import cv2

import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

import albumentations as A

from torch.utils.data import TensorDataset,DataLoader,Dataset
from torchvision import models
from collections import defaultdict
from torch.utils.data.sampler import RandomSampler
import torch.optim as optim
from torch.optim import lr_scheduler
from sklearn import model_selection
from tqdm import tqdm
import gc


# generate data from csv file
class Build_dataset(Dataset):
    def __init__(self,csv,split,mode,transform=None):
        self.csv = csv.reset_index(drop=True)
        self.split = split
        self.mode = mode
        self.transform = transform

    def __len__(self):
        return self.csv.shape[0]

    def __getitem__(self,index):
        row = self.csv.iloc[index]

        image = cv2.imread(row.filepath)
        image = cv2.cvtColor(image,cv2.COLOR_RGB2BGR)

        if self.transform is not None:
            res = self.transform(image=image)
            image = res['image'].astype(np.float32)
        else:
            image = image.astype(np.float32)

        image = image.transpose(2,1)
        data = torch.tensor(image).float()

        if self.mode == 'test':
            return data
        else:
            return data,torch.tensor(self.csv.iloc[index].target).long()

# training data           
def train_epoch(model,loader,optimizer,loss_fn,device,scheduler,n_examples):

    model = model.train()

losses = []
correct_predictions = 0

for inputs,labels in tqdm(loader):
    inputs = inputs.to(device)
    labels = labels.to(device)

    outputs = model(inputs)

    _,preds = torch.max(outputs,dim=1)
    loss = loss_fn(outputs,labels)
    
    
    correct_predictions += torch.sum(preds == labels)
    losses.append(loss.item())
    
    

    loss.backward()
    optimizer.step()
    optimizer.zero_grad()
# here you delete inputs and labels and then use gc.collect
    del inputs,labels
    gc.collect()


return correct_predictions.double() / n_examples,np.mean(losses)

# validation data 
def val_epoch(model,n_examples):

    model = model.eval()

    losses = []
    correct_predictions = 0

    with torch.no_grad():
        for inputs,labels in tqdm(loader):
            inputs = inputs.to(device)
            labels = labels.to(device)
            outputs = model(inputs)
            _,dim=1)
            loss = loss_fn(outputs,labels)
            correct_predictions += torch.sum(preds == labels)
            losses.append(loss.item())
            # here you delete inputs and labels and then use gc.collect
            del inputs,labels
            gc.collect()
        

    return correct_predictions.double() / n_examples,np.mean(losses)

        


 def train(fold,model,num_epochs):

    df_train = df[df.kfold != fold].reset_index(drop=True)
    df_valid = df[df.kfold == fold].reset_index(drop=True)
    # generate data
    dataset_train = Build_dataset(df_train,'train',transform=transforms_train)
    dataset_valid = Build_dataset(df_valid,'val',transform=transforms_val)

    #load data 
    train_loader = DataLoader(dataset_train,batch_size = 64,sampler=RandomSampler(dataset_train),num_workers=4)
    valid_loader = DataLoader(dataset_valid,batch_size = 32,shuffle = True,num_workers= 4 )

    dataset_train_size = len(dataset_train)

    dataset_valid_size = len(dataset_valid)

    optimizer = optim.Adam(model.parameters(),lr = 1e-4)

    model = model.to(device)

    scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,patience = 3,threshold = 0.001,mode = 
'max')

    loss_fn = nn.CrossEntropyLoss().to(device)

    history = defaultdict(list)

    best_accuracy = 0.0

    for epoch in range(num_epochs):
        print(f'Epoch {epoch+1} / {num_epochs}')
        print ('-'*30)
    
        train_acc,train_loss = train_epoch(model,train_loader,dataset_train_size)
        print(f'Train loss {train_loss} accuracy {train_acc}')
        valid_acc,valid_loss = val_epoch(model,valid_loader,dataset_valid_size)
        print(f'Val   loss {valid_loss} accuracy {valid_acc}')
        print()
    
        history['train_acc'].append(train_acc)
        history['train_loss'].append(train_loss)
        history['val_acc'].append(valid_acc)
        history['val_loss'].append(valid_loss)
    
        if valid_acc > best_accuracy:
            print('saving model')
            torch.save(model.state_dict(),f'best_model_{fold}.bin')
            best_accuracy = valid_acc
    
    print(f'Best Accuracy: {best_accuracy}')

    model.load_state_dict(torch.load(f'best_model_{fold}.bin'))

    return model,history



 if __name__ == '__main__':
    #competition data -2020
    data_dir = "../input/jpeg-melanoma-384x384"
    #competition data - 2019
    data_dir2 = "../input/jpeg-isic2019-384x384"
    # device
    device = torch.device("cuda")

    # augmenting images


    image_size = 384
    transforms_train = A.Compose([
        A.Transpose(p=0.5),A.VerticalFlip(p=0.5),A.HorizontalFlip(p=0.5),A.RandomBrightness(limit=0.2,p=0.75),A.RandomContrast(limit=0.2,A.OneOf([
            A.MedianBlur(blur_limit=5),A.GaussianBlur(blur_limit=5),A.GaussNoise(var_limit=(5.0,30.0)),],p=0.7),A.OneOf([
            A.OpticalDistortion(distort_limit=1.0),A.GridDistortion(num_steps=5,distort_limit=1.),A.ElasticTransform(alpha=3),A.CLAHE(clip_limit=4.0,A.HueSaturationValue(hue_shift_limit=10,sat_shift_limit=20,val_shift_limit=10,p=0.5),A.ShiftScaleRotate(shift_limit=0.1,scale_limit=0.1,rotate_limit=15,border_mode=0,p=0.85),A.Resize(image_size,image_size),A.Cutout(max_h_size=int(image_size * 0.375),max_w_size=int(image_size * 0.375),num_holes=1,A.Normalize()
    ])

    transforms_val = A.Compose([
        A.Resize(image_size,A.Normalize()
    ])
    # create data
    df_train = pd.read_csv(os.path.join(data_dir,"train.csv"))  #/kaggle/input/siim-isic-melanoma-classification/train.csv
    df_train.head()

    df_train['is_ext'] = 0
    df_train['filepath'] = df_train['image_name'].apply(lambda x: os.path.join(data_dir,f'{x}.jpg'))

    # dataset from 2020 data
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('seborrheic keratosis','BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lichenoid keratosis','BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('solar lentigo','BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lentigo NOS','BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('cafe-au-lait macule','unknown'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('atypical melanocytic proliferation','unknown'))

        
    # shuffle data
    df = df_train.sample(frac=1).reset_index(drop=True)

    # creating 8 different target values
    new_target = {d: idx for idx,d in enumerate(sorted(df.diagnosis.unique()))}
    df['target'] = df['diagnosis'].map(new_target)
    mel_idx = new_target['melanoma']

    # creating 10 fold cross validation data
    df = df_train.sample(frac=1).reset_index(drop=True)
    df['kfold'] = -1
    y = df_train.target.values
    kf = model_selection.StratifiedKFold(n_splits=10,shuffle=True)
    idx = kf.get_n_splits(X=df,y=y)
    print(idx)
    for fold,(x,y) in enumerate(kf.split(X=df,y=y)):
        df.loc[y,'kfold'] = fold

    df = df[['filepath','diagnosis','target','is_ext','kfold']]

    class_names = list(df['diagnosis'].unique())


    # create model

    def create_model(n_classes):
        model = models.resnet18(pretrained=True)

        n_features = model.fc.in_features
        model.fc = nn.Linear(n_features,n_classes)
        return model.to(device)
    
    base_model = create_model(len(class_names)) # model ready
    
    
    
    # run the model
    for i in range(10):
        #train
        base_model,history = train(i,base_model,num_epochs = 2) # train data

prediction.py

from torchvision import models
import torch 
import torch.nn as nn
import albumentations as A
import cv2
import os 
import numpy as np

device = torch.device("cuda")
MODEL = None
MODEL_PATH = "../input/prediction/best_model_4.bin"


def create_model(n_classes):
    model = models.resnet18(pretrained=True)

    n_features = model.fc.in_features
    model.fc = nn.Linear(n_features,n_classes)
    return model.to(device)
# generate the data to tensor with transform application

# converting the image to tensor by using the transforms function

class get_image:
    def __init__(self,image_path,targets,transform = None):
        self.image_path = image_path
        self.targets = targets
        self.transform = transform

    def __len__(self):
        return len(self.image_path)
    def __getitem__(self,item):
        targets = self.targets[item]
        resize = 384
        image = cv2.imread(self.image_path[item])
        image = cv2.resize(image,(resize,resize))
        image = cv2.cvtColor(image,cv2.COLOR_RGB2BGR)
    
        if self.transform is not None:
            res = self.transform(image = image)
            image = res['image'].astype(np.float32)
        image = image.transpose(2,1)
        data = torch.tensor(image).float()
        targets = torch.tensor(targets)
    
        return data,targets
    
# load the data by using torch data
# predict values 

# predict function
def predict(image_path,model_path):
    image_size = 384

    transforms_val = A.Compose([
        A.Resize(image_size,A.Normalize()
    ])

    test_images = [image_path]
    test_targets = [0]

    test_data = get_image(
        image_path = test_images,targets = test_targets,transform=transforms_val)
    # loading the data
    test_dataloader = torch.utils.data.DataLoader(test_data,batch_size=1,shuffle = False,num_workers=0)
    model = create_model(n_classes = 4)
    model.load_state_dict(torch.load(model_path))
    model.to(device)
    model.eval()
    prediction = []

    with torch.no_grad():
        for test_data,test_target in test_dataloader:
            test_data = test_data.to(device)
            test_target = test_target.to(device)
        
            outputs = model(test_data)
            _,preds = torch.max(outputs.cpu(),1)
        
            #prediction.extend(preds)
        
            prediction = np.vstack((preds)).ravel()
        
            return prediction
        
def upload_predict():
    image_file = "../input/whatever/ISIC_0075663.jpg"

    if image_file:
        pred = predict(image_file,MODEL,MODEL_PATH)
        print(pred)
    
    return pred

标签及其计数在此处给出

3    27126
2     5193
1      584
0      223

这里0被认为是恶性癌症,其他标签属于不同类型。

以下是数据链接:https://www.kaggle.com/cdeotte/jpeg-melanoma-384x384

解决方法

我认为您可能会回答您的问题!你说:

有4个标签,整个数据不平衡

假设标签0没有癌症,而1,2,3是患有不同类型皮肤癌的病例。如果您说预测类是不平衡的,那么我猜整个样本的98%为0,因此您的算法只是将每种情况都预测为0,这样98%的时间它就会正确。当您的算法进入测试集时,它将简单地预测一切都为0。

因此问题不出在您的代码上。您必须通过上采样少数类,下采样多数类,为数据分配权重/偏差或使用某种模型集合来平衡数据集,请参见https://elitedatascience.com/imbalanced-classes。查看信用卡欺诈检测教程,例如https://towardsdatascience.com/credit-card-fraud-detection-1b3b3b44109b

相关问答

Selenium Web驱动程序和Java。元素在(x,y)点处不可单击。其...
Python-如何使用点“。” 访问字典成员?
Java 字符串是不可变的。到底是什么意思?
Java中的“ final”关键字如何工作?(我仍然可以修改对象。...
“loop:”在Java代码中。这是什么,为什么要编译?
java.lang.ClassNotFoundException:sun.jdbc.odbc.JdbcOdbc...