问题描述
我是 Tensorboard 的新手。
我不记得要求提供 hp_metric
图表,但它在这里。
它是什么以及如何摆脱它?
使用 Pytorch Lightning 重现完整代码(不是我认为任何人都应该重现这个来回答):
请注意取消引用 TensorBoard 的唯一行是
self.logger.experiment.add_scalars("losses",{"train_loss": loss},global_step=self.current_epoch)
import torch
from torch import nn
import torch.nn.functional as F
from typing import List,Optional
from pytorch_lightning.core.lightning import LightningModule
from Testing.Research.toy_datasets.ClustersDataset import ClustersDataset
from torch.utils.data import DataLoader
from Testing.Research.config.ConfigProvider import ConfigProvider
from pytorch_lightning import Trainer,seed_everything
from torch import optim
import os
from pytorch_lightning.loggers import TensorBoardLogger
class VAEFC(LightningModule):
# see https://towardsdatascience.com/understanding-variational-autoencoders-vaes-f70510919f73
# for possible upgrades,see https://arxiv.org/pdf/1602.02282.pdf
# https://stats.stackexchange.com/questions/332179/how-to-weight-kld-loss-vs-reconstruction-loss-in-variational-auto-encoder
def __init__(self,encoder_layer_sizes: List,decoder_layer_sizes: List,config):
super(VAEFC,self).__init__()
self._config = config
self.logger: Optional[TensorBoardLogger] = None
assert len(encoder_layer_sizes) >= 3,"must have at least 3 layers (2 hidden)"
# encoder layers
self._encoder_layers = nn.ModuleList()
for i in range(1,len(encoder_layer_sizes) - 1):
enc_layer = nn.Linear(encoder_layer_sizes[i - 1],encoder_layer_sizes[i])
self._encoder_layers.append(enc_layer)
# predict mean and covariance vectors
self._mean_layer = nn.Linear(encoder_layer_sizes[
len(encoder_layer_sizes) - 2],encoder_layer_sizes[len(encoder_layer_sizes) - 1])
self._logvar_layer = nn.Linear(encoder_layer_sizes[
len(encoder_layer_sizes) - 2],encoder_layer_sizes[len(encoder_layer_sizes) - 1])
# decoder layers
self._decoder_layers = nn.ModuleList()
for i in range(1,len(decoder_layer_sizes)):
dec_layer = nn.Linear(decoder_layer_sizes[i - 1],decoder_layer_sizes[i])
self._decoder_layers.append(dec_layer)
self._recon_function = nn.MSELoss(reduction='mean')
def _encode(self,x):
for i in range(len(self._encoder_layers)):
layer = self._encoder_layers[i]
x = F.relu(layer(x))
mean_output = self._mean_layer(x)
logvar_output = self._logvar_layer(x)
return mean_output,logvar_output
def _reparametrize(self,mu,logvar):
if not self.training:
return mu
std = logvar.mul(0.5).exp_()
if std.is_cuda:
eps = torch.cuda.FloatTensor(std.size()).normal_()
else:
eps = torch.FloatTensor(std.size()).normal_()
reparameterized = eps.mul(std).add_(mu)
return reparameterized
def _decode(self,z):
for i in range(len(self._decoder_layers) - 1):
layer = self._decoder_layers[i]
z = F.relu((layer(z)))
decoded = self._decoder_layers[len(self._decoder_layers) - 1](z)
# decoded = F.sigmoid(self._decoder_layers[len(self._decoder_layers)-1](z))
return decoded
def _loss_function(self,recon_x,x,logvar,reconstruction_function):
"""
recon_x: generating images
x: origin images
mu: latent mean
logvar: latent log variance
"""
binary_cross_entropy = reconstruction_function(recon_x,x) # mse loss Todo see if mse or cross entropy
# loss = 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
kld_element = mu.pow(2).add_(logvar.exp()).mul_(-1).add_(1).add_(logvar)
kld = torch.sum(kld_element).mul_(-0.5)
# KL divergence Kullback–Leibler divergence,regularization term for VAE
# It is a measure of how different two probability distributions are different from each other.
# We are trying to force the distributions closer while keeping the reconstruction loss low.
# see https://towardsdatascience.com/understanding-variational-autoencoders-vaes-f70510919f73
# read on weighting the regularization term here:
# https://stats.stackexchange.com/questions/332179/how-to-weight-kld-loss-vs-reconstruction-loss-in-variational
# -auto-encoder
return binary_cross_entropy + kld * self._config.regularization_factor
def training_step(self,batch,batch_index):
orig_batch,noisy_batch,_ = batch
noisy_batch = noisy_batch.view(noisy_batch.size(0),-1)
recon_batch,logvar = self.forward(noisy_batch)
loss = self._loss_function(
recon_batch,orig_batch,reconstruction_function=self._recon_function
)
# self.logger.experiment.add_scalars("losses",{"train_loss": loss})
self.logger.experiment.add_scalars("losses",global_step=self.current_epoch)
# self.logger.experiment.add_scalar("train_loss",loss,self.current_epoch)
self.logger.experiment.flush()
return loss
def train_DataLoader(self):
default_dataset,train_dataset,test_dataset = ClustersDataset.clusters_dataset_by_config()
train_DataLoader = DataLoader(train_dataset,batch_size=self._config.batch_size,shuffle=True)
return train_DataLoader
def test_DataLoader(self):
default_dataset,test_dataset = ClustersDataset.clusters_dataset_by_config()
test_DataLoader = DataLoader(test_dataset,shuffle=True)
return test_DataLoader
def configure_optimizers(self):
optimizer = optim.Adam(model.parameters(),lr=self._config.learning_rate)
return optimizer
def forward(self,x):
mu,logvar = self._encode(x)
z = self._reparametrize(mu,logvar)
decoded = self._decode(z)
return decoded,logvar
if __name__ == "__main__":
config = ConfigProvider.get_config()
seed_everything(config.random_seed)
latent_dim = config.latent_dim
enc_layer_sizes = config.enc_layer_sizes + [latent_dim]
dec_layer_sizes = [latent_dim] + config.dec_layer_sizes
model = VAEFC(config=config,encoder_layer_sizes=enc_layer_sizes,decoder_layer_sizes=dec_layer_sizes)
logger = TensorBoardLogger(save_dir='tb_logs',name='VAEFC')
logger.hparams = config # Todo only put here relevant stuff
# trainer = Trainer(gpus=1)
trainer = Trainer(deterministic=config.is_deterministic,#auto_lr_find=config.auto_lr_find,#log_gpu_memory='all',# min_epochs=99999,max_epochs=config.num_epochs,default_root_dir=os.getcwd(),logger=logger
)
# trainer.tune(model)
trainer.fit(model)
print("done training vae with lightning")
ClustersDataset.py
from torch.utils.data import Dataset
import matplotlib.pyplot as plt
import torch
import numpy as np
from Testing.Research.config.ConfigProvider import ConfigProvider
class ClustersDataset(Dataset):
__default_dataset = None
__default_dataset_train = None
__default_dataset_test = None
def __init__(self,cluster_size: int,noise_factor: float = 0,transform=None,n_clusters=2,centers_radius=4.0):
super(ClustersDataset,self).__init__()
self._cluster_size = cluster_size
self._noise_factor = noise_factor
self._n_clusters = n_clusters
self._centers_radius = centers_radius
# self._transform = transform
self._size = self._cluster_size * self._n_clusters
self._create_data_clusters()
self._combine_clusters_to_array()
self._normalize_data()
self._add_noise()
# self._plot()
pass
@staticmethod
def clusters_dataset_by_config():
if ClustersDataset.__default_dataset is not None:
return \
ClustersDataset.__default_dataset,\
ClustersDataset.__default_dataset_train,\
ClustersDataset.__default_dataset_test
config = ConfigProvider.get_config()
default_dataset = ClustersDataset(
cluster_size=config.cluster_size,noise_factor=config.noise_factor,n_clusters=config.n_clusters,centers_radius=config.centers_radius
)
train_size = int(config.train_size * len(default_dataset))
test_size = len(default_dataset) - train_size
train_dataset,test_dataset = torch.utils.data.random_split(default_dataset,[train_size,test_size])
ClustersDataset.__default_dataset = default_dataset
ClustersDataset.__default_dataset_train = train_dataset
ClustersDataset.__default_dataset_test = test_dataset
return default_dataset,test_dataset
def _create_data_clusters(self):
self._clusters = [torch.zeros((self._cluster_size,2)) for _ in range(self._n_clusters)]
centers_radius = self._centers_radius
for i,c in enumerate(self._clusters):
r,y = 3.0,centers_radius * np.cos(i * np.pi * 2 / self._n_clusters),centers_radius * np.sin(
i * np.pi * 2 / self._n_clusters)
cluster_length = 1.1
cluster_start = i * 2 * np.pi / self._n_clusters
cluster_end = cluster_length * (i + 1) * 2 * np.pi / self._n_clusters
cluster_inds = torch.linspace(start=cluster_start,end=cluster_end,steps=self._cluster_size,dtype=torch.float)
c[:,0] = r * torch.sin(cluster_inds) + y
c[:,1] = r * torch.cos(cluster_inds) + x
def _plot(self):
plt.figure()
plt.scatter(self._noisy_values[:,0],self._noisy_values[:,1],s=1,color='b',label="noisy_values")
plt.scatter(self._values[:,self._values[:,color='r',label="values")
plt.legend(loc="upper left")
plt.show()
def _combine_clusters_to_array(self):
size = self._size
self._values = torch.zeros(size,2)
self._labels = torch.zeros(size,dtype=torch.long)
for i,c in enumerate(self._clusters):
self._values[i * self._cluster_size: (i + 1) * self._cluster_size,:] = self._clusters[i]
self._labels[i * self._cluster_size: (i + 1) * self._cluster_size] = i
def _add_noise(self):
size = self._size
mean = torch.zeros(size,2)
std = torch.ones(size,2)
noise = torch.normal(mean,std)
self._noisy_values = torch.zeros(size,2)
self._noisy_values[:] = self._values
self._noisy_values = self._noisy_values + noise * self._noise_factor
def _normalize_data(self):
values_min,values_max = torch.min(self._values),torch.max(self._values)
self._values = (self._values - values_min) / (values_max - values_min)
self._values = self._values * 2 - 1
def __len__(self):
return self._size # number of samples in the dataset
def __getitem__(self,index):
item = self._values[index,:]
noisy_item = self._noisy_values[index,:]
# if self._transform is not None:
# noisy_item = self._transform(item)
return item,noisy_item,self._labels[index]
@property
def values(self):
return self._values
@property
def noisy_values(self):
return self._noisy_values
配置值(ConfigProvider 只是将这些作为对象返回)
num_epochs: 15
batch_size: 128
learning_rate: 0.0001
auto_lr_find: False
noise_factor: 0.1
regularization_factor: 0.0
cluster_size: 5000
n_clusters: 5
centers_radius: 4.0
train_size: 0.8
latent_dim: 8
enc_layer_sizes: [2,200,200]
dec_layer_sizes: [200,2]
retrain_vae: False
random_seed: 11
is_deterministic: True
解决方法
这是pytorch Lightning中tensorboard的默认设置。您可以将 default_hp_metric
设置为 false 以消除此指标。
TensorBoardLogger(save_dir='tb_logs',name='VAEFC',default_hp_metric=False)
hp_metric
可帮助您跟踪不同超参数的模型性能。您可以在 tensorboard 中的 hparams
查看。
hp_metric
(超参数指标)用于帮助您调整超参数。
您可以将此指标设置为您喜欢的任何值,如 pytorch official docs 中所述。
然后,您可以查看您的超参数,并根据您选择的任何指标查看哪个结果最好。
或者,如果您不想要它,您可以按照@joe32140 的回答中的建议禁用它:
您可以将 default_hp_metric 设置为 false 以消除此指标。
TensorBoardLogger(save_dir='tb_logs',default_hp_metric=False)