问题描述
我正在使用pytorch制作具有4个过程的A3C，如下代码所示。
但是令我惊讶的是，在训练A3C动作值时，所有动作都变成了nan。 最初，动作值不是nan。
但是经过一整夜的训练后，它进入了南。 有人可以帮忙让我知道那里是什么问题。
class SharedAdam(torch.optim.Adam):
def __init__(self,params,lr=1e-3,betas=(0.9,0.99),eps=1e-8,weight_decay=0):
super(SharedAdam,self).__init__(params,lr=lr,betas=betas,eps=eps,weight_decay=weight_decay)
# State initialization
for group in self.param_groups:
for p in group['params']:
state = self.state[p]
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p.data)
state['exp_avg_sq'] = torch.zeros_like(p.data)
# share in memory
state['exp_avg'].share_memory_()
state['exp_avg_sq'].share_memory_()
class ActorCritic(torch.nn.Module):
def __init__(self,num_inputs,action_space):
super(ActorCritic,self).__init__()
self.num_inputs = num_inputs
self.action_space = action_space
self.lstm = nn.LSTMCell(num_inputs,num_inputs)
num_outputs = action_space
self.fc1 = nn.Linear(num_inputs,256)
self.fc1.apply(init_weights)
self.fc2 = nn.Linear(256,256)
self.fc2.apply(init_weights)
self.critic_linear = nn.Linear(256,1)
self.critic_linear.apply(init_weights)
self.actor_linear = nn.Linear(256,num_outputs)
self.actor_linear.apply(init_weights)
self.lstm.bias_ih.data.fill_(0)
self.lstm.bias_hh.data.fill_(0)
self.sig1 = nn.Sigmoid()
self.train()
def forward(self,inputs):
inputs,(hx,cx) = inputs
hx,cx = self.lstm(inputs,cx))
x = self.sig1(self.fc1(hx))
x = torch.tanh(self.fc2(x))
return self.critic_linear(x),self.actor_linear(x),cx)
def save(self,filename,directory):
torch.save(self.state_dict(),'%s/%s_actor.pth' % (directory,filename))
def load(self,directory):
self.load_state_dict(torch.load('%s/%s_actor.pth' % (directory,filename)))
及以下是培训代码
def train(rank,model,optimizer,data):
try:
data = data.dropna()
count = 0
data = torch.DoubleTensor(np.asarray(data))
env = ENV(params.state_dim,params.action_dim,data)
print("env created\n")
# init training variables
max_timesteps = data.shape[0] - 1
state = env.reset()
done = True
episode_length = 0
count = 0
while count<max_timesteps-1:
episode_length += 1
if done:
cx = Variable(torch.zeros(1,params.state_dim))
hx = Variable(torch.zeros(1,params.state_dim))
else:
cx = Variable(cx.data)
hx = Variable(hx.data)
values = []
log_probs = []
rewards = []
entropies = []
while count<max_timesteps-1:
value,action_values,cx) = model((Variable(state.unsqueeze(0)),cx)))
prob = F.softmax(action_values,dim = -1)
log_prob = F.log_softmax(action_values,dim=-1).reshape(-1,)
entropy = -(log_prob * prob).sum(1,keepdim=True)
entropies.append(entropy)
action = sample(prob)
log_prob = log_prob.gather(0,Variable(action))
state,reward,done = env.step(action)
done = (done or count == max_timesteps-2)
reward = max(min(reward,1),-1)
count +=1
if done:
episode_length = 0
state = env.reset()
values.append(value)
log_probs.append(log_prob)
rewards.append(reward)
print(ticker,"rank ",rank," action:",action,"reward ",reward)
if done:
break
R = torch.zeros(1,1)
if not done:
value,_,_ = model((Variable(state.unsqueeze(0)),cx)))
R = value.data
values.append(Variable(R))
policy_loss = 0
value_loss = 0
R = Variable(R)
gae = torch.zeros(1,1)
for i in reversed(range(len(rewards))):
R = params.gamma * R + rewards[i]
advantage = R - values[i]
value_loss = value_loss + 0.5 * advantage.pow(2)
TD = rewards[i] + params.gamma * values[i + 1].data - values[i].data
gae = gae * params.gamma * params.tau + TD
policy_loss = policy_loss - log_probs[i] * Variable(gae) - 0.01 * entropies[i]
optimizer.zero_grad()
(policy_loss + 0.5 * value_loss).backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),40)
optimizer.step()
except:
traceback.print_exc()
以下是用于采样动作的代码
def sample(logits):
noise = torch.rand(logits.shape)
return torch.argmax(logits - torch.log(-torch.log(noise)),1)
解决方法
暂无找到可以解决该程序问题的有效方法，小编努力寻找整理中！
如果你已经找到好的解决方法，欢迎将解决方案带上本链接一起发送给小编。
小编邮箱:dio#foxmail.com (将#修改为@）