PyTorch 深度学习实战（23）：多任务强化学习（Multi-Task RL）之扩展

之前的PyTorch 深度学习实战（23）：多任务强化学习（Multi-Task RL)总结扩展运用代码如下：

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from torch.distributions import Normal
from torch.amp import autocast, GradScaler
from metaworld.envs import ALL_V2_ENVIRONMENTS_GOAL_OBSERVABLE
import time
from collections import deque# ================== 配置参数 ==================
class MultiTaskPPOConfig:# 任务配置task_names = ['reach-v2-goal-observable','push-v2-goal-observable','pick-place-v2-goal-observable']num_tasks = 3# 网络架构shared_dim = 512task_specific_dim = 256meta_controller_dim = 128shared_layers = 2task_specific_layers = 1# 训练参数lr = 5e-5meta_lr = 1e-5gamma = 0.99gae_lambda = 0.97clip_epsilon = 0.15ppo_epochs = 5batch_size = 4096max_episodes = 10000max_steps = 200grad_clip = 0.5entropy_coef = 0.1# 探索参数initial_std = 1.5min_std = 0.2std_decay = 0.999# 课程学习安排curriculum_schedule = {0: ['reach-v2-goal-observable'],1000: ['reach-v2-goal-observable', 'push-v2-goal-observable'],3000: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'],6000: ['reach-v2-goal-observable', 'push-v2-goal-observable', 'pick-place-v2-goal-observable']}# 监控配置log_interval = 50eval_interval = 500eval_episodes = 10device = torch.device("cuda" if torch.cuda.is_available() else "cpu")# ================== MetaController ==================
class MetaController(nn.Module):def __init__(self, num_tasks, state_dim):super().__init__()self.net = nn.Sequential(nn.Linear(state_dim, MultiTaskPPOConfig.meta_controller_dim),nn.LayerNorm(MultiTaskPPOConfig.meta_controller_dim),nn.GELU(),nn.Linear(MultiTaskPPOConfig.meta_controller_dim, num_tasks))# 初始化参数for layer in self.net:if isinstance(layer, nn.Linear):nn.init.orthogonal_(layer.weight, gain=0.01)nn.init.constant_(layer.bias, 0.0)def forward(self, state):logits = self.net(state)return torch.softmax(logits, -1), logits# ================== 共享策略网络 ==================
class SharedPolicy(nn.Module):def __init__(self, state_dim, action_dim):super().__init__()self.action_dim = action_dimself.current_std = MultiTaskPPOConfig.initial_std# 共享网络层self.shared_net = nn.Sequential(nn.Linear(state_dim, MultiTaskPPOConfig.shared_dim),nn.LayerNorm(MultiTaskPPOConfig.shared_dim),nn.GELU(),nn.Linear(MultiTaskPPOConfig.shared_dim, MultiTaskPPOConfig.shared_dim),nn.GELU())# 多任务头部self.task_heads = nn.ModuleList()self.value_heads = nn.ModuleList()for _ in range(MultiTaskPPOConfig.num_tasks):# 动作头task_head = nn.Sequential(nn.Linear(MultiTaskPPOConfig.shared_dim, MultiTaskPPOConfig.task_specific_dim),nn.GELU(),nn.Linear(MultiTaskPPOConfig.task_specific_dim, action_dim))self.task_heads.append(task_head)# 值函数头value_head = nn.Sequential(nn.Linear(MultiTaskPPOConfig.shared_dim, MultiTaskPPOConfig.task_specific_dim),nn.GELU(),nn.Linear(MultiTaskPPOConfig.task_specific_dim, 1))self.value_heads.append(value_head)# 可学习的对数标准差self.log_std = nn.Parameter(torch.zeros(1, action_dim))# 初始化参数self._init_weights()def _init_weights(self):for head in self.task_heads:for layer in head:if isinstance(layer, nn.Linear):nn.init.orthogonal_(layer.weight, gain=0.01)nn.init.constant_(layer.bias, 0.0)for head in self.value_heads:for layer in head:if isinstance(layer, nn.Linear):nn.init.orthogonal_(layer.weight, gain=1.0)nn.init.constant_(layer.bias, 0.0)def decay_action_std(self):"""衰减动作标准差"""self.current_std = max(self.current_std * MultiTaskPPOConfig.std_decay,MultiTaskPPOConfig.min_std)def forward(self, states, task_ids):# 确保输入是float32states = states.float() if states.dtype != torch.float32 else statesshared_features = self.shared_net(states)batch_size = states.size(0)# 初始化输出张量action_means = torch.zeros(batch_size, self.action_dim,dtype=torch.float32,device=states.device)action_stds = torch.exp(self.log_std).expand(batch_size, -1) * self.current_stdvalues = torch.zeros(batch_size, 1,dtype=torch.float32,device=states.device)unique_task_ids = torch.unique(task_ids)for task_id_tensor in unique_task_ids:task_id = task_id_tensor.item()mask = (task_ids == task_id_tensor)if not mask.any():continueselected_features = shared_features[mask]# 计算任务特定输出with autocast(device_type=states.device.type, enabled=False):  # 禁用混合精度task_action = self.task_heads[task_id](selected_features.float())task_value = self.value_heads[task_id](selected_features.float())action_means[mask] = task_actionvalues[mask] = task_valuereturn action_means, action_stds, values# ================== 训练系统 ==================
class EnhancedMultiTaskPPOTrainer:def __init__(self):# 初始化多任务环境self.envs = []self.state_dim = Noneself.action_dim = None# 验证环境并获取维度for task_name in MultiTaskPPOConfig.task_names:env = ALL_V2_ENVIRONMENTS_GOAL_OBSERVABLE[task_name]()obs, _ = env.reset()if self.state_dim is None:self.state_dim = obs.shape[0]self.action_dim = env.action_space.shape[0]else:assert obs.shape[0] == self.state_dim, f"状态维度不一致: {task_name}"self.envs.append(env)# 初始化策略网络self.policy = SharedPolicy(self.state_dim, self.action_dim).to(MultiTaskPPOConfig.device)self.optimizer = optim.AdamW(self.policy.parameters(), lr=MultiTaskPPOConfig.lr)self.scheduler = optim.lr_scheduler.CosineAnnealingLR(self.optimizer,T_max=MultiTaskPPOConfig.max_episodes,eta_min=1e-6)self.scaler = GradScaler(enabled=MultiTaskPPOConfig.device.type == 'cuda')# 初始化MetaControllerself.meta_controller = MetaController(MultiTaskPPOConfig.num_tasks,self.state_dim).to(MultiTaskPPOConfig.device)self.meta_optimizer = optim.Adam(self.meta_controller.parameters(),lr=MultiTaskPPOConfig.meta_lr)# 初始化经验回放缓冲self.buffer = deque(maxlen=MultiTaskPPOConfig.max_steps)# 课程学习状态self.current_phase = 0self.phase_thresholds = sorted(MultiTaskPPOConfig.curriculum_schedule.keys())# 训练统计self.episode_rewards = {i: deque(maxlen=100) for i in range(MultiTaskPPOConfig.num_tasks)}self.episode_lengths = {i: deque(maxlen=100) for i in range(MultiTaskPPOConfig.num_tasks)}self.meta_data = {'states': [],'chosen_tasks': [],'rewards': []}# 评估统计self.eval_rewards = {i: [] for i in range(MultiTaskPPOConfig.num_tasks)}self.eval_success = {i: [] for i in range(MultiTaskPPOConfig.num_tasks)}def get_current_tasks(self, episode):"""获取当前课程阶段的任务列表"""if len(self.phase_thresholds) > 1 and self.current_phase < len(self.phase_thresholds) - 1:if episode >= self.phase_thresholds[self.current_phase + 1]:self.current_phase += 1task_names = MultiTaskPPOConfig.curriculum_schedule[self.phase_thresholds[self.current_phase]]return [MultiTaskPPOConfig.task_names.index(name) for name in task_names]def collect_experience(self, num_steps, episode):"""集成课程学习和meta controller的经验收集"""current_tasks = self.get_current_tasks(episode)for _ in range(num_steps):# 从当前课程任务中随机选择基础任务base_task_id = np.random.choice(current_tasks)env = self.envs[base_task_id]if not hasattr(env, '_last_obs'):state, _ = env.reset()else:state = env._last_obs# MetaController调整state_tensor = torch.FloatTensor(state).unsqueeze(0).to(MultiTaskPPOConfig.device)with torch.no_grad():task_probs, _ = self.meta_controller(state_tensor)task_probs = task_probs.squeeze().cpu().numpy()# 过滤概率分布mask = np.zeros_like(task_probs)mask[current_tasks] = 1filtered_probs = task_probs * maskfiltered_probs = filtered_probs / (filtered_probs.sum() + 1e-6)# 任务选择策略if np.random.rand() < 0.7:task_id = np.random.choice(current_tasks, p=filtered_probs[current_tasks])else:task_id = np.random.choice(current_tasks)# 记录meta controller决策self.meta_data['states'].append(state_tensor)self.meta_data['chosen_tasks'].append(task_id)# 执行选择的taskenv = self.envs[task_id]with torch.no_grad():task_id_tensor = torch.tensor([task_id], dtype=torch.long, device=MultiTaskPPOConfig.device)action_mean, action_std, value = self.policy(state_tensor, task_id_tensor)dist = Normal(action_mean.float(), action_std.float())  # 确保分布参数是float32action = dist.sample().squeeze(0)log_prob = dist.log_prob(action).sum(-1, keepdim=True)action_np = action.cpu().numpy()next_state, reward, done, trunc, info = env.step(action_np)# 记录数据self.buffer.append({'state': state,'action': action_np,'log_prob': log_prob.cpu(),'reward': float(reward),'done': bool(done),'task_id': task_id,'value': float(value.item()),'success': info.get('success', False)})# 记录meta controller的反馈self.meta_data['rewards'].append(reward)state = next_state if not (done or trunc) else env.reset()[0]def compute_gae(self, values, rewards, dones):"""计算广义优势估计(GAE)"""advantages = []last_advantage = 0next_value = 0next_non_terminal = 1.0for t in reversed(range(len(rewards))):delta = rewards[t] + MultiTaskPPOConfig.gamma * next_value * next_non_terminal - values[t]last_advantage = delta + MultiTaskPPOConfig.gamma * MultiTaskPPOConfig.gae_lambda * next_non_terminal * last_advantageadvantages.append(last_advantage)next_value = values[t]next_non_terminal = 1.0 - dones[t]advantages = torch.tensor(advantages[::-1], dtype=torch.float32).to(MultiTaskPPOConfig.device)returns = advantages + torch.tensor(values, dtype=torch.float32).to(MultiTaskPPOConfig.device)return (advantages - advantages.mean()) / (advantages.std() + 1e-8), returnsdef calculate_task_weights(self):"""基于最近表现计算任务权重"""task_weights = torch.ones(MultiTaskPPOConfig.num_tasks,device=MultiTaskPPOConfig.device)for task_id in range(MultiTaskPPOConfig.num_tasks):if len(self.episode_rewards[task_id]) > 10:# 计算最近10个episode的成功率recent_rewards = list(self.episode_rewards[task_id])[-10:]success_rate = sum(1 for r in recent_rewards if r > 0) / len(recent_rewards)# 动态调整权重if success_rate < 0.3:task_weights[task_id] = 2.0  # 困难任务加倍权重elif success_rate > 0.8:task_weights[task_id] = 0.5  # 简单任务减半权重return task_weights / task_weights.sum()def update_meta_controller(self):"""更新任务选择策略"""if len(self.meta_data['states']) == 0:returnstates = torch.cat(self.meta_data['states'])chosen_tasks = torch.tensor(self.meta_data['chosen_tasks'],device=MultiTaskPPOConfig.device)rewards = torch.tensor(self.meta_data['rewards'],dtype=torch.float32,device=MultiTaskPPOConfig.device)# 清空数据self.meta_data = {'states': [],'chosen_tasks': [],'rewards': []}# 归一化奖励rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-6)# 更新MetaControllertask_probs, logits = self.meta_controller(states)selected_probs = task_probs.gather(1, chosen_tasks.unsqueeze(1))loss = -torch.log(selected_probs + 1e-6) * rewards.unsqueeze(1)loss = loss.mean()self.meta_optimizer.zero_grad()loss.backward()torch.nn.utils.clip_grad_norm_(self.meta_controller.parameters(),MultiTaskPPOConfig.grad_clip)self.meta_optimizer.step()def update_policy(self):"""策略更新方法"""if not self.buffer:return 0, 0, 0# 从缓冲中提取数据batch = list(self.buffer)states = torch.tensor([x['state'] for x in batch],dtype=torch.float32,device=MultiTaskPPOConfig.device)actions = torch.FloatTensor(np.array([x['action'] for x in batch])).to(MultiTaskPPOConfig.device)old_log_probs = torch.cat([x['log_prob'] for x in batch]).to(MultiTaskPPOConfig.device)rewards = torch.FloatTensor([x['reward'] for x in batch]).to(MultiTaskPPOConfig.device)dones = torch.FloatTensor([x['done'] for x in batch]).to(MultiTaskPPOConfig.device)task_ids = torch.tensor([x['task_id'] for x in batch],dtype=torch.long,device=MultiTaskPPOConfig.device)values = torch.FloatTensor([x['value'] for x in batch]).to(MultiTaskPPOConfig.device)successes = torch.FloatTensor([x['success'] for x in batch]).to(MultiTaskPPOConfig.device)# 计算GAE和returnsadvantages, returns = self.compute_gae(values.cpu().numpy(), rewards.cpu().numpy(), dones.cpu().numpy())# 计算任务权重task_weights = self.calculate_task_weights()# 自动混合精度训练total_policy_loss = 0total_value_loss = 0total_entropy = 0for _ in range(MultiTaskPPOConfig.ppo_epochs):# 随机打乱数据perm = torch.randperm(len(batch))for i in range(0, len(batch), MultiTaskPPOConfig.batch_size):idx = perm[i:i + MultiTaskPPOConfig.batch_size]# 获取小批量数据batch_states = states[idx]batch_actions = actions[idx]batch_old_log_probs = old_log_probs[idx]batch_returns = returns[idx]batch_advantages = advantages[idx]batch_task_ids = task_ids[idx]with autocast(device_type=MultiTaskPPOConfig.device.type,enabled=MultiTaskPPOConfig.device.type == 'cuda'):# 前向传播action_means, action_stds, new_values = self.policy(batch_states, batch_task_ids)dist = Normal(action_means, action_stds)new_log_probs = dist.log_prob(batch_actions).sum(-1, keepdim=True)entropy = dist.entropy().mean()# 计算重要性采样比率ratio = (new_log_probs - batch_old_log_probs).exp()# 策略损失surr1 = ratio * batch_advantages.unsqueeze(-1)surr2 = torch.clamp(ratio, 1 - MultiTaskPPOConfig.clip_epsilon,1 + MultiTaskPPOConfig.clip_epsilon) * batch_advantages.unsqueeze(-1)policy_loss_per_task = -torch.min(surr1, surr2)# 应用任务权重selected_weights = task_weights[batch_task_ids].unsqueeze(-1)policy_loss = (policy_loss_per_task * selected_weights).mean()policy_loss -= MultiTaskPPOConfig.entropy_coef * entropy# 值函数损失 (带clip)value_pred_clipped = values[idx] + (new_values - values[idx]).clamp(-MultiTaskPPOConfig.clip_epsilon,MultiTaskPPOConfig.clip_epsilon)value_loss1 = (new_values.squeeze() - batch_returns).pow(2)value_loss2 = (value_pred_clipped.squeeze() - batch_returns).pow(2)value_loss = 0.5 * torch.max(value_loss1, value_loss2).mean()# 总损失loss = policy_loss + value_loss# 反向传播self.scaler.scale(loss).backward()total_policy_loss += policy_loss.item()total_value_loss += value_loss.item()total_entropy += entropy.item()# 梯度裁剪和参数更新self.scaler.unscale_(self.optimizer)torch.nn.utils.clip_grad_norm_(self.policy.shared_net.parameters(), 1.0)torch.nn.utils.clip_grad_norm_(list(self.policy.task_heads.parameters()) +list(self.policy.value_heads.parameters()),0.5)self.scaler.step(self.optimizer)self.scaler.update()self.optimizer.zero_grad()self.scheduler.step()# 衰减动作噪声self.policy.decay_action_std()return (total_policy_loss / MultiTaskPPOConfig.ppo_epochs,total_value_loss / MultiTaskPPOConfig.ppo_epochs,total_entropy / MultiTaskPPOConfig.ppo_epochs)def evaluate_policy(self):"""评估当前策略性能"""eval_results = {i: {'rewards': [], 'successes': []} for i in range(MultiTaskPPOConfig.num_tasks)}for task_id in range(MultiTaskPPOConfig.num_tasks):env = self.envs[task_id]for _ in range(MultiTaskPPOConfig.eval_episodes):state, _ = env.reset()episode_reward = 0done = Falsesuccess = Falsefor _ in range(MultiTaskPPOConfig.max_steps):with torch.no_grad():state_tensor = torch.FloatTensor(state).unsqueeze(0).to(MultiTaskPPOConfig.device)task_id_tensor = torch.tensor([task_id], dtype=torch.long, device=MultiTaskPPOConfig.device)action_mean, _, _ = self.policy(state_tensor, task_id_tensor)action = action_mean.squeeze(0).cpu().numpy()state, reward, done, trunc, info = env.step(action)episode_reward += rewardsuccess = success or info.get('success', False)if done or trunc:breakeval_results[task_id]['rewards'].append(episode_reward)eval_results[task_id]['successes'].append(success)# 记录评估结果for task_id in range(MultiTaskPPOConfig.num_tasks):avg_reward = np.mean(eval_results[task_id]['rewards'])success_rate = np.mean(eval_results[task_id]['successes'])self.eval_rewards[task_id].append(avg_reward)self.eval_success[task_id].append(success_rate)return eval_resultsdef train(self):print(f"开始训练，设备：{MultiTaskPPOConfig.device}")print(f"课程安排：{MultiTaskPPOConfig.curriculum_schedule}")start_time = time.time()# 初始评估self.evaluate_policy()for episode in range(MultiTaskPPOConfig.max_episodes):# 经验收集阶段self.collect_experience(MultiTaskPPOConfig.max_steps, episode)# 策略优化阶段policy_loss, value_loss, entropy = self.update_policy()# MetaController更新self.update_meta_controller()# 记录统计信息for exp in self.buffer:task_id = exp['task_id']self.episode_rewards[task_id].append(exp['reward'])self.episode_lengths[task_id].append(1)# 定期输出日志if (episode + 1) % MultiTaskPPOConfig.log_interval == 0:avg_rewards = {k: np.mean(v) if v else 0 for k, v in self.episode_rewards.items()}success_rates = {k: np.mean([1 if r > 0 else 0 for r in v]) if v else 0for k, v in self.episode_rewards.items()}time_cost = time.time() - start_time# 打印当前课程阶段current_task_names = MultiTaskPPOConfig.curriculum_schedule[self.phase_thresholds[self.current_phase]]print(f"\nEpisode {episode + 1:5d} | Time: {time_cost:6.1f}s")print(f"当前课程阶段: {current_task_names} (Phase {self.current_phase})")print(f"动作标准差: {self.policy.current_std:.3f} | 学习率: {self.scheduler.get_last_lr()[0]:.2e}")for task_id in range(MultiTaskPPOConfig.num_tasks):task_name = MultiTaskPPOConfig.task_names[task_id]print(f"  {task_name:25s} | Avg Reward: {avg_rewards[task_id]:7.2f} | Success Rate: {success_rates[task_id]:.2f}")print(f"  Policy Loss: {policy_loss:.4f} | Value Loss: {value_loss:.4f} | Entropy: {entropy:.4f}")start_time = time.time()# 定期评估if (episode + 1) % MultiTaskPPOConfig.eval_interval == 0:eval_results = self.evaluate_policy()if (episode + 1) % 1000 == 0:print("\n评估结果:")for task_id in range(MultiTaskPPOConfig.num_tasks):task_name = MultiTaskPPOConfig.task_names[task_id]avg_reward = np.mean(eval_results[task_id]['rewards'])success_rate = np.mean(eval_results[task_id]['successes'])print(f"  {task_name:25s} | Avg Reward: {avg_reward:7.2f} | Success Rate: {success_rate:.2f}")# 训练结束保存模型torch.save({'policy_state_dict': self.policy.state_dict(),'meta_controller_state_dict': self.meta_controller.state_dict(),'optimizer_state_dict': self.optimizer.state_dict()}, "multitask_ppo_model.pth")if __name__ == "__main__":trainer = EnhancedMultiTaskPPOTrainer()print(f"状态维度: {trainer.state_dim}, 动作维度: {trainer.action_dim}")trainer.train()

部分输出为：

Episode    50 | Time:  216.6s
当前课程阶段: ['reach-v2-goal-observable'] (Phase 0)
动作标准差: 1.427 | 学习率: 5.00e-05reach-v2-goal-observable  | Avg Reward:    1.42 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.00 | Success Rate: 0.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: -0.1777 | Value Loss: 471.4303 | Entropy: 1.7773Episode   100 | Time:  193.3s
当前课程阶段: ['reach-v2-goal-observable'] (Phase 0)
动作标准差: 1.357 | 学习率: 5.00e-05reach-v2-goal-observable  | Avg Reward:    1.42 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.00 | Success Rate: 0.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: -0.1729 | Value Loss: 357.7264 | Entropy: 1.7293......Episode  2800 | Time:  198.6s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 4.11e-05reach-v2-goal-observable  | Avg Reward:    1.44 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0092 | Value Loss: 191.3147 | Entropy: -0.0918Episode  2850 | Time:  212.2s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 4.08e-05reach-v2-goal-observable  | Avg Reward:    1.44 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0090 | Value Loss: 183.6324 | Entropy: -0.0902Episode  2900 | Time:  210.4s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 4.05e-05reach-v2-goal-observable  | Avg Reward:    1.44 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0089 | Value Loss: 188.5185 | Entropy: -0.0889Episode  2950 | Time:  210.1s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 4.02e-05reach-v2-goal-observable  | Avg Reward:    1.44 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0087 | Value Loss: 183.0386 | Entropy: -0.0874Episode  3000 | Time:  212.0s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 3.99e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0086 | Value Loss: 182.9761 | Entropy: -0.0858评估结果:reach-v2-goal-observable  | Avg Reward:  106.66 | Success Rate: 0.00push-v2-goal-observable   | Avg Reward:    3.99 | Success Rate: 0.00pick-place-v2-goal-observable | Avg Reward:    4.49 | Success Rate: 0.00Episode  3050 | Time:  234.3s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.96e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0084 | Value Loss: 28.1028 | Entropy: -0.0843Episode  3100 | Time:  210.3s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.93e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0083 | Value Loss: 0.1660 | Entropy: -0.0829Episode  3150 | Time:  209.8s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.90e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0082 | Value Loss: 0.1506 | Entropy: -0.0818Episode  3200 | Time:  210.2s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.86e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0080 | Value Loss: 0.1429 | Entropy: -0.0801Episode  3250 | Time:  210.3s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.83e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0079 | Value Loss: 0.1725 | Entropy: -0.0785Episode  3300 | Time:  209.7s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.80e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0077 | Value Loss: 0.1990 | Entropy: -0.0771Episode  3350 | Time:  209.5s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.76e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0076 | Value Loss: 0.2084 | Entropy: -0.0758Episode  3400 | Time:  210.1s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.73e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0075 | Value Loss: 0.2057 | Entropy: -0.0745Episode  3450 | Time:  210.9s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.70e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0073 | Value Loss: 0.2251 | Entropy: -0.0733Episode  3500 | Time:  210.1s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.66e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0072 | Value Loss: 0.2199 | Entropy: -0.0723......