在morvan的DDPG基础上进行的改进，可能做得不完全正确，如有问题还请提出来。
import tensorflow as tf
import numpy as np
import gym
import time
##################### hyper parameters ####################
MAX_EPISODES = 200
MAX_EP_STEPS = 200
LR_A = 0.001 # learning rate for actor
LR_C = 0.002 # learning rate for critic
GAMMA = 0.9 # reward discount
TAU = 0.01 # soft replacement
MEMORY_CAPACITY = 10000
BATCH_SIZE = 32
RENDER = False
ENV_NAME = 'Pendulum-v0'
############################### DDPG ####################################
class TD3(object):
def __init__(self, a_dim, s_dim, a_bound,):
self.memory = np.zeros((MEMORY_CAPACITY, s_dim * 2 + a_dim + 1), dtype=np.float32)
self.pointer = 0
self.update_cnt = 0 #更新次数
self.policy_target_update_interval = 3 #策略网络更新频率
self.sess = tf.Session()
self.a_dim, self.s_dim, self.a_bound = a_dim, s_dim, a_bound,
self.S = tf.placeholder(tf.float32, [None, s_dim], 's')
self.S_ = tf.placeholder(tf.float32, [None, s_dim], 's_')
self.R = tf.placeholder(tf.float32, [None, 1], 'r')
with tf.variable_scope('Actor'):
self.a = self._build_a(self.S, scope='eval', trainable=True)
a_ = self._build_a(self.S_, scope='target', trainable=False)
sample = tf.distributions.Normal(loc=0., scale=1.)
noise = tf.clip_by_value(sample.sample(1) * 0.5, -1, 1)
noise_a_ = a_ + noise
with tf.variable_scope('Critic'):
# assign self.a = a in memory when calculating q for td_error,
# otherwise the self.a is from Actor when updating Actor
q1 = self._build_c(self.S, self.a, scope='eval1', trainable=True)
q1_ = self._build_c(self.S_, noise_a_, scope='target1', trainable=False)
q2 = self._build_c(self.S, self.a, scope='eval2', trainable=True)
q2_ = self._build_c(self.S_, noise_a_, scope='target2', trainable=False)
# networks parameters
self.ae_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/eval')
self.at_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/target')
self.ce_params1 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/eval1')
self.ct_params1 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/target1')
self.ce_params2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/eval2')
self.ct_params2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/target2')
# target net replacement
self.soft_replace = [tf.assign(t, (1 - TAU) * t + TAU * e)
for t, e in zip(self.at_params + self.ct_params1 + self.ct_params2, self.ae_params + self.ce_params1 + self.ce_params2 )]
self.hard_replace = [tf.assign(t, e)
for t, e in zip(self.at_params + self.ct_params1 + self.ct_params2, self.ae_params + self.ce_params1 + self.ce_params2 )]
q_target = self.R + GAMMA * tf.minimum(q1_, q2_)
# in the feed_dic for the td_error, the self.a should change to actions in memory
td_error1 = tf.losses.mean_squared_error(labels=q_target, predictions=q1)
self.ctrain1 = tf.train.AdamOptimizer(LR_C).minimize(td_error1, var_list=self.ce_params1)
td_error2 = tf.losses.mean_squared_error(labels=q_target, predictions=q2)
self.ctrain2 = tf.train.AdamOptimizer(LR_C).minimize(td_error2, var_list=self.ce_params2)
a_loss = - tf.reduce_mean(q1) # maximize the q
self.atrain = tf.train.AdamOptimizer(LR_A).minimize(a_loss, var_list=self.ae_params)
self.sess.run(tf.global_variables_initializer())
self.sess.run(self.hard_replace) # 初始化目标网络
def choose_action(self, s):
return self.sess.run(self.a, {self.S: s[np.newaxis, :]})[0]
def learn(self):
# soft target replacement
self.update_cnt += 1
indices = np.random.choice(MEMORY_CAPACITY, size=BATCH_SIZE)
bt = self.memory[indices, :]
bs = bt[:, :self.s_dim]
ba = bt[:, self.s_dim: self.s_dim + self.a_dim]
br = bt[:, -self.s_dim - 1: -self.s_dim]
bs_ = bt[:, -self.s_dim:]
self.sess.run(self.ctrain1, {self.S: bs, self.a: ba, self.R: br, self.S_: bs_})
self.sess.run(self.ctrain2, {self.S: bs, self.a: ba, self.R: br, self.S_: bs_})
if self.update_cnt % self.policy_target_update_interval == 0:
self.sess.run(self.atrain, {self.S: bs})
self.sess.run(self.soft_replace)
def store_transition(self, s, a, r, s_):
transition = np.hstack((s, a, [r], s_))
index = self.pointer % MEMORY_CAPACITY # replace the old memory with new memory
self.memory[index, :] = transition
self.pointer += 1
def _build_a(self, s, scope, trainable):
with tf.variable_scope(scope):
net = tf.layers.dense(s, 30, activation=tf.nn.relu, name='l1', trainable=trainable)
a = tf.layers.dense(net, self.a_dim, activation=tf.nn.tanh, name='a', trainable=trainable)
return tf.multiply(a, self.a_bound, name='scaled_a')
def _build_c(self, s, a, scope, trainable):
with tf.variable_scope(scope):
n_l1 = 30
w1_s = tf.get_variable('w1_s', [self.s_dim, n_l1], trainable=trainable)
w1_a = tf.get_variable('w1_a', [self.a_dim, n_l1], trainable=trainable)
b1 = tf.get_variable('b1', [1, n_l1], trainable=trainable)
net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1)
return tf.layers.dense(net, 1, trainable=trainable) # Q(s,a)
############################### training ####################################
env = gym.make(ENV_NAME)
env = env.unwrapped
env.seed(1)
s_dim = env.observation_space.shape[0]
a_dim = env.action_space.shape[0]
a_bound = env.action_space.high
td3 = TD3(a_dim, s_dim, a_bound)
var = 3 # control exploration
t1 = time.time()
for i in range(MAX_EPISODES):
s = env.reset()
ep_reward = 0
for j in range(MAX_EP_STEPS):
if RENDER:
env.render()
# Add exploration noise
a = td3.choose_action(s)
a = np.clip(np.random.normal(a, var), -2, 2) # add randomness to action selection for exploration
s_, r, done, info = env.step(a)
td3.store_transition(s, a, r / 10, s_)
if td3.pointer > MEMORY_CAPACITY:
var *= .9995 # decay the action randomness
td3.learn()
s = s_
ep_reward += r
if j == MAX_EP_STEPS-1:
print('Episode:', i, ' Reward: %i' % int(ep_reward), 'Explore: %.2f' % var, )
if ep_reward > -300:RENDER = True
break
print('Running time: ', time.time() - t1)imp
