PyBrain库的example之NFQ流程图分析

__author__ = 'Thomas Rueckstiess, ruecksti@in.tum.de'

from pybrain.rl.environments.cartpole import CartPoleEnvironment, DiscreteBalanceTask, CartPoleRenderer
from pybrain.rl.agents import LearningAgent
from pybrain.rl.experiments import EpisodicExperiment
from pybrain.rl.learners.valuebased import NFQ, ActionValueNetwork
#,ActionValueLSTMNetwork
from pybrain.rl.explorers import BoltzmannExplorer

from numpy import array, arange, meshgrid, pi, zeros, mean
from matplotlib import pyplot as plt

# switch this to True if you want to see the cart balancing the pole (slower)
render = False  #True #

plt.ion()

env = CartPoleEnvironment()
if render:
    renderer = CartPoleRenderer()
    env.setRenderer(renderer)
    renderer.start()


# balancetask. py inside only used 2 sensors, so here can not use(4,3), just use (2,3)
# there is a debug in vesion 0.30, now, new version 0.33 had correct it!!
module = ActionValueNetwork(4,3)  #(4,3) #  0.33 had correct it
#module = ActionValueLSTMNetwork(2,3)

task = DiscreteBalanceTask(env, 100)
learner = NFQ()
learner.explorer.epsilon = 0.4

agent = LearningAgent(module, learner)
testagent = LearningAgent(module, None)
experiment = EpisodicExperiment(task, agent)


def plotPerformance(values, fig):
    plt.figure(fig.number)
    plt.clf()
    plt.plot(values, 'o-')
    plt.gcf().canvas.draw()


performance = []

if not render:
    pf_fig = plt.figure()

#while (True):
for _ in xrange(60): #60
    # one learning step after one episode of world-interaction!!!
    experiment.doEpisodes(1)
    agent.learn(2)  # 5

    # test performance (these real-world experiences are not used for training)
    if render:
        env.delay = True
    experiment.agent = testagent
    #r = mean([sum(x) for x in experiment.doEpisodes(5)])
    env.delay = False
    testagent.reset()
    experiment.agent = agent

    #performance.append(r)
    print "update step", len(performance)

    #print "reward avg", r
    print "explorer epsilon", learner.explorer.epsilon
    print "num episodes", agent.history.getNumSequences()
    print "update step", len(performance)

if not render:
    plotPerformance(performance, pf_fig)

str = raw_input("please input sth to end!")
print "you put :",str