Tabular Methods Comparison

import numpy as np
import matplotlib.pyplot as plt
import gymnasium as gym

from rlforge.agents.tabular import SarsaAgent, QAgent, ExpectedSarsaAgent
from rlforge.experiments import ExperimentRunner

env = gym.make("CliffWalking-v1")
sarsa_agent = SarsaAgent(step_size=0.5,
                         discount=1,
                         num_states=env.observation_space.n,
                         num_actions=env.action_space.n,
                         epsilon=0.1)
q_agent = QAgent(step_size=0.5,
                 discount=1,
                 num_states=env.observation_space.n,
                 num_actions=env.action_space.n,
                 epsilon=0.1)
expected_sarsa_agent = ExpectedSarsaAgent(step_size=0.5,
                                          discount=1,
                                          num_states=env.observation_space.n,
                                          num_actions=env.action_space.n,
                                          epsilon=0.1)

algorithms = ["SARSA", "Q Learning", "Expected SARSA"]
agents = [sarsa_agent, q_agent, expected_sarsa_agent]

rewards_list = []
steps_per_episode_list = []
results_by_agent = {}

for name, agent in zip(algorithms, agents):
    runner = ExperimentRunner(env, agent)

    results = runner.run_episodic(
        num_runs=100,
        num_episodes=500,
        max_steps_per_episode=10000
    )

    # Store results for later analysis
    rewards_list.append(results["rewards"])
    steps_per_episode_list.append(results["steps"])
    results_by_agent[name] = results

    # Optional: print a quick summary for each agent
    print(f"\n{name} Summary:")
    runner.summary(last_n=10)

SARSA Summary:
============================================================
 Experiment Summary (Episodic)
============================================================
Runs: 100
Average runtime per run: 0.894 seconds
Episodes per run (Max): 500
First episode mean reward: -1772.490
Last episode mean reward: -28.440
Overall mean reward: -36.818
Mean reward (last 10 episodes): -25.545
First episode mean steps: 494.4
Last episode mean steps: 24.5
Overall mean steps: 26.7
============================================================

Q Learning Summary:
============================================================
 Experiment Summary (Episodic)
============================================================
Runs: 100
Average runtime per run: 0.764 seconds
Episodes per run (Max): 500
First episode mean reward: -1790.100
Last episode mean reward: -49.340
Overall mean reward: -57.028
Mean reward (last 10 episodes): -51.990
First episode mean steps: 481.3
Last episode mean steps: 16.7
Overall mean steps: 20.8
============================================================

Expected SARSA Summary:
============================================================
 Experiment Summary (Episodic)
============================================================
Runs: 100
Average runtime per run: 1.362 seconds
Episodes per run (Max): 500
First episode mean reward: -1535.060
Last episode mean reward: -19.210
Overall mean reward: -28.337
Mean reward (last 10 episodes): -19.407
First episode mean steps: 428.2
Last episode mean steps: 17.2
Overall mean steps: 20.9
============================================================

plt.figure(figsize=(10, 6))

for idx, algo_rewards in enumerate(rewards_list):

  plt.plot(np.mean(algo_rewards, axis=1), label = algorithms[idx])

plt.xlabel("Episode")
plt.ylabel("Average Reward During Episode")
plt.ylim(-100,0)
plt.legend()
plt.grid()

plt.figure(figsize=(10, 6))

for idx, algo_steps in enumerate(steps_per_episode_list):

  plt.plot(np.mean(algo_steps, axis=1), label = algorithms[idx])

plt.xlabel("Episode")
plt.ylabel("Average Steps per Episodes")
plt.legend()
plt.grid()