old_contents: '''
  In this example the Q-learning algorithm is used via self-play
  to learn the state-action values for all Tic-Tac-Toe positions.
  '''
  from capstone.game.games import TicTacToe
  from capstone.game.utils import tic2pdf
  from capstone.rl import Environment, GameMDP
  from capstone.rl.learners import QLearningSelfPlay
  from capstone.rl.value_functions import TabularF
  
  game = TicTacToe()
  env = Environment(GameMDP(game))
  qlearning = QLearningSelfPlay(env, n_episodes=1000, random_state=0)
  qlearning.learn()
  
  for move in game.legal_moves():
      print('-' * 80)
      value = qlearning.qf[(game, move)]
      new_game = game.copy().make_move(move)
      print(value)
      print(new_game)
  
new_contents: '''
  The Q-learning algorithm is used to learn the state-action values for all
  Tic-Tac-Toe positions by playing games against itself (self-play).
  '''
  from capstone.game.games import TicTacToe
  from capstone.game.players import GreedyQF, RandPlayer
  from capstone.game.utils import play_series, tic2pdf
  from capstone.rl import Environment, GameMDP
  from capstone.rl.learners import QLearningSelfPlay
  
  game = TicTacToe()
  env = Environment(GameMDP(game))
  qlearning = QLearningSelfPlay(env, n_episodes=100000, verbose=0)
  qlearning.learn()
  
  for move in game.legal_moves():
      print('-' * 80)
      value = qlearning.qf[game, move]
      new_game = game.copy().make_move(move)
      print(value)
      print(new_game)
  
  players = [GreedyQF(qlearning.qf), RandPlayer()]
  play_series(TicTacToe(), players, n_matches=10000)
  
  # show number of unvisited state?
  
subject: Fix Tic-Tac-Toe Q-learning tabular self-play example