Epsilon-Greedy 법으로 만족도가 높은 레스토랑을 찾는 방법을 생각해 보았습니다.

목적

모델

결과

코드

import random
import numpy as np

class ChoiceRestaurant():

  def __init__(self, online_reviews, max_episode_steps=30):
    self.online_reviews = online_reviews
    self.max_episode_steps = max_episode_steps
    self.toss_count = 0

  def __len__(self):
    return len(self.online_reviews)

  def reset(self):
    self.toss_count = 0

  def step(self, action):
    final = self.max_episode_steps - 1
    if self.toss_count > final:
      raise Exception("The step count exceeded maximum. Please reset env.")
    else:
      done = True if self.toss_count == final else False

    if action >= len(self.online_reviews):
      raise Exception("The No.{} restaurant doesn't exitst.".format(action))
    else:
      average_online_reviews = sum(self.online_reviews)/len(self.online_reviews)
      online_review = self.online_reviews[action]
      sigma = 0.5
      if average_online_reviews < random.gauss(online_review, sigma):
        reward = 1.0
      else:
        reward = 0.0
      self.toss_count += 1
      return reward, done

class EpsilonGreedyAgent():

  def __init__(self, epsilon):
    self.epsilon = epsilon
    self.V = []

  def policy(self):
    restaurants = range(len(self.V))
    if random.random() < self.epsilon:
      return random.choice(restaurants)
    else:
      return np.argmax(self.V)

  def play(self, env):
    # Initialize estimation.
    N = [0] * len(env)
    self.V = [0] * len(env)
    env.reset()
    done = False
    rewards = []
    while not done:
      selected_restaurant = self.policy()
      reward, done = env.step(selected_restaurant)
      rewards.append(reward)

      n = N[selected_restaurant]
      restaurant_average = self.V[selected_restaurant]
      new_average = (restaurant_average * n + reward) / (n+1)
      N[selected_restaurant] += 1
      self.V[selected_restaurant] = new_average

    return rewards

if __name__ == "__main__":
  import pandas as pd
  import matplotlib.pyplot as plt

  def main():
    env = ChoiceRestaurant([2.8, 4.1, 3.5, 4.1, 3.7, 3.9, 3.6, 3.4, 4.0, 3.6])
    epsilons = [0.0, 0.1, 0.2, 0.3, 0.8]
    game_steps = list(range(10, 3010, 10))
    result = {}
    for e in epsilons:
      agent = EpsilonGreedyAgent(epsilon=e)
      means = []
      for s in game_steps:
        env.max_episode_steps = s
        rewards = agent.play(env)
        means.append(np.mean(rewards))
      result["epsilon={}".format(e)] = means
    result["restaurant choice count"] = game_steps
    result = pd.DataFrame(result)
    result.set_index("restaurant choice count", drop=True, inplace=True)
    result.plot.line(figsize=(10,5))
    plt.show()

  main()

참고