前言
本来想做强化学习小车，但是技术栈似乎飞了，所以还是一步一步来嘛。
Part 1 Code
import copy import random import json import matplotlib.pyplot as plt class OoxxMachine: def __init__(self): self.race = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] # 用于表示棋盘 0代表没下过 1 A玩家 2 B玩家 self.flag = "in_race" # all situation is "in_race" or "a_win" or "b_win" or "all_lose" # ---------------------------------------------------------------- self.learn_rate = 0.1 self.rand_poss = 0.05 self.net_values = {} self.default_value = 0.5 def update_win(self): """ [[2,1,1][1,1,0][1,2,0]] be like: 0 1 2 ------------- 0 | x | o | o | 1 | x | x | | 2 | x | o | | ------------- """ if self.race[0][0] == self.race[0][1] == self.race[0][2]: if self.race[0][0] == 1: self.flag = "a_win" elif self.race[0][0] == 2: self.flag = "b_win" else: pass if self.race[1][0] == self.race[1][1] == self.race[1][2]: if self.race[1][0] == 1: self.flag = "a_win" elif self.race[1][0] == 2: self.flag = "b_win" else: pass if self.race[2][0] == self.race[2][1] == self.race[2][2]: if self.race[2][0] == 1: self.flag = "a_win" elif self.race[2][0] == 2: self.flag = "b_win" else: pass if self.race[0][0] == self.race[1][0] == self.race[2][0]: if self.race[0][0] == 1: self.flag = "a_win" elif self.race[0][0] == 2: self.flag = "b_win" else: pass if self.race[0][1] == self.race[1][1] == self.race[2][1]: if self.race[0][1] == 1: self.flag = "a_win" elif self.race[0][1] == 2: self.flag = "b_win" else: pass if self.race[0][2] == self.race[1][2] == self.race[2][2]: if self.race[0][2] == 1: self.flag = "a_win" elif self.race[0][2] == 2: self.flag = "b_win" else: pass if self.race[0][0] == self.race[1][1] == self.race[2][2]: if self.race[2][2] == 1: self.flag = "a_win" elif self.race[2][2] == 2: self.flag = "b_win" else: pass if self.race[0][2] == self.race[1][1] == self.race[2][0]: if self.race[0][2] == 1: self.flag = "a_win" elif self.race[0][2] == 2: self.flag = "b_win" else: pass all_chess = 0 for i in range(0, 3): for j in range(0, 3): if self.race[i][j] != 0: all_chess += 1 # print(all_chess) if all_chess == 8 and self.flag == "in_race": self.flag = "all_lose" return False def reset(self): self.race = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] self.flag = "in_race" def do_once(self, racer: "str == a or b", location: list) -> str: could_do = True for i in range(0, 3): if 0 in self.race[i]: could_do = True else: pass if not could_do: self.flag = 'all_lose' return "fin" if racer == "a": if self.race[location[0]][location[1]] == 0: self.race[location[0]][location[1]] = 1 else: raise ValueError("this location has been used") if racer == "b": if self.race[location[0]][location[1]] == 0: self.race[location[0]][location[1]] = 2 else: raise ValueError("this location has been used") return "fin" # 我对强化学习的理解还不够透彻 def refresh_net(self, now_race: list, next_race: list) -> bool: # 传参： 赛场情况 需要更新的价值（在此赛场情况之前的价值） （本赛场）是否获胜 hash_value: int = hash(str(now_race)) hash_next: int = hash(str(next_race)) """ # 如果给下死了就给Value 置于 0 if self.flag == 'b_win' or self.flag == 'all_lose': self.net_values[hash_value] = 0 return False # 更新下一次预期之获胜情况 copy_race = self.race self.race = next_race self.update_win() if self.flag == 'a_win': next_value = 1 self.net_values[hash_value] = 1 elif self.flag == 'b_win' or "all_lose": next_value = 0 self.net_values[hash_value] = 0 self.race = copy_race self.update_win() """ next_value = self.net_values[hash_next] if hash_value not in self.net_values: self.net_values[hash_value] = self.default_value value = self.default_value else: value = self.net_values[hash_value] value = value + (next_value - value) * self.learn_rate self.net_values[hash_value] = value return True def save_net(self, filename='net.json'): with open(filename, 'w') as file: json.dump(self.net_values, file) print(f"Net values saved to {filename}.") def read_net(self, filename='net.json'): with open(filename, 'r') as file: self.net_values = json.load(file) print(f"Net values loaded from {filename}.") def random_player(self, player: str): possible_location = [] race_copy = self.race for i in range(0, 3): for j in range(0, 3): if race_copy[i][j] == 0: possible_location.append([i, j]) if not possible_location: self.flag = "all_lose" return False location = random.choice(possible_location) self.do_once(player, location) def start_train(self, epoch: int = 1000) -> bool: self.reset() a_win_times = 1 b_win_times = 1 win_rate = [] for times in range(1, epoch): win_rate.append(a_win_times / (a_win_times + b_win_times)) plt.plot(win_rate) # print(self.race) if self.flag == "a_win": a_win_times += 1 elif self.flag == "b_win": b_win_times += 1 # print(times) # print(self.net_values) self.reset() if random.randint(0, 1): while self.flag == "in_race": self.update_win() if random.random() >= self.rand_poss: next_races = [] for i in range(0, 3): for j in range(0, 3): if self.race[i][j] == 0: races_copy = copy.deepcopy(self.race) races_copy[i][j] = 1 next_races.append(races_copy) else: pass values = [] for next_race in next_races: copy_race = copy.deepcopy(self.race) self.race = copy.deepcopy(next_race) self.update_win() if self.flag == 'a_win': self.net_values[hash(str(next_race))] = 1 elif self.flag == 'b_win' or "all_lose": self.net_values[hash(str(next_race))] = 0 self.race = copy.deepcopy(copy_race) self.update_win() next_hash = hash(str(next_race)) if next_hash not in self.net_values: self.net_values[next_hash] = self.default_value values.append(self.default_value) else: values.append(self.net_values[next_hash]) max_value = max(values) max_indices = [index for index, value in enumerate(values) if value == max_value] random_max_index = random.choice(max_indices) next_race = next_races[random_max_index] # print(next_races) self.refresh_net(self.race, next_race) self.race = next_race # print(self.race) else: # print("random") if self.random_player("a"): pass else: break self.random_player("b") else: while self.flag == "in_race": self.update_win() self.random_player("b") if random.random() >= self.rand_poss: next_races = [] for i in range(0, 3): for j in range(0, 3): if self.race[i][j] == 0: races_copy = copy.deepcopy(self.race) races_copy[i][j] = 1 next_races.append(races_copy) else: pass values = [] for next_race in next_races: copy_race = copy.deepcopy(self.race) self.race = copy.deepcopy(next_race) self.update_win() if self.flag == 'a_win': self.net_values[hash(str(next_race))] = 1 elif self.flag == 'b_win' or "all_lose": self.net_values[hash(str(next_race))] = 0 self.race = copy.deepcopy(copy_race) self.update_win() next_hash = hash(str(next_race)) if next_hash not in self.net_values: self.net_values[next_hash] = self.default_value values.append(self.default_value) else: values.append(self.net_values[next_hash]) max_value = max(values) max_indices = [index for index, value in enumerate(values) if value == max_value] random_max_index = random.choice(max_indices) next_race = next_races[random_max_index] # print(next_races) self.refresh_net(self.race, next_race) self.race = next_race # print(self.race) else: # print("random") if self.random_player("a"): pass else: break # do the race once at here print(f"a wins {str(a_win_times)} b wins {str(b_win_times)}") print(f"A的胜率是{str(a_win_times / (a_win_times + b_win_times))}") plt.show() return True if __name__ == "__main__": aa = OoxxMachine() # aa.read_net() aa.start_train(10000) # aa.save_net()
