前言
本来想做强化学习小车，但是技术栈似乎飞了，所以还是一步一步来嘛。
Part 1 Code
import copy
import random
import json
import matplotlib.pyplot as plt
class OoxxMachine:
def __init__(self):
self.race = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]
# 用于表示棋盘 0代表没下过 1 A玩家 2 B玩家
self.flag = "in_race"
# all situation is "in_race" or "a_win" or "b_win" or "all_lose"
# ----------------------------------------------------------------
self.learn_rate = 0.1
self.rand_poss = 0.05
self.net_values = {}
self.default_value = 0.5
def update_win(self):
"""
[[2,1,1][1,1,0][1,2,0]]
be like:
0 1 2
-------------
0 | x | o | o |
1 | x | x | |
2 | x | o | |
-------------
"""
if self.race[0][0] == self.race[0][1] == self.race[0][2]:
if self.race[0][0] == 1:
self.flag = "a_win"
elif self.race[0][0] == 2:
self.flag = "b_win"
else:
pass
if self.race[1][0] == self.race[1][1] == self.race[1][2]:
if self.race[1][0] == 1:
self.flag = "a_win"
elif self.race[1][0] == 2:
self.flag = "b_win"
else:
pass
if self.race[2][0] == self.race[2][1] == self.race[2][2]:
if self.race[2][0] == 1:
self.flag = "a_win"
elif self.race[2][0] == 2:
self.flag = "b_win"
else:
pass
if self.race[0][0] == self.race[1][0] == self.race[2][0]:
if self.race[0][0] == 1:
self.flag = "a_win"
elif self.race[0][0] == 2:
self.flag = "b_win"
else:
pass
if self.race[0][1] == self.race[1][1] == self.race[2][1]:
if self.race[0][1] == 1:
self.flag = "a_win"
elif self.race[0][1] == 2:
self.flag = "b_win"
else:
pass
if self.race[0][2] == self.race[1][2] == self.race[2][2]:
if self.race[0][2] == 1:
self.flag = "a_win"
elif self.race[0][2] == 2:
self.flag = "b_win"
else:
pass
if self.race[0][0] == self.race[1][1] == self.race[2][2]:
if self.race[2][2] == 1:
self.flag = "a_win"
elif self.race[2][2] == 2:
self.flag = "b_win"
else:
pass
if self.race[0][2] == self.race[1][1] == self.race[2][0]:
if self.race[0][2] == 1:
self.flag = "a_win"
elif self.race[0][2] == 2:
self.flag = "b_win"
else:
pass
all_chess = 0
for i in range(0, 3):
for j in range(0, 3):
if self.race[i][j] != 0:
all_chess += 1
# print(all_chess)
if all_chess == 8 and self.flag == "in_race":
self.flag = "all_lose"
return False
def reset(self):
self.race = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]
self.flag = "in_race"
def do_once(self, racer: "str == a or b", location: list) -> str:
could_do = True
for i in range(0, 3):
if 0 in self.race[i]:
could_do = True
else:
pass
if not could_do:
self.flag = 'all_lose'
return "fin"
if racer == "a":
if self.race[location[0]][location[1]] == 0:
self.race[location[0]][location[1]] = 1
else:
raise ValueError("this location has been used")
if racer == "b":
if self.race[location[0]][location[1]] == 0:
self.race[location[0]][location[1]] = 2
else:
raise ValueError("this location has been used")
return "fin"
# 我对强化学习的理解还不够透彻
def refresh_net(self, now_race: list, next_race: list) -> bool:
# 传参： 赛场情况 需要更新的价值（在此赛场情况之前的价值） （本赛场）是否获胜
hash_value: int = hash(str(now_race))
hash_next: int = hash(str(next_race))
"""
# 如果给下死了就给Value 置于 0
if self.flag == 'b_win' or self.flag == 'all_lose':
self.net_values[hash_value] = 0
return False
# 更新下一次预期之获胜情况
copy_race = self.race
self.race = next_race
self.update_win()
if self.flag == 'a_win':
next_value = 1
self.net_values[hash_value] = 1
elif self.flag == 'b_win' or "all_lose":
next_value = 0
self.net_values[hash_value] = 0
self.race = copy_race
self.update_win()
"""
next_value = self.net_values[hash_next]
if hash_value not in self.net_values:
self.net_values[hash_value] = self.default_value
value = self.default_value
else:
value = self.net_values[hash_value]
value = value + (next_value - value) * self.learn_rate
self.net_values[hash_value] = value
return True
def save_net(self, filename='net.json'):
with open(filename, 'w') as file:
json.dump(self.net_values, file)
print(f"Net values saved to {filename}.")
def read_net(self, filename='net.json'):
with open(filename, 'r') as file:
self.net_values = json.load(file)
print(f"Net values loaded from {filename}.")
def random_player(self, player: str):
possible_location = []
race_copy = self.race
for i in range(0, 3):
for j in range(0, 3):
if race_copy[i][j] == 0:
possible_location.append([i, j])
if not possible_location:
self.flag = "all_lose"
return False
location = random.choice(possible_location)
self.do_once(player, location)
def start_train(self, epoch: int = 1000) -> bool:
self.reset()
a_win_times = 1
b_win_times = 1
win_rate = []
for times in range(1, epoch):
win_rate.append(a_win_times / (a_win_times + b_win_times))
plt.plot(win_rate)
# print(self.race)
if self.flag == "a_win":
a_win_times += 1
elif self.flag == "b_win":
b_win_times += 1
# print(times)
# print(self.net_values)
self.reset()
if random.randint(0, 1):
while self.flag == "in_race":
self.update_win()
if random.random() >= self.rand_poss:
next_races = []
for i in range(0, 3):
for j in range(0, 3):
if self.race[i][j] == 0:
races_copy = copy.deepcopy(self.race)
races_copy[i][j] = 1
next_races.append(races_copy)
else:
pass
values = []
for next_race in next_races:
copy_race = copy.deepcopy(self.race)
self.race = copy.deepcopy(next_race)
self.update_win()
if self.flag == 'a_win':
self.net_values[hash(str(next_race))] = 1
elif self.flag == 'b_win' or "all_lose":
self.net_values[hash(str(next_race))] = 0
self.race = copy.deepcopy(copy_race)
self.update_win()
next_hash = hash(str(next_race))
if next_hash not in self.net_values:
self.net_values[next_hash] = self.default_value
values.append(self.default_value)
else:
values.append(self.net_values[next_hash])
max_value = max(values)
max_indices = [index for index, value in enumerate(values) if value == max_value]
random_max_index = random.choice(max_indices)
next_race = next_races[random_max_index]
# print(next_races)
self.refresh_net(self.race, next_race)
self.race = next_race
# print(self.race)
else:
# print("random")
if self.random_player("a"):
pass
else:
break
self.random_player("b")
else:
while self.flag == "in_race":
self.update_win()
self.random_player("b")
if random.random() >= self.rand_poss:
next_races = []
for i in range(0, 3):
for j in range(0, 3):
if self.race[i][j] == 0:
races_copy = copy.deepcopy(self.race)
races_copy[i][j] = 1
next_races.append(races_copy)
else:
pass
values = []
for next_race in next_races:
copy_race = copy.deepcopy(self.race)
self.race = copy.deepcopy(next_race)
self.update_win()
if self.flag == 'a_win':
self.net_values[hash(str(next_race))] = 1
elif self.flag == 'b_win' or "all_lose":
self.net_values[hash(str(next_race))] = 0
self.race = copy.deepcopy(copy_race)
self.update_win()
next_hash = hash(str(next_race))
if next_hash not in self.net_values:
self.net_values[next_hash] = self.default_value
values.append(self.default_value)
else:
values.append(self.net_values[next_hash])
max_value = max(values)
max_indices = [index for index, value in enumerate(values) if value == max_value]
random_max_index = random.choice(max_indices)
next_race = next_races[random_max_index]
# print(next_races)
self.refresh_net(self.race, next_race)
self.race = next_race
# print(self.race)
else:
# print("random")
if self.random_player("a"):
pass
else:
break
# do the race once at here
print(f"a wins {str(a_win_times)} b wins {str(b_win_times)}")
print(f"A的胜率是{str(a_win_times / (a_win_times + b_win_times))}")
plt.show()
return True
if __name__ == "__main__":
aa = OoxxMachine()
# aa.read_net()
aa.start_train(10000)
# aa.save_net()