调整评估相关代码
This commit is contained in:
zhiyang7
parent
ee409846f3
commit
a755ffe719
|
|
@ -1 +1,5 @@
|
|||
*.pyc
|
||||
*.pyc
|
||||
baselines*/
|
||||
douzero_checkpoints/
|
||||
.vscode/
|
||||
*.pkl
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from douzero.env.game import GameEnv
|
|||
import torch
|
||||
import numpy as np
|
||||
import BidModel
|
||||
import psutil
|
||||
|
||||
def load_card_play_models(card_play_model_path_dict):
|
||||
players = {}
|
||||
|
|
@ -102,7 +103,7 @@ def mp_simulate(card_play_data_list, card_play_model_path_dict, q, output, bid_o
|
|||
else:
|
||||
print("".join([EnvCard2RealCard[c] for c in action]), end=end)
|
||||
count+=1
|
||||
if idx % 10 == 0 and not bid_output:
|
||||
if output and idx % 10 == 0 and not bid_output:
|
||||
print("\nindex", idx)
|
||||
# print("End -------")
|
||||
env.reset()
|
||||
|
|
@ -149,8 +150,14 @@ def evaluate(landlord, landlord_up, landlord_down, eval_data, num_workers, outpu
|
|||
target=mp_simulate,
|
||||
args=(card_paly_data, card_play_model_path_dict, q, output, output_bid, title))
|
||||
p.start()
|
||||
|
||||
processes.append(p)
|
||||
|
||||
parent = psutil.Process()
|
||||
parent.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS)
|
||||
for child in parent.children():
|
||||
child.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS)
|
||||
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
|
|
@ -165,4 +172,4 @@ def evaluate(landlord, landlord_up, landlord_down, eval_data, num_workers, outpu
|
|||
print('WP results:')
|
||||
print('landlord : Farmers - {} : {}'.format(num_landlord_wins / num_total_wins, num_farmer_wins / num_total_wins))
|
||||
print('ADP results:')
|
||||
print('landlord : Farmers - {} : {}'.format(num_landlord_scores / num_total_wins, 2 * num_farmer_scores / num_total_wins))
|
||||
print('landlord : Farmers - {} : {}'.format(num_landlord_scores / num_total_wins, 3 * num_farmer_scores / num_total_wins))
|
||||
|
|
|
|||
174
evaluate.py
174
evaluate.py
|
|
@ -1,88 +1,34 @@
|
|||
import os
|
||||
import os
|
||||
import argparse
|
||||
|
||||
from douzero.evaluation.simulation import evaluate
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(
|
||||
'Dou Dizhu Evaluation')
|
||||
parser.add_argument('--landlord', type=str,
|
||||
default='baselines/douzero_12/landlord_weights_39762328900.ckpt')
|
||||
parser.add_argument('--landlord_up', type=str,
|
||||
default='baselines/douzero_12/landlord_up_weights_39762328900.ckpt')
|
||||
parser.add_argument('--landlord_front', type=str,
|
||||
default='baselines/douzero_12/landlord_front_weights_39762328900.ckpt')
|
||||
parser.add_argument('--landlord_down', type=str,
|
||||
default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt')
|
||||
parser.add_argument('--eval_data', type=str,
|
||||
default='eval_data_1000.pkl')
|
||||
parser.add_argument('--num_workers', type=int, default=5)
|
||||
parser.add_argument('--gpu_device', type=str, default='0')
|
||||
parser.add_argument('--output', type=bool, default=True)
|
||||
parser.add_argument('--bid', type=bool, default=True)
|
||||
parser.add_argument('--title', type=str, default='New')
|
||||
args = parser.parse_args()
|
||||
args.output = True
|
||||
args.bid = False
|
||||
if args.output or args.bid:
|
||||
args.num_workers = 1
|
||||
t = 3
|
||||
frame = 64000
|
||||
adp_frame = 2511184300
|
||||
# args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
|
||||
args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame
|
||||
args.landlord_down = 'baselines/resnet_landlord_%i.ckpt' % frame
|
||||
args.landlord = 'baselines/douzero_ADP/landlord.ckpt'
|
||||
# args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt'
|
||||
# args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt'
|
||||
|
||||
def make_evaluate(args, t, frame, adp_frame, folder_a = 'baselines', folder_b = 'baselines'):
|
||||
if t == 1:
|
||||
args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
|
||||
args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt'
|
||||
args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt'
|
||||
args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_a, frame)
|
||||
args.landlord_up = 'random'
|
||||
args.landlord_front = 'random'
|
||||
args.landlord_down = 'random'
|
||||
print('%i vs random' % frame)
|
||||
elif t == 2:
|
||||
args.landlord = 'baselines/douzero_ADP/landlord.ckpt'
|
||||
args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame
|
||||
args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame
|
||||
args.landlord = 'random'
|
||||
args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_a, frame)
|
||||
args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_a, frame)
|
||||
args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_a, frame)
|
||||
print('random vs %i' % frame)
|
||||
elif t == 3:
|
||||
args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
|
||||
args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame
|
||||
args.landlord_front = 'baselines/resnet_landlord_front_%i.ckpt' % frame
|
||||
args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame
|
||||
args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_a, frame)
|
||||
args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_b, adp_frame)
|
||||
args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_b, adp_frame)
|
||||
args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_b, adp_frame)
|
||||
print('%i vs %i' % (frame, adp_frame))
|
||||
elif t == 4:
|
||||
args.landlord = 'baselines/douzero_ADP/landlord.ckpt'
|
||||
args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt'
|
||||
args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt'
|
||||
elif t == 5:
|
||||
args.landlord = 'baselines/douzero_WP/landlord.ckpt'
|
||||
args.landlord_up = 'baselines/douzero_WP/landlord_up.ckpt'
|
||||
args.landlord_down = 'baselines/douzero_WP/landlord_down.ckpt'
|
||||
elif t == 6:
|
||||
args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
|
||||
args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame
|
||||
args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame
|
||||
elif t == 7:
|
||||
args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame
|
||||
args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame
|
||||
args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame
|
||||
elif t == 8:
|
||||
args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame
|
||||
args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame
|
||||
args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame
|
||||
elif t == 9:
|
||||
args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
|
||||
args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % adp_frame
|
||||
args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % adp_frame
|
||||
elif t == 10:
|
||||
# landlord_down_weights_10777798400
|
||||
args.landlord = 'baselines/douzero_ADP/landlord.ckpt'
|
||||
args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame
|
||||
args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame
|
||||
elif t == 11:
|
||||
args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame
|
||||
args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt'
|
||||
args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt'
|
||||
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device
|
||||
args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_b, adp_frame)
|
||||
args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_a, frame)
|
||||
args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_a, frame)
|
||||
args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_a, frame)
|
||||
print('%i vs %i' % (adp_frame, frame))
|
||||
|
||||
evaluate(args.landlord,
|
||||
args.landlord_up,
|
||||
|
|
@ -92,3 +38,77 @@ if __name__ == '__main__':
|
|||
args.output,
|
||||
args.bid,
|
||||
args.title)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(
|
||||
'Dou Dizhu Evaluation')
|
||||
parser.add_argument('--landlord', type=str,
|
||||
default='baselines/douzero_12/landlord_weights_39762328900.ckpt')
|
||||
parser.add_argument('--landlord_up', type=str,
|
||||
default='baselines/douzero_12/landlord_up_weights_39762328900.ckpt')
|
||||
parser.add_argument('--landlord_front', type=str,
|
||||
default='baselines/douzero_12/landlord_front_weights_39762328900.ckpt')
|
||||
parser.add_argument('--landlord_down', type=str,
|
||||
default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt')
|
||||
parser.add_argument('--eval_data', type=str,
|
||||
default='eval_data_200.pkl')
|
||||
parser.add_argument('--num_workers', type=int, default=5)
|
||||
parser.add_argument('--gpu_device', type=str, default='0')
|
||||
parser.add_argument('--output', type=bool, default=True)
|
||||
parser.add_argument('--bid', type=bool, default=True)
|
||||
parser.add_argument('--title', type=str, default='New')
|
||||
args = parser.parse_args()
|
||||
# args.output = True
|
||||
args.output = False
|
||||
args.bid = False
|
||||
if args.output or args.bid:
|
||||
args.num_workers = 1
|
||||
|
||||
t = 3
|
||||
frame = 13252000
|
||||
adp_frame = 8697600
|
||||
|
||||
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device
|
||||
|
||||
# eval_list = [
|
||||
# [8697600, None],
|
||||
# [8697600, 8697600],
|
||||
# [11402400, None],
|
||||
# [11402400, 8697600],
|
||||
# [11402400, 11402400],
|
||||
# [13252000, None],
|
||||
# [13252000, 8697600],
|
||||
# [13252000, 11402400],
|
||||
# [13252000, 13252000],
|
||||
# [15096800, None],
|
||||
# [15096800, 8697600],
|
||||
# [15096800, 11402400],
|
||||
# [15096800, 13252000],
|
||||
# [15096800, 15096800],
|
||||
# [25397600, None],
|
||||
# [25397600, 8697600],
|
||||
# [25397600, 11402400],
|
||||
# [25397600, 13252000],
|
||||
# [25397600, 15096800],
|
||||
# [25397600, 25397600],
|
||||
# ]
|
||||
|
||||
eval_list = [
|
||||
[4968800, 8697600, 'baselines', 'baselines2'],
|
||||
[4968800, 4968800, 'baselines', 'baselines'],
|
||||
]
|
||||
|
||||
for vs in reversed(eval_list):
|
||||
frame = vs[0]
|
||||
adp_frame = vs[1]
|
||||
folder_a = vs[2]
|
||||
folder_b = vs[3]
|
||||
if adp_frame is None:
|
||||
make_evaluate(args, 1, frame, None)
|
||||
make_evaluate(args, 2, frame, None)
|
||||
else:
|
||||
make_evaluate(args, 3, frame, adp_frame, folder_a , folder_b)
|
||||
if frame != adp_frame:
|
||||
make_evaluate(args, 4, frame, adp_frame, folder_a, folder_b)
|
||||
|
|
|
|||
|
|
@ -2,3 +2,4 @@ torch>=1.6.0
|
|||
GitPython
|
||||
gitdb2
|
||||
rlcard
|
||||
psutil
|
||||
|
|
|
|||
Loading…
Reference in New Issue