From a755ffe7190e1d16bb485004703c084c9fc9edba Mon Sep 17 00:00:00 2001 From: zhiyang7 Date: Wed, 8 Dec 2021 17:16:27 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=AF=84=E4=BC=B0=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 6 +- douzero/evaluation/simulation.py | 11 +- evaluate.py | 174 +++++++++++++++++-------------- requirements.txt | 1 + 4 files changed, 112 insertions(+), 80 deletions(-) diff --git a/.gitignore b/.gitignore index 7e99e36..0f34f11 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -*.pyc \ No newline at end of file +*.pyc +baselines*/ +douzero_checkpoints/ +.vscode/ +*.pkl diff --git a/douzero/evaluation/simulation.py b/douzero/evaluation/simulation.py index 4ded8f5..2e66a97 100644 --- a/douzero/evaluation/simulation.py +++ b/douzero/evaluation/simulation.py @@ -6,6 +6,7 @@ from douzero.env.game import GameEnv import torch import numpy as np import BidModel +import psutil def load_card_play_models(card_play_model_path_dict): players = {} @@ -102,7 +103,7 @@ def mp_simulate(card_play_data_list, card_play_model_path_dict, q, output, bid_o else: print("".join([EnvCard2RealCard[c] for c in action]), end=end) count+=1 - if idx % 10 == 0 and not bid_output: + if output and idx % 10 == 0 and not bid_output: print("\nindex", idx) # print("End -------") env.reset() @@ -149,8 +150,14 @@ def evaluate(landlord, landlord_up, landlord_down, eval_data, num_workers, outpu target=mp_simulate, args=(card_paly_data, card_play_model_path_dict, q, output, output_bid, title)) p.start() + processes.append(p) + parent = psutil.Process() + parent.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) + for child in parent.children(): + child.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) + for p in processes: p.join() @@ -165,4 +172,4 @@ def evaluate(landlord, landlord_up, landlord_down, eval_data, num_workers, outpu print('WP results:') print('landlord : Farmers - {} : {}'.format(num_landlord_wins / num_total_wins, num_farmer_wins / num_total_wins)) print('ADP results:') - print('landlord : Farmers - {} : {}'.format(num_landlord_scores / num_total_wins, 2 * num_farmer_scores / num_total_wins)) + print('landlord : Farmers - {} : {}'.format(num_landlord_scores / num_total_wins, 3 * num_farmer_scores / num_total_wins)) diff --git a/evaluate.py b/evaluate.py index 45ea06e..07e7cb9 100644 --- a/evaluate.py +++ b/evaluate.py @@ -1,88 +1,34 @@ -import os +import os import argparse from douzero.evaluation.simulation import evaluate -if __name__ == '__main__': - parser = argparse.ArgumentParser( - 'Dou Dizhu Evaluation') - parser.add_argument('--landlord', type=str, - default='baselines/douzero_12/landlord_weights_39762328900.ckpt') - parser.add_argument('--landlord_up', type=str, - default='baselines/douzero_12/landlord_up_weights_39762328900.ckpt') - parser.add_argument('--landlord_front', type=str, - default='baselines/douzero_12/landlord_front_weights_39762328900.ckpt') - parser.add_argument('--landlord_down', type=str, - default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt') - parser.add_argument('--eval_data', type=str, - default='eval_data_1000.pkl') - parser.add_argument('--num_workers', type=int, default=5) - parser.add_argument('--gpu_device', type=str, default='0') - parser.add_argument('--output', type=bool, default=True) - parser.add_argument('--bid', type=bool, default=True) - parser.add_argument('--title', type=str, default='New') - args = parser.parse_args() - args.output = True - args.bid = False - if args.output or args.bid: - args.num_workers = 1 - t = 3 - frame = 64000 - adp_frame = 2511184300 - # args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame - args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame - args.landlord_down = 'baselines/resnet_landlord_%i.ckpt' % frame - args.landlord = 'baselines/douzero_ADP/landlord.ckpt' - # args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt' - # args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt' + +def make_evaluate(args, t, frame, adp_frame, folder_a = 'baselines', folder_b = 'baselines'): if t == 1: - args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame - args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt' - args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt' + args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_a, frame) + args.landlord_up = 'random' + args.landlord_front = 'random' + args.landlord_down = 'random' + print('%i vs random' % frame) elif t == 2: - args.landlord = 'baselines/douzero_ADP/landlord.ckpt' - args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame - args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame + args.landlord = 'random' + args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_a, frame) + args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_a, frame) + args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_a, frame) + print('random vs %i' % frame) elif t == 3: - args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame - args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame - args.landlord_front = 'baselines/resnet_landlord_front_%i.ckpt' % frame - args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame + args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_a, frame) + args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_b, adp_frame) + args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_b, adp_frame) + args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_b, adp_frame) + print('%i vs %i' % (frame, adp_frame)) elif t == 4: - args.landlord = 'baselines/douzero_ADP/landlord.ckpt' - args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt' - args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt' - elif t == 5: - args.landlord = 'baselines/douzero_WP/landlord.ckpt' - args.landlord_up = 'baselines/douzero_WP/landlord_up.ckpt' - args.landlord_down = 'baselines/douzero_WP/landlord_down.ckpt' - elif t == 6: - args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame - args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame - args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame - elif t == 7: - args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame - args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame - args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame - elif t == 8: - args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame - args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame - args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame - elif t == 9: - args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame - args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % adp_frame - args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % adp_frame - elif t == 10: - # landlord_down_weights_10777798400 - args.landlord = 'baselines/douzero_ADP/landlord.ckpt' - args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame - args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame - elif t == 11: - args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame - args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt' - args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt' - os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' - os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device + args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_b, adp_frame) + args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_a, frame) + args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_a, frame) + args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_a, frame) + print('%i vs %i' % (adp_frame, frame)) evaluate(args.landlord, args.landlord_up, @@ -92,3 +38,77 @@ if __name__ == '__main__': args.output, args.bid, args.title) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + 'Dou Dizhu Evaluation') + parser.add_argument('--landlord', type=str, + default='baselines/douzero_12/landlord_weights_39762328900.ckpt') + parser.add_argument('--landlord_up', type=str, + default='baselines/douzero_12/landlord_up_weights_39762328900.ckpt') + parser.add_argument('--landlord_front', type=str, + default='baselines/douzero_12/landlord_front_weights_39762328900.ckpt') + parser.add_argument('--landlord_down', type=str, + default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt') + parser.add_argument('--eval_data', type=str, + default='eval_data_200.pkl') + parser.add_argument('--num_workers', type=int, default=5) + parser.add_argument('--gpu_device', type=str, default='0') + parser.add_argument('--output', type=bool, default=True) + parser.add_argument('--bid', type=bool, default=True) + parser.add_argument('--title', type=str, default='New') + args = parser.parse_args() + # args.output = True + args.output = False + args.bid = False + if args.output or args.bid: + args.num_workers = 1 + + t = 3 + frame = 13252000 + adp_frame = 8697600 + + os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device + + # eval_list = [ + # [8697600, None], + # [8697600, 8697600], + # [11402400, None], + # [11402400, 8697600], + # [11402400, 11402400], + # [13252000, None], + # [13252000, 8697600], + # [13252000, 11402400], + # [13252000, 13252000], + # [15096800, None], + # [15096800, 8697600], + # [15096800, 11402400], + # [15096800, 13252000], + # [15096800, 15096800], + # [25397600, None], + # [25397600, 8697600], + # [25397600, 11402400], + # [25397600, 13252000], + # [25397600, 15096800], + # [25397600, 25397600], + # ] + + eval_list = [ + [4968800, 8697600, 'baselines', 'baselines2'], + [4968800, 4968800, 'baselines', 'baselines'], + ] + + for vs in reversed(eval_list): + frame = vs[0] + adp_frame = vs[1] + folder_a = vs[2] + folder_b = vs[3] + if adp_frame is None: + make_evaluate(args, 1, frame, None) + make_evaluate(args, 2, frame, None) + else: + make_evaluate(args, 3, frame, adp_frame, folder_a , folder_b) + if frame != adp_frame: + make_evaluate(args, 4, frame, adp_frame, folder_a, folder_b) diff --git a/requirements.txt b/requirements.txt index 5cd7b9e..0e25259 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ torch>=1.6.0 GitPython gitdb2 rlcard +psutil