From 364f88201480db44c635efabcd251b828a3b2bdb Mon Sep 17 00:00:00 2001 From: ZaneYork Date: Sun, 12 Dec 2021 16:49:54 +0800 Subject: [PATCH] =?UTF-8?q?eval=E5=8F=82=E6=95=B0=E5=B0=81=E8=A3=85?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- evaluate.py | 117 ++++++++++++++++++---------------------------------- 1 file changed, 39 insertions(+), 78 deletions(-) diff --git a/evaluate.py b/evaluate.py index 622a60c..08d4f3c 100644 --- a/evaluate.py +++ b/evaluate.py @@ -4,37 +4,32 @@ import argparse from douzero.evaluation.simulation import evaluate -def make_evaluate(args, t, frame, adp_frame, folder_a = 'baselines', folder_b = 'baselines', prefix_a = '', prefix_b = ''): - if t == 0: +def make_evaluate(args, vs_config): + landlord_config = vs_config['landlord'] + farmer_config = vs_config['farmer'] + if landlord_config['prefix'] == 'random': args.landlord = 'random' + else: + args.landlord = '%s/%s_landlord_%i.ckpt' % ( + landlord_config['folder'], landlord_config['prefix'], landlord_config['frame'] + ) + if farmer_config['prefix'] == 'random': args.landlord_up = 'random' args.landlord_front = 'random' args.landlord_down = 'random' - print('random vs random') - if t == 1: - args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_a, frame) - args.landlord_up = 'random' - args.landlord_front = 'random' - args.landlord_down = 'random' - print('%i vs random' % frame) - elif t == 2: - args.landlord = 'random' - args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_a, frame) - args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_a, frame) - args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_a, frame) - print('random vs %i' % frame) - elif t == 3: - args.landlord = '%s/%sresnet_landlord_%i.ckpt' % (folder_a, prefix_a, frame) - args.landlord_up = '%s/%sresnet_landlord_up_%i.ckpt' % (folder_b, prefix_b, adp_frame) - args.landlord_front = '%s/%sresnet_landlord_front_%i.ckpt' % (folder_b, prefix_b, adp_frame) - args.landlord_down = '%s/%sresnet_landlord_down_%i.ckpt' % (folder_b, prefix_b, adp_frame) - print('%i vs %i' % (frame, adp_frame)) - elif t == 4: - args.landlord = '%s/%sresnet_landlord_%i.ckpt' % (folder_b, prefix_b, adp_frame) - args.landlord_up = '%s/%sresnet_landlord_up_%i.ckpt' % (folder_a, prefix_a, frame) - args.landlord_front = '%s/%sresnet_landlord_front_%i.ckpt' % (folder_a, prefix_a, frame) - args.landlord_down = '%s/%sresnet_landlord_down_%i.ckpt' % (folder_a, prefix_a, frame) - print('%i vs %i' % (adp_frame, frame)) + else: + args.landlord_up = '%s/%s_landlord_up_%i.ckpt' % ( + farmer_config['folder'], farmer_config['prefix'], farmer_config['frame'] + ) + args.landlord_front = '%s/%s_landlord_front_%i.ckpt' % ( + farmer_config['folder'], farmer_config['prefix'], farmer_config['frame'] + ) + args.landlord_down = '%s/%s_landlord_down_%i.ckpt' % ( + farmer_config['folder'], farmer_config['prefix'], farmer_config['frame'] + ) + + print('%s %i\tvs\t%s %i' % ( + landlord_config['prefix'], landlord_config['frame'], farmer_config['prefix'], farmer_config['frame'])) evaluate(args.landlord, args.landlord_up, @@ -60,7 +55,7 @@ if __name__ == '__main__': default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt') parser.add_argument('--eval_data', type=str, default='eval_data_200.pkl') - parser.add_argument('--num_workers', type=int, default=2) + parser.add_argument('--num_workers', type=int, default=3) parser.add_argument('--gpu_device', type=str, default='0') parser.add_argument('--output', type=bool, default=True) parser.add_argument('--bid', type=bool, default=True) @@ -72,60 +67,26 @@ if __name__ == '__main__': if args.output or args.bid: args.num_workers = 1 - t = 3 - frame = 13252000 - adp_frame = 8697600 - os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device - # eval_list = [ - # [8697600, None], - # [8697600, 8697600], - # [11402400, None], - # [11402400, 8697600], - # [11402400, 11402400], - # [13252000, None], - # [13252000, 8697600], - # [13252000, 11402400], - # [13252000, 13252000], - # [15096800, None], - # [15096800, 8697600], - # [15096800, 11402400], - # [15096800, 13252000], - # [15096800, 15096800], - # [25397600, None], - # [25397600, 8697600], - # [25397600, 11402400], - # [25397600, 13252000], - # [25397600, 15096800], - # [25397600, 25397600], - # ] - eval_list = [ - # [4968800, 8697600, 'baselines', 'baselines2'], - # [4968800, 4968800, 'baselines', 'baselines'], - # [14102400, 4968800, 'baselines', 'baselines'], - # [14102400, 13252000, 'baselines', 'baselines2'], - # [14102400, 15096800, 'baselines', 'baselines2'], - # [34828000, 40132800, 'baselines2', 'baselines2'], - # [14102400, None, 'baselines', 'baselines'], - [19918400, 19918400, 'baselines', 'baselines', 'legacy_', 'legacy_'], - [9161600, 19918400, 'baselines', 'baselines', '', 'legacy_'], + # { + # 'landlord': { 'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 48545600}, + # 'farmer': { 'folder': 'baselines', 'prefix': 'resnet', 'frame': 11534400}, + # 'two_way': True + # }, + { + 'landlord': {'folder': 'baselines', 'prefix': 'legacy_resnet', 'frame': 11754400}, + 'farmer': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 11534400}, + 'two_way': True + } ] for vs in reversed(eval_list): - frame = vs[0] - adp_frame = vs[1] - folder_a = vs[2] - folder_b = vs[3] - if adp_frame is None: - if frame is None: - make_evaluate(args, 0, None, None, folder_a , folder_b, vs[4], vs[5]) - else: - make_evaluate(args, 1, frame, None, folder_a , folder_b, vs[4], vs[5]) - make_evaluate(args, 2, frame, None, folder_a , folder_b, vs[4], vs[5]) - else: - make_evaluate(args, 3, frame, adp_frame, folder_a , folder_b, vs[4], vs[5]) - if frame != adp_frame: - make_evaluate(args, 4, frame, adp_frame, folder_a, folder_b, vs[4], vs[5]) + make_evaluate(args, vs) + if vs['two_way']: + vs['landlord_bak'] = vs['landlord'] + vs['landlord'] = vs['farmer'] + vs['farmer'] = vs['landlord_bak'] + make_evaluate(args, vs)