import os import argparse from pickle import FALSE import csv from douzero.evaluation.simulation import evaluate def make_evaluate(args, vs_config): landlord_config = vs_config['landlord'] farmer_config = vs_config['farmer'] if landlord_config['prefix'] == 'random': args.landlord = 'random' else: args.landlord = '%s/%s_landlord_%i.ckpt' % ( landlord_config['folder'], landlord_config['prefix'], landlord_config['frame'] ) if farmer_config['prefix'] == 'random': args.landlord_up = 'random' args.landlord_front = 'random' args.landlord_down = 'random' else: args.landlord_up = '%s/%s_landlord_up_%i.ckpt' % ( farmer_config['folder'], farmer_config['prefix'], farmer_config['frame'] ) args.landlord_front = '%s/%s_landlord_front_%i.ckpt' % ( farmer_config['folder'], farmer_config['prefix'], farmer_config['frame'] ) args.landlord_down = '%s/%s_landlord_down_%i.ckpt' % ( farmer_config['folder'], farmer_config['prefix'], farmer_config['frame'] ) print('%s %i\tvs\t%s %i' % ( landlord_config['prefix'], landlord_config['frame'], farmer_config['prefix'], farmer_config['frame'])) landlord_wp, farmer_wp, landlord_adp, farmer_adp = evaluate(args.landlord, args.landlord_up, args.landlord_front, args.landlord_down, args.eval_data, args.num_workers, args.output, args.title) with open('result.csv', 'w') as csvfile: writer = csv.writer(csvfile) writer.writerow([ landlord_wp, farmer_wp, landlord_adp, farmer_adp ]) if __name__ == '__main__': parser = argparse.ArgumentParser( 'Dou Dizhu Evaluation') parser.add_argument('--landlord', type=str, default='baselines/lite_uni_5001600.ckpt') parser.add_argument('--landlord_up', type=str, default='baselines/lite_uni_5001600.ckpt') parser.add_argument('--landlord_front', type=str, default='baselines/lite_uni_5001600.ckpt') parser.add_argument('--landlord_down', type=str, default='baselines/lite_uni_5001600.ckpt') parser.add_argument('--eval_data', type=str, default='eval_data_200_r.pkl') parser.add_argument('--num_workers', type=int, default=3) parser.add_argument('--gpu_device', type=str, default='0') parser.add_argument('--output', type=bool, default=True) parser.add_argument('--title', type=str, default='New') args = parser.parse_args() # args.output = True args.output = False if args.output: args.num_workers = 1 os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device landlord_wp, farmer_wp, landlord_adp, farmer_adp = evaluate(args.landlord, args.landlord_up, args.landlord_front, args.landlord_down, args.eval_data, args.num_workers, args.output, args.title) print(landlord_wp, farmer_wp, landlord_adp, farmer_adp) os._exit(0) baselines = [ {'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 736107200}, {'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 479412800}, {'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 262529600}, {'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 143539200}, ] evaluations = [ {'folder': 'baselines', 'prefix': 'lite_general', 'frame': 299457600}, {'folder': 'baselines', 'prefix': 'lite_resnet', 'frame': 124064400}, ] eval_list = [ { 'farmer': baseline, 'landlord': evaluation, 'two_way': True } for evaluation in evaluations for baseline in baselines ] eval_list.extend([ { 'farmer': evaluation, 'landlord': evaluation, 'two_way': False } for evaluation in evaluations ]) # eval_list = [ # { # 'farmer': { 'folder': 'baselines', 'prefix': 'lite_general', 'frame': 221553600}, # 'landlord': { 'folder': 'baselines', 'prefix': 'lite_resnet', 'frame': 102494400}, # 'two_way': False # }, # ] for vs in eval_list: make_evaluate(args, vs) if vs['two_way']: vs['landlord_bak'] = vs['landlord'] vs['landlord'] = vs['farmer'] vs['farmer'] = vs['landlord_bak'] make_evaluate(args, vs)