diff --git a/evaluate.py b/evaluate.py index 83c387a..959df46 100644 --- a/evaluate.py +++ b/evaluate.py @@ -1,5 +1,6 @@ import os import argparse +from pickle import FALSE from douzero.evaluation.simulation import evaluate @@ -67,35 +68,44 @@ if __name__ == '__main__': os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device + baselines = [ + {'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 736107200}, + {'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 479412800}, + {'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 262529600}, + {'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 143539200}, + ] + evaluations = [ + {'folder': 'baselines', 'prefix': 'lite_general', 'frame': 299457600}, + {'folder': 'baselines', 'prefix': 'lite_resnet', 'frame': 124064400}, + ] + eval_list = [ { - 'landlord': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 119822400}, - 'farmer': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 119822400}, - 'two_way': False - }, - { - 'farmer': { 'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 143539200}, - 'landlord': { 'folder': 'baselines', 'prefix': 'resnet', 'frame': 119822400}, + 'farmer': baseline, + 'landlord': evaluation, 'two_way': True - }, - # { - # 'landlord': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 11534400}, - # 'farmer': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 11534400}, - # 'two_way': False - # }, - # { - # 'landlord': {'folder': 'baselines', 'prefix': 'legacy_resnet', 'frame': 11754400}, - # 'farmer': {'folder': 'baselines', 'prefix': 'legacy_resnet', 'frame': 11754400}, - # 'two_way': False - # }, - # { - # 'landlord': {'folder': 'baselines', 'prefix': 'legacy_resnet', 'frame': 11754400}, - # 'farmer': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 11534400}, - # 'two_way': True - # }, + } + for evaluation in evaluations for baseline in baselines ] - for vs in reversed(eval_list): + eval_list.append([ + { + 'farmer': evaluation, + 'landlord': evaluation, + 'two_way': False + } + for evaluation in evaluations + ]) + + # eval_list = [ + # { + # 'farmer': { 'folder': 'baselines', 'prefix': 'lite_general', 'frame': 221553600}, + # 'landlord': { 'folder': 'baselines', 'prefix': 'lite_resnet', 'frame': 102494400}, + # 'two_way': False + # }, + # ] + + for vs in eval_list: make_evaluate(args, vs) if vs['two_way']: vs['landlord_bak'] = vs['landlord']