Douzero_Resnet/evaluate.py

import os
import argparse

from douzero.evaluation.simulation import evaluate


def make_evaluate(args, vs_config):
    landlord_config = vs_config['landlord']
    farmer_config = vs_config['farmer']
    if landlord_config['prefix'] == 'random':
        args.landlord = 'random'
    else:
        args.landlord = '%s/%s_landlord_%i.ckpt' % (
            landlord_config['folder'], landlord_config['prefix'], landlord_config['frame']
        )
    if farmer_config['prefix'] == 'random':
        args.landlord_up = 'random'
        args.landlord_front = 'random'
        args.landlord_down = 'random'
    else:
        args.landlord_up = '%s/%s_landlord_up_%i.ckpt' % (
            farmer_config['folder'], farmer_config['prefix'], farmer_config['frame']
        )
        args.landlord_front = '%s/%s_landlord_front_%i.ckpt' % (
            farmer_config['folder'], farmer_config['prefix'], farmer_config['frame']
        )
        args.landlord_down = '%s/%s_landlord_down_%i.ckpt' % (
            farmer_config['folder'], farmer_config['prefix'], farmer_config['frame']
        )

    print('%s %i\tvs\t%s %i' % (
        landlord_config['prefix'], landlord_config['frame'], farmer_config['prefix'], farmer_config['frame']))

    evaluate(args.landlord,
             args.landlord_up,
             args.landlord_front,
             args.landlord_down,
             args.eval_data,
             args.num_workers,
             args.output,
             args.title)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        'Dou Dizhu Evaluation')
    parser.add_argument('--landlord', type=str,
                        default='baselines/douzero_12/landlord_weights_39762328900.ckpt')
    parser.add_argument('--landlord_up', type=str,
                        default='baselines/douzero_12/landlord_up_weights_39762328900.ckpt')
    parser.add_argument('--landlord_front', type=str,
                        default='baselines/douzero_12/landlord_front_weights_39762328900.ckpt')
    parser.add_argument('--landlord_down', type=str,
                        default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt')
    parser.add_argument('--eval_data', type=str,
                        default='eval_data_200.pkl')
    parser.add_argument('--num_workers', type=int, default=3)
    parser.add_argument('--gpu_device', type=str, default='0')
    parser.add_argument('--output', type=bool, default=True)
    parser.add_argument('--title', type=str, default='New')
    args = parser.parse_args()
    # args.output = True
    args.output = False
    if args.output:
        args.num_workers = 1

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device

    eval_list = [
        {
            'landlord': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 119822400},
            'farmer': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 119822400},
            'two_way': False
        },
        {
            'farmer': { 'folder': 'baselines', 'prefix': 'legacy_general', 'frame': 143539200},
            'landlord': { 'folder': 'baselines', 'prefix': 'resnet', 'frame': 119822400},
            'two_way': True
        },
        # {
        #     'landlord': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 11534400},
        #     'farmer': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 11534400},
        #     'two_way': False
        # },
        # {
        #     'landlord': {'folder': 'baselines', 'prefix': 'legacy_resnet', 'frame': 11754400},
        #     'farmer': {'folder': 'baselines', 'prefix': 'legacy_resnet', 'frame': 11754400},
        #     'two_way': False
        # },
        # {
        #     'landlord': {'folder': 'baselines', 'prefix': 'legacy_resnet', 'frame': 11754400},
        #     'farmer': {'folder': 'baselines', 'prefix': 'resnet', 'frame': 11534400},
        #     'two_way': True
        # },
    ]

    for vs in reversed(eval_list):
        make_evaluate(args, vs)
        if vs['two_way']:
            vs['landlord_bak'] = vs['landlord']
            vs['landlord'] = vs['farmer']
            vs['farmer'] = vs['landlord_bak']
            make_evaluate(args, vs)