Douzero_Resnet/evaluate.py

import os
import argparse

from douzero.evaluation.simulation import evaluate


def make_evaluate(args, t, frame, adp_frame, folder_a = 'baselines', folder_b = 'baselines', prefix_a = '', prefix_b = ''):
    if t == 0:
        args.landlord = 'random'
        args.landlord_up = 'random'
        args.landlord_front = 'random'
        args.landlord_down = 'random'
        print('random vs random')
    if t == 1:
        args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_a, frame)
        args.landlord_up = 'random'
        args.landlord_front = 'random'
        args.landlord_down = 'random'
        print('%i vs random' % frame)
    elif t == 2:
        args.landlord = 'random'
        args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_a, frame)
        args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_a, frame)
        args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_a, frame)
        print('random vs %i' % frame)
    elif t == 3:
        args.landlord = '%s/%sresnet_landlord_%i.ckpt' % (folder_a, prefix_a, frame)
        args.landlord_up = '%s/%sresnet_landlord_up_%i.ckpt' % (folder_b, prefix_b, adp_frame)
        args.landlord_front = '%s/%sresnet_landlord_front_%i.ckpt' % (folder_b, prefix_b, adp_frame)
        args.landlord_down = '%s/%sresnet_landlord_down_%i.ckpt' % (folder_b, prefix_b, adp_frame)
        print('%i vs %i' % (frame, adp_frame))
    elif t == 4:
        args.landlord = '%s/%sresnet_landlord_%i.ckpt' % (folder_b, prefix_b, adp_frame)
        args.landlord_up = '%s/%sresnet_landlord_up_%i.ckpt' % (folder_a, prefix_a, frame)
        args.landlord_front = '%s/%sresnet_landlord_front_%i.ckpt' % (folder_a, prefix_a, frame)
        args.landlord_down = '%s/%sresnet_landlord_down_%i.ckpt' % (folder_a, prefix_a, frame)
        print('%i vs %i' % (adp_frame, frame))

    evaluate(args.landlord,
             args.landlord_up,
             args.landlord_front,
             args.landlord_down,
             args.eval_data,
             args.num_workers,
             args.output,
             args.bid,
             args.title)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        'Dou Dizhu Evaluation')
    parser.add_argument('--landlord', type=str,
                        default='baselines/douzero_12/landlord_weights_39762328900.ckpt')
    parser.add_argument('--landlord_up', type=str,
                        default='baselines/douzero_12/landlord_up_weights_39762328900.ckpt')
    parser.add_argument('--landlord_front', type=str,
                        default='baselines/douzero_12/landlord_front_weights_39762328900.ckpt')
    parser.add_argument('--landlord_down', type=str,
                        default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt')
    parser.add_argument('--eval_data', type=str,
                        default='eval_data_200.pkl')
    parser.add_argument('--num_workers', type=int, default=2)
    parser.add_argument('--gpu_device', type=str, default='0')
    parser.add_argument('--output', type=bool, default=True)
    parser.add_argument('--bid', type=bool, default=True)
    parser.add_argument('--title', type=str, default='New')
    args = parser.parse_args()
    # args.output = True
    args.output = False
    args.bid = False
    if args.output or args.bid:
        args.num_workers = 1

    t = 3
    frame = 13252000
    adp_frame = 8697600

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device

    # eval_list = [
    #     [8697600, None],
    #     [8697600, 8697600],
    #     [11402400, None],
    #     [11402400, 8697600],
    #     [11402400, 11402400],
    #     [13252000, None],
    #     [13252000, 8697600],
    #     [13252000, 11402400],
    #     [13252000, 13252000],
    #     [15096800, None],
    #     [15096800, 8697600],
    #     [15096800, 11402400],
    #     [15096800, 13252000],
    #     [15096800, 15096800],
    #     [25397600, None],
    #     [25397600, 8697600],
    #     [25397600, 11402400],
    #     [25397600, 13252000],
    #     [25397600, 15096800],
    #     [25397600, 25397600],
    # ]
    
    eval_list = [
        # [4968800, 8697600, 'baselines', 'baselines2'],
        # [4968800, 4968800, 'baselines', 'baselines'],
        # [14102400, 4968800, 'baselines', 'baselines'],
        # [14102400, 13252000, 'baselines', 'baselines2'],
        # [14102400, 15096800, 'baselines', 'baselines2'],
        # [34828000, 40132800, 'baselines2', 'baselines2'],
        # [14102400, None, 'baselines', 'baselines'],
        [19918400, 19918400, 'baselines', 'baselines', 'legacy_', 'legacy_'],
        [9161600, 19918400, 'baselines', 'baselines', '', 'legacy_'],
    ]

    for vs in reversed(eval_list):
        frame = vs[0]
        adp_frame = vs[1]
        folder_a = vs[2]
        folder_b = vs[3]
        if adp_frame is None:
            if frame is None:
                make_evaluate(args, 0, None, None, folder_a , folder_b, vs[4], vs[5])
            else:
                make_evaluate(args, 1, frame, None, folder_a , folder_b, vs[4], vs[5])
                make_evaluate(args, 2, frame, None, folder_a , folder_b, vs[4], vs[5])
        else:
            make_evaluate(args, 3, frame, adp_frame, folder_a , folder_b, vs[4], vs[5])
            if frame != adp_frame:
                make_evaluate(args, 4, frame, adp_frame, folder_a, folder_b, vs[4], vs[5])