调整评估相关代码

.gitignore

@@ -1 +1,5 @@
-*.pyc
+*.pyc
+baselines*/
+douzero_checkpoints/
+.vscode/
+*.pkl

douzero/evaluation/simulation.py

@@ -6,6 +6,7 @@ from douzero.env.game import GameEnv
 import torch
 import numpy as np
 import BidModel
+import psutil
 
 def load_card_play_models(card_play_model_path_dict):
     players = {}
@@ -102,7 +103,7 @@ def mp_simulate(card_play_data_list, card_play_model_path_dict, q, output, bid_o
                 else:
                     print("".join([EnvCard2RealCard[c] for c in action]), end=end)
                 count+=1
-        if idx % 10 == 0 and not bid_output:
+        if output and idx % 10 == 0 and not bid_output:
             print("\nindex", idx)
         # print("End -------")
         env.reset()
@@ -149,8 +150,14 @@ def evaluate(landlord, landlord_up, landlord_down, eval_data, num_workers, outpu
                 target=mp_simulate,
                 args=(card_paly_data, card_play_model_path_dict, q, output, output_bid, title))
         p.start()
+
         processes.append(p)
 
+    parent = psutil.Process()
+    parent.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS)
+    for child in parent.children():
+        child.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS)
+
     for p in processes:
         p.join()
 
@@ -165,4 +172,4 @@ def evaluate(landlord, landlord_up, landlord_down, eval_data, num_workers, outpu
     print('WP results:')
     print('landlord : Farmers - {} : {}'.format(num_landlord_wins / num_total_wins, num_farmer_wins / num_total_wins))
     print('ADP results:')
-    print('landlord : Farmers - {} : {}'.format(num_landlord_scores / num_total_wins, 2 * num_farmer_scores / num_total_wins)) 
+    print('landlord : Farmers - {} : {}'.format(num_landlord_scores / num_total_wins, 3 * num_farmer_scores / num_total_wins)) 

evaluate.py

@@ -1,88 +1,34 @@
-import os 
+import os
 import argparse
 
 from douzero.evaluation.simulation import evaluate
 
-if __name__ == '__main__':
+
-    parser = argparse.ArgumentParser(
+def make_evaluate(args, t, frame, adp_frame, folder_a = 'baselines', folder_b = 'baselines'):
-                    'Dou Dizhu Evaluation')
-    parser.add_argument('--landlord', type=str,
-            default='baselines/douzero_12/landlord_weights_39762328900.ckpt')
-    parser.add_argument('--landlord_up', type=str,
-            default='baselines/douzero_12/landlord_up_weights_39762328900.ckpt')
-    parser.add_argument('--landlord_front', type=str,
-            default='baselines/douzero_12/landlord_front_weights_39762328900.ckpt')
-    parser.add_argument('--landlord_down', type=str,
-            default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt')
-    parser.add_argument('--eval_data', type=str,
-            default='eval_data_1000.pkl')
-    parser.add_argument('--num_workers', type=int, default=5)
-    parser.add_argument('--gpu_device', type=str, default='0')
-    parser.add_argument('--output', type=bool, default=True)
-    parser.add_argument('--bid', type=bool, default=True)
-    parser.add_argument('--title', type=str, default='New')
-    args = parser.parse_args()
-    args.output = True
-    args.bid = False
-    if args.output or args.bid:
-        args.num_workers = 1
-    t = 3
-    frame = 64000
-    adp_frame = 2511184300
-    # args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
-    args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame
-    args.landlord_down = 'baselines/resnet_landlord_%i.ckpt' % frame
-    args.landlord = 'baselines/douzero_ADP/landlord.ckpt'
-    # args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt'
-    # args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt'
     if t == 1:
-        args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
+        args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_a, frame)
-        args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt'
+        args.landlord_up = 'random'
-        args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt'
+        args.landlord_front = 'random'
+        args.landlord_down = 'random'
+        print('%i vs random' % frame)
     elif t == 2:
-        args.landlord = 'baselines/douzero_ADP/landlord.ckpt'
+        args.landlord = 'random'
-        args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame
+        args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_a, frame)
-        args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame
+        args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_a, frame)
+        args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_a, frame)
+        print('random vs %i' % frame)
     elif t == 3:
-        args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
+        args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_a, frame)
-        args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame
+        args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_b, adp_frame)
-        args.landlord_front = 'baselines/resnet_landlord_front_%i.ckpt' % frame
+        args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_b, adp_frame)
-        args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame
+        args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_b, adp_frame)
+        print('%i vs %i' % (frame, adp_frame))
     elif t == 4:
-        args.landlord = 'baselines/douzero_ADP/landlord.ckpt'
+        args.landlord = '%s/resnet_landlord_%i.ckpt' % (folder_b, adp_frame)
-        args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt'
+        args.landlord_up = '%s/resnet_landlord_up_%i.ckpt' % (folder_a, frame)
-        args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt'
+        args.landlord_front = '%s/resnet_landlord_front_%i.ckpt' % (folder_a, frame)
-    elif t == 5:
+        args.landlord_down = '%s/resnet_landlord_down_%i.ckpt' % (folder_a, frame)
-        args.landlord = 'baselines/douzero_WP/landlord.ckpt'
+        print('%i vs %i' % (adp_frame, frame))
-        args.landlord_up = 'baselines/douzero_WP/landlord_up.ckpt'
-        args.landlord_down = 'baselines/douzero_WP/landlord_down.ckpt'
-    elif t == 6:
-        args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
-        args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame
-        args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame
-    elif t == 7:
-        args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame
-        args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % frame
-        args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % frame
-    elif t == 8:
-        args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame
-        args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame
-        args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame
-    elif t == 9:
-        args.landlord = 'baselines/resnet_landlord_%i.ckpt' % frame
-        args.landlord_up = 'baselines/resnet_landlord_up_%i.ckpt' % adp_frame
-        args.landlord_down = 'baselines/resnet_landlord_down_%i.ckpt' % adp_frame
-    elif t == 10:
-        # landlord_down_weights_10777798400
-        args.landlord = 'baselines/douzero_ADP/landlord.ckpt'
-        args.landlord_up = 'baselines/douzero_ADP/landlord_up_weights_%i.ckpt' % adp_frame
-        args.landlord_down = 'baselines/douzero_ADP/landlord_down_weights_%i.ckpt' % adp_frame
-    elif t == 11:
-        args.landlord = 'baselines/douzero_ADP/landlord_weights_%i.ckpt' % adp_frame
-        args.landlord_up = 'baselines/douzero_ADP/landlord_up.ckpt'
-        args.landlord_down = 'baselines/douzero_ADP/landlord_down.ckpt'
-    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
-    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device
 
     evaluate(args.landlord,
              args.landlord_up,
@@ -92,3 +38,77 @@ if __name__ == '__main__':
              args.output,
              args.bid,
              args.title)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        'Dou Dizhu Evaluation')
+    parser.add_argument('--landlord', type=str,
+                        default='baselines/douzero_12/landlord_weights_39762328900.ckpt')
+    parser.add_argument('--landlord_up', type=str,
+                        default='baselines/douzero_12/landlord_up_weights_39762328900.ckpt')
+    parser.add_argument('--landlord_front', type=str,
+                        default='baselines/douzero_12/landlord_front_weights_39762328900.ckpt')
+    parser.add_argument('--landlord_down', type=str,
+                        default='baselines/douzero_12/landlord_down_weights_39762328900.ckpt')
+    parser.add_argument('--eval_data', type=str,
+                        default='eval_data_200.pkl')
+    parser.add_argument('--num_workers', type=int, default=5)
+    parser.add_argument('--gpu_device', type=str, default='0')
+    parser.add_argument('--output', type=bool, default=True)
+    parser.add_argument('--bid', type=bool, default=True)
+    parser.add_argument('--title', type=str, default='New')
+    args = parser.parse_args()
+    # args.output = True
+    args.output = False
+    args.bid = False
+    if args.output or args.bid:
+        args.num_workers = 1
+
+    t = 3
+    frame = 13252000
+    adp_frame = 8697600
+
+    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
+    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_device
+
+    # eval_list = [
+    #     [8697600, None],
+    #     [8697600, 8697600],
+    #     [11402400, None],
+    #     [11402400, 8697600],
+    #     [11402400, 11402400],
+    #     [13252000, None],
+    #     [13252000, 8697600],
+    #     [13252000, 11402400],
+    #     [13252000, 13252000],
+    #     [15096800, None],
+    #     [15096800, 8697600],
+    #     [15096800, 11402400],
+    #     [15096800, 13252000],
+    #     [15096800, 15096800],
+    #     [25397600, None],
+    #     [25397600, 8697600],
+    #     [25397600, 11402400],
+    #     [25397600, 13252000],
+    #     [25397600, 15096800],
+    #     [25397600, 25397600],
+    # ]
+    
+    eval_list = [
+        [4968800, 8697600, 'baselines', 'baselines2'],
+        [4968800, 4968800, 'baselines', 'baselines'],
+    ]
+
+    for vs in reversed(eval_list):
+        frame = vs[0]
+        adp_frame = vs[1]
+        folder_a = vs[2]
+        folder_b = vs[3]
+        if adp_frame is None:
+            make_evaluate(args, 1, frame, None)
+            make_evaluate(args, 2, frame, None)
+        else:
+            make_evaluate(args, 3, frame, adp_frame, folder_a , folder_b)
+            if frame != adp_frame:
+                make_evaluate(args, 4, frame, adp_frame, folder_a, folder_b)

requirements.txt

@@ -2,3 +2,4 @@ torch>=1.6.0
 GitPython
 gitdb2
 rlcard
+psutil