当前位置: 首页 > 工具软件 > num.cr > 使用案例 >

输出多标签分类模型每class指标OP,OR,OF1,CP,CR,CF1

子车海
2023-12-01

需要逐类对预测结果进行分析。

相关代码地址:https://github.com/Xingxiangrui/multi_label_analyse

目录

一、结果读出

1.1 写出结果

1.2 结果读出

二、运算指标

2.1 指标与print

2.2 指标的运算

2.3 输出

三、precision,recall,F1

3.1 定义函数进行运算

3.2 函数的调用

3.3 运行结果


一、结果读出

1.1 写出结果

PyTorch实操(五)加载模型验证并将所有结果写入文件

1.2 结果读出

一定要确定路径之中为rb,不然会将文件重写写入wb,文件会被清零。

with open('checkpoint/coco/resnet101_on_coco/model_results.pkl', 'rb') as f:
    print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
    model_results = pickle.load(f)
with open('checkpoint/coco/resnet101_on_coco/coco_labels_in_np.pkl','rb') as f:
    print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
    labels=pickle.load(f)

print('model_results[3]',model_results[3])
print('labels[3]',labels[3])

二、运算指标

2.1 指标与print

需要算出OP,OR,OF1等各个指标。直接运用self.state['ap_meter']得出相应的指标

    def on_end_epoch(self, training, model, criterion, data_loader, optimizer=None, display=True):
        map = 100 * self.state['ap_meter'].value().mean()
        loss = self.state['meter_loss'].value()[0]
        OP, OR, OF1, CP, CR, CF1 = self.state['ap_meter'].overall()
        OP_k, OR_k, OF1_k, CP_k, CR_k, CF1_k = self.state['ap_meter'].overall_topk(3)
        if display:
            if training:
                print('Epoch: [{0}]\t'
                      'Loss {loss:.4f}\t'
                      'mAP {map:.3f}'.format(self.state['epoch'], loss=loss, map=map))
                print('OP: {OP:.4f}\t'
                      'OR: {OR:.4f}\t'
                      'OF1: {OF1:.4f}\t'
                      'CP: {CP:.4f}\t'
                      'CR: {CR:.4f}\t'
                      'CF1: {CF1:.4f}'.format(OP=OP, OR=OR, OF1=OF1, CP=CP, CR=CR, CF1=CF1))
            else:
                print('Test: \t Loss {loss:.4f}\t mAP {map:.3f}'.format(loss=loss, map=map))
                print('OP: {OP:.4f}\t'
                      'OR: {OR:.4f}\t'
                      'OF1: {OF1:.4f}\t'
                      'CP: {CP:.4f}\t'
                      'CR: {CR:.4f}\t'
                      'CF1: {CF1:.4f}'.format(OP=OP, OR=OR, OF1=OF1, CP=CP, CR=CR, CF1=CF1))
                print('OP_3: {OP:.4f}\t'
                      'OR_3: {OR:.4f}\t'
                      'OF1_3: {OF1:.4f}\t'
                      'CP_3: {CP:.4f}\t'
                      'CR_3: {CR:.4f}\t'
                      'CF1_3: {CF1:.4f}'.format(OP=OP_k, OR=OR_k, OF1=OF1_k, CP=CP_k, CR=CR_k, CF1=CF1_k))

        return map

相应的函数为.overall()

2.2 指标的运算

    def overall(self):
        if self.scores.numel() == 0:
            return 0
        scores = self.scores.cpu().numpy()
        targets = self.targets.cpu().numpy()
        targets[targets == -1] = 0
        return self.evaluation(scores, targets)

    def overall_topk(self, k):
        targets = self.targets.cpu().numpy()
        targets[targets == -1] = 0
        n, c = self.scores.size()
        scores = np.zeros((n, c)) - 1
        index = self.scores.topk(k, 1, True, True)[1].cpu().numpy()
        tmp = self.scores.cpu().numpy()
        for i in range(n):
            for ind in index[i]:
                scores[i, ind] = 1 if tmp[i, ind] >= 0 else -1
        return self.evaluation(scores, targets)


    def evaluation(self, scores_, targets_):
        n, n_class = scores_.shape
        Nc, Np, Ng = np.zeros(n_class), np.zeros(n_class), np.zeros(n_class)
        for k in range(n_class):
            scores = scores_[:, k]
            targets = targets_[:, k]
            targets[targets == -1] = 0
            Ng[k] = np.sum(targets == 1)
            Np[k] = np.sum(scores >= 0)
            Nc[k] = np.sum(targets * (scores >= 0))
        Np[Np == 0] = 1
        OP = np.sum(Nc) / np.sum(Np)
        OR = np.sum(Nc) / np.sum(Ng)
        OF1 = (2 * OP * OR) / (OP + OR)

        CP = np.sum(Nc / Np) / n_class
        CR = np.sum(Nc / Ng) / n_class
        CF1 = (2 * CP * CR) / (CP + CR)
        return OP, OR, OF1, CP, CR, CF1

2.3 输出

输入为  self.state['ap_meter']

其中包括了 self.scores和self.targets,且最终转为了numpy数组

scores = self.scores.cpu().numpy() 
targets = self.targets.cpu().numpy()

在往上,此变量是通过add加入了网络的预测输出,与label

    def on_end_batch(self, training, model, criterion, data_loader, optimizer=None, display=True):

        Engine.on_end_batch(self, training, model, criterion, data_loader, optimizer, display=False)

        # measure mAP
        self.state['ap_meter'].add(self.state['output'].data, self.state['target_gt'])

三、precision,recall,F1

3.1 定义函数进行运算

我们自己定义函数运算相应的指标

def scores_evaluation(scores_, targets_):
    print('evaluation start...')
    n, n_class = scores_.shape
    print('img_numbers=',n,'n_class=',n_class)
    Nc, Np, Ng = np.zeros(n_class), np.zeros(n_class), np.zeros(n_class)
    cls_P,cls_R,cls_F1=np.zeros(n_class),np.zeros(n_class),np.zeros(n_class)
    for k in range(n_class):
        scores = scores_[:, k]         # all img scores on class_k
        targets = targets_[:, k]       # all img labels on class_k
        targets[targets == -1] = 0     # set img labels from -1 to 0
        Ng[k] = np.sum(targets == 1)   # ture:     all ture labels sum number
        Np[k] = np.sum(scores >= 0)    # positive: all predict positive sum number
        Nc[k] = np.sum(targets * (scores >= 0)) # true_positive: true_positive sum number
        cls_P[k]=Nc[k]/Np[k]
        cls_R[k]=Nc[k]/Ng[k]
        cls_F1[k]=(2 * cls_P[k] * cls_R[k]) / (cls_P[k] + cls_R[k])
    Np[Np == 0] = 1
    print('np.sum(Nc),true_positive=',np.sum(Nc))
    print('np.sum(Np),positive=',np.sum(Np))
    print('np.sum(Ng),ture=',np.sum(Ng))

    # for all labels num_imgs*n_classes
    OP = np.sum(Nc) / np.sum(Np)        # precision: true_positive/positive
    OR = np.sum(Nc) / np.sum(Ng)        # recall:    true_positive/true
    OF1 = (2 * OP * OR) / (OP + OR)     # F1_score: harmonic mean of precision and recall
    # average by class
    CP = np.sum(Nc / Np) / n_class      # precision: true_positive/positive
    CR = np.sum(Nc / Ng) / n_class      # recall:    true_positive/true
    CF1 = (2 * CP * CR) / (CP + CR)     # F1_score: harmonic mean of precision and recall

    return OP, OR, OF1, CP, CR, CF1,cls_P,cls_R,cls_F1

3.2 函数的调用

直接调用函数进行运算,加载出模型,需要为numpy格式,dim0为img_idx,  dim1为classes,直接讲numpy送入函数即可returen出需要的值。

# load results on pkl files
# file in format { 0: [[ -4.905565    -7.9314375  ... -6.8639855   -8.622047    -8.28002]] numpy 1*80
#                  1: [[ -8.905565    -8.9314375  ... -5.8639855   -6.622047    -4.28002]] numpy 1*80
#                   ...
#                  1355: ..........    numpy 1*80 }
# label in format (0: [[1. 0. 0. 0. 0. 0. 0. 0. ..... 1 ]]  numpy 1*80
#                  1: [[0. 1. .....                  ...]]}
with open('checkpoint/coco/resnet101_on_coco/model_results.pkl', 'rb') as f:
    print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
    model_results = pickle.load(f)
with open('checkpoint/coco/resnet101_on_coco/coco_labels_in_np.pkl','rb') as f:
    print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
    labels=pickle.load(f)
# print('model_results[3]',model_results[3])
# print('labels[3]',labels[3])

# concat all numpy
total_results=model_results[0]
total_labels=labels[0]
for img_idx in range(len(model_results)-1):
    if img_idx%1000==0:
        print(img_idx,'/',len(model_results))
    total_results=np.append(total_results,model_results[img_idx+1],axis=0)
    total_labels=np.append(total_labels,labels[img_idx+1],axis=0)

print('np.shape(total_results)',np.shape(total_results))
print('np.shape(total_labels)',np.shape(total_labels))

with open('checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl', 'wb') as f:
    print("writing checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl")
    pickle.dump(total_results, f)
with open('checkpoint/coco/resnet101_on_coco/coco_labels_numpy.pkl','wb') as f:
    print("writing checkpoint/coco/resnet101_on_coco/oco_labels_numpy.pkl")
    pickle.dump(total_labels, f)

OP,OR,OF1,CP,CR,CF1,cls_P,cls_R,cls_F1=scores_evaluation(scores_=total_results,targets_=total_labels)
all_evaluate_results={'OP':OP,'OR':OR,'OF1':OF1,'CP':CP,'CR':CR,'CF1':CF1,'cls_P':cls_P,'cls_R':cls_R,'cls_F1':cls_F1}
with open('checkpoint/coco/resnet101_on_coco/all_evaluate_results.pkl','wb') as f:
    print('writing checkpoint/coco/resnet101_on_coco/all_evaluate_results.pkl')
    pickle.dump(all_evaluate_results, f)

最终将模型写入文件。

3.3 运行结果

针对结果

np.shape(total_results) (40137, 80)
np.shape(total_labels) (40137, 80)
writing checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl
writing checkpoint/coco/resnet101_on_coco/oco_labels_numpy.pkl
evaluation start...
img_numbers= 40137 n_class= 80
np.sum(Nc),true_positive= 93107.0
np.sum(Np),positive= 122286.0
np.sum(Ng),ture= 116592.0
class_Precison [0.96273115 0.7880597  0.76769912 0.86618005 0.93787336 0.83281005
 0.91071429 0.6272687  0.7920434  0.74237508 0.83457249 0.826
 0.76213592 0.5688509  0.82555781 0.94503171 0.88401697 0.89315353
 0.9        0.80672269 0.96170213 0.93786982 0.99096386 0.98452381
 0.45626478 0.83036406 0.45948866 0.80294358 0.70716889 0.83989835
 0.86316776 0.7015625  0.71230398 0.91428571 0.82326112 0.8512931
 0.89946619 0.90539484 0.96890756 0.6308514  0.71782763 0.63008487
 0.59097222 0.53621561 0.51172414 0.61934082 0.79247911 0.62579281
 0.66046512 0.73846154 0.77116402 0.71869328 0.66125761 0.8952734
 0.72968198 0.68949343 0.65346734 0.66802089 0.61627201 0.79196217
 0.64725882 0.88643791 0.77614679 0.82966102 0.6955414  0.66727773
 0.73536585 0.59127907 0.57725948 0.69874477 0.2293578  0.74602304
 0.58472086 0.57300144 0.83785617 0.69811321 0.58131488 0.8566879
 0.42105263 0.60055096]
class_recall [0.9385227  0.71095153 0.83014354 0.87612797 0.93452381 0.78592593
 0.91569087 0.68920233 0.83587786 0.79610299 0.75844595 0.70118846
 0.60153257 0.6109128  0.72613738 0.90608108 0.82182774 0.86013986
 0.90184049 0.86486486 0.94957983 0.92961877 0.97193501 0.97408716
 0.52674672 0.7695621  0.56161972 0.76242236 0.66438356 0.87318362
 0.92749245 0.8424015  0.81730104 0.92434663 0.90362954 0.93491124
 0.92582418 0.89628483 0.96647108 0.68956044 0.69724771 0.70336491
 0.7254902  0.67730496 0.62405383 0.71339174 0.78159341 0.60285132
 0.69437653 0.76056338 0.87014925 0.68512111 0.72123894 0.86481647
 0.78967495 0.75851393 0.73819255 0.7948895  0.65909091 0.77786378
 0.76919192 0.91561181 0.80469245 0.79464286 0.81008902 0.69932757
 0.804      0.6        0.7734375  0.84428716 0.33783784 0.86404066
 0.7556962  0.654814   0.72476526 0.70916667 0.55629139 0.74309392
 0.22857143 0.63929619]
class F1 score [0.9504728  0.74752242 0.79770115 0.87112561 0.93619559 0.80868902
 0.9131958  0.65677868 0.81337047 0.76830087 0.79469027 0.75849403
 0.67237687 0.58913204 0.77266255 0.9251466  0.85178876 0.87633588
 0.90091931 0.83478261 0.95560254 0.93372607 0.9813572  0.97927768
 0.48897897 0.79880775 0.50544662 0.7821585  0.68510889 0.85621762
 0.89417476 0.7655584  0.76119884 0.91928865 0.86157518 0.89114495
 0.91245487 0.9008168  0.96768779 0.65890074 0.70738802 0.66471133
 0.65135859 0.59855845 0.56233422 0.66304769 0.78699862 0.61410788
 0.67699642 0.74934952 0.81767181 0.70150576 0.68994709 0.87978142
 0.75849403 0.72235872 0.69325088 0.72595396 0.63696266 0.78484967
 0.70297715 0.90078871 0.79016189 0.81177446 0.74845785 0.68292683
 0.76815287 0.59560761 0.66110184 0.76465201 0.27322404 0.80070651
 0.65930425 0.61118203 0.77721838 0.70359653 0.56852792 0.79585799
 0.2962963  0.61931818]
All evaluate results:
OP: 0.7614	OR: 0.7986	OF1: 0.7795	CP: 0.7446	CR: 0.7693	CF1: 0.7568
 类似资料: