caffe python 介面api
阿新 • • 發佈：2022-04-20
目錄
caffe 某層featuremap ，和權重
featuremap_1 = net.blobs['conv1_1'].data #某一層feature map 輸出
filters = net.params['conv1_1'][0].data #某一層權重
只有deply，直接輸出caffemodel
net = caffe.Net(path_deply, phase=caffe.TEST) net.save("./111.caffemodel")
儲存caffe權重到pkl檔案
import pickle as pkl #feature map and shape print("=================feature map===================") for layer_name, blob in net.blobs.iteritems(): print layer_name + '\t' + str(blob.data.shape) print("=================weights===================") for layer_name, blob in net.params.iteritems(): len_ = len(blob) print layer_name + " has " + str(len_) + " params" for i in range(len_): print layer_name + ' idx= ' + str(i) + '\t' + str(blob[i].data.shape) ############################################################################################3 name_weights = {} # 儲存每層的引數資訊 keys = open('keys.txt', 'w') keys.write('generated by Net-Caffe/convert_to_pkl.py\n\n') # 遍歷每一網路層 for param_name in net.params.keys(): name_weights[param_name] = {} # 得到此層的引數 layer_params = net.params[param_name] if len(layer_params) == 1: # 如果引數只有一個，則說明是反捲積層, # SfSNet整個模型裡就只有反捲積層只有一組weight引數 weight = layer_params[0].data name_weights[param_name]['weight'] = weight print('%s:\n\t%s (weight)' % (param_name, weight.shape)) keys.write('%s:\n\t%s (weight)\n' % (param_name, weight.shape)) elif len(layer_params) == 2: # 如果引數有兩個，則說明是卷積層或者全連線層。 # 卷積層或者全連線層都有兩組引數：weight和bias # 權重引數 weight = layer_params[0].data name_weights[param_name]['weight'] = weight # 偏置引數 bias = layer_params[1].data name_weights[param_name]['bias'] = bias print('%s:\n\t%s (weight)' % (param_name, weight.shape)) print('\t%s (bias)' % str(bias.shape)) keys.write('%s:\n\t%s (weight)\n' % (param_name, weight.shape)) keys.write('\t%s (bias)\n' % str(bias.shape)) elif len(layer_params) == 5: # 如果有三個，則說明是BatchNorm層。 # BN層共有三個引數，分別是：running_mean、running_var和一個縮放參數。 running_mean = layer_params[0].data # running_mean name_weights[param_name]['running_mean'] = running_mean / layer_params[2].data running_var = layer_params[1].data # running_var name_weights[param_name]['running_var'] = running_var / layer_params[2].data aa = layer_params[2].data print("bn=",aa) name_weights[param_name]['weight'] = layer_params[3].data name_weights[param_name]['bias'] = layer_params[4].data print('%s:\n\t%s (running_var)' % (param_name, running_var.shape),) print('\t%s (running_mean)' % str(running_mean.shape)) keys.write('%s:\n\t%s (running_var)\n' % (param_name, running_var.shape)) keys.write('\t%s (running_mean)\n' % str(running_mean.shape)) keys.write('\t%s (weight)\n' % str(layer_params[3].data.shape)) keys.write('\t%s (bias)\n' % str(layer_params[4].data.shape)) else: # 如果報錯，大家要檢查自己模型哈 raise RuntimeError("還有引數個數超過3個的層，別漏了兄dei！！！\n") keys.close() # 儲存name_weights with open('weights20220420.pkl', 'wb') as f: pkl.dump(name_weights, f, protocol=2)
提取到的caffe權重pkl檔案載入到pytorch中：
import torch import torchvision import pickle as pkl from torch import nn import torch.nn.functional as F import pickle as pkl import cv2 import numpy as np if __name__ == '__main__': net = net() # net.eval() index = 0 print("*" * 50) for name, param in list(net.named_parameters()): print(str(index) + ':', name, param.size()) index += 1 print("*" * 50) for k, v in net.state_dict().items(): print(k) print(v.shape) # print(k,v) print("@" * 50) from torch import from_numpy with open('/media/algo/data_1/project/net_my_pytorch/bk/0420all_tiqu_convet/weights20220420.pkl', 'rb') as wp: name_weights = pkl.load(wp,encoding='latin1')#pickle py2 save and py3 read state_dict = {} # aaa = name_weights['Scale1']['weight'] # aa = from_numpy(name_weights['Scale1']['weight']).reshape([1,3,1,1]) # print(aa) # state_dict['Scale1.scale'] = from_numpy(name_weights['Scale1']['weight']).reshape([1,3,1,1]) #conv1_1 state_dict['conv1_1.weight'] = from_numpy(name_weights['conv1_1']['weight']) state_dict['conv1_1.bias'] = from_numpy(name_weights['conv1_1']['bias']) #conv1_1_bn state_dict['conv1_1_bn.running_var'] = from_numpy(name_weights['conv1_1/bn']['running_var']) state_dict['conv1_1_bn.running_mean'] = from_numpy(name_weights['conv1_1/bn']['running_mean']) state_dict['conv1_1_bn.weight'] = from_numpy(name_weights['conv1_1/bn']['weight']) state_dict['conv1_1_bn.bias'] = from_numpy(name_weights['conv1_1/bn']['bias']) # conv1_2 state_dict['conv1_2.weight'] = from_numpy(name_weights['conv1_2']['weight']) state_dict['conv1_2.bias'] = from_numpy(name_weights['conv1_2']['bias']) # conv1_2_bn state_dict['conv1_2_bn.running_var'] = from_numpy(name_weights['conv1_2/bn']['running_var']) state_dict['conv1_2_bn.running_mean'] = from_numpy(name_weights['conv1_2/bn']['running_mean']) state_dict['conv1_2_bn.weight'] = from_numpy(name_weights['conv1_2/bn']['weight']) state_dict['conv1_2_bn.bias'] = from_numpy(name_weights['conv1_2/bn']['bias']) # conv2_1 state_dict['conv2_1.weight'] = from_numpy(name_weights['conv2_1']['weight']) state_dict['conv2_1.bias'] = from_numpy(name_weights['conv2_1']['bias']) # conv2_1_bn state_dict['conv2_1_bn.running_var'] = from_numpy(name_weights['conv2_1/bn']['running_var']) state_dict['conv2_1_bn.running_mean'] = from_numpy(name_weights['conv2_1/bn']['running_mean']) state_dict['conv2_1_bn.weight'] = from_numpy(name_weights['conv2_1/bn']['weight']) state_dict['conv2_1_bn.bias'] = from_numpy(name_weights['conv2_1/bn']['bias']) # conv2_2 state_dict['conv2_2.weight'] = from_numpy(name_weights['conv2_2']['weight']) state_dict['conv2_2.bias'] = from_numpy(name_weights['conv2_2']['bias']) # conv2_2_bn state_dict['conv2_2_bn.running_var'] = from_numpy(name_weights['conv2_2/bn']['running_var']) state_dict['conv2_2_bn.running_mean'] = from_numpy(name_weights['conv2_2/bn']['running_mean']) state_dict['conv2_2_bn.weight'] = from_numpy(name_weights['conv2_2/bn']['weight']) state_dict['conv2_2_bn.bias'] = from_numpy(name_weights['conv2_2/bn']['bias']) ############################ # conv3_1 state_dict['conv3_1.weight'] = from_numpy(name_weights['conv3_1']['weight']) state_dict['conv3_1.bias'] = from_numpy(name_weights['conv3_1']['bias']) # conv3_1_bn state_dict['conv3_1_bn.running_var'] = from_numpy(name_weights['conv3_1/bn']['running_var']) state_dict['conv3_1_bn.running_mean'] = from_numpy(name_weights['conv3_1/bn']['running_mean']) state_dict['conv3_1_bn.weight'] = from_numpy(name_weights['conv3_1/bn']['weight']) state_dict['conv3_1_bn.bias'] = from_numpy(name_weights['conv3_1/bn']['bias']) # conv3_2 state_dict['conv3_2.weight'] = from_numpy(name_weights['conv3_2']['weight']) state_dict['conv3_2.bias'] = from_numpy(name_weights['conv3_2']['bias']) # conv3_2_bn state_dict['conv3_2_bn.running_var'] = from_numpy(name_weights['conv3_2/bn']['running_var']) state_dict['conv3_2_bn.running_mean'] = from_numpy(name_weights['conv3_2/bn']['running_mean']) state_dict['conv3_2_bn.weight'] = from_numpy(name_weights['conv3_2/bn']['weight']) state_dict['conv3_2_bn.bias'] = from_numpy(name_weights['conv3_2/bn']['bias']) ############################ # conv4_1 state_dict['conv4_1.weight'] = from_numpy(name_weights['conv4_1']['weight']) state_dict['conv4_1.bias'] = from_numpy(name_weights['conv4_1']['bias']) # conv3_1_bn state_dict['conv4_1_bn.running_var'] = from_numpy(name_weights['conv4_1/bn']['running_var']) state_dict['conv4_1_bn.running_mean'] = from_numpy(name_weights['conv4_1/bn']['running_mean']) state_dict['conv4_1_bn.weight'] = from_numpy(name_weights['conv4_1/bn']['weight']) state_dict['conv4_1_bn.bias'] = from_numpy(name_weights['conv4_1/bn']['bias']) # conv3_2 state_dict['conv4_2.weight'] = from_numpy(name_weights['conv4_2']['weight']) state_dict['conv4_2.bias'] = from_numpy(name_weights['conv4_2']['bias']) # conv3_2_bn state_dict['conv4_2_bn.running_var'] = from_numpy(name_weights['conv4_2/bn']['running_var']) state_dict['conv4_2_bn.running_mean'] = from_numpy(name_weights['conv4_2/bn']['running_mean']) state_dict['conv4_2_bn.weight'] = from_numpy(name_weights['conv4_2/bn']['weight']) state_dict['conv4_2_bn.bias'] = from_numpy(name_weights['conv4_2/bn']['bias']) ############################ # conv5_1 state_dict['conv5_1.weight'] = from_numpy(name_weights['conv5_1']['weight']) state_dict['conv5_1.bias'] = from_numpy(name_weights['conv5_1']['bias']) # conv5_1_bn state_dict['conv5_1_bn.running_var'] = from_numpy(name_weights['conv5_1/bn']['running_var']) state_dict['conv5_1_bn.running_mean'] = from_numpy(name_weights['conv5_1/bn']['running_mean']) state_dict['conv5_1_bn.weight'] = from_numpy(name_weights['conv5_1/bn']['weight']) state_dict['conv5_1_bn.bias'] = from_numpy(name_weights['conv5_1/bn']['bias']) # conv5_2 state_dict['conv5_2.weight'] = from_numpy(name_weights['conv5_2']['weight']) state_dict['conv5_2.bias'] = from_numpy(name_weights['conv5_2']['bias']) # conv5_2_bn state_dict['conv5_2_bn.running_var'] = from_numpy(name_weights['conv5_2/bn']['running_var']) state_dict['conv5_2_bn.running_mean'] = from_numpy(name_weights['conv5_2/bn']['running_mean']) state_dict['conv5_2_bn.weight'] = from_numpy(name_weights['conv5_2/bn']['weight']) state_dict['conv5_2_bn.bias'] = from_numpy(name_weights['conv5_2/bn']['bias']) ############################ # conv6_1 state_dict['conv6_1.weight'] = from_numpy(name_weights['conv6_1']['weight']) state_dict['conv6_1.bias'] = from_numpy(name_weights['conv6_1']['bias']) # conv6_1_bn state_dict['conv6_1_bn.running_var'] = from_numpy(name_weights['conv6_1/bn']['running_var']) state_dict['conv6_1_bn.running_mean'] = from_numpy(name_weights['conv6_1/bn']['running_mean']) state_dict['conv6_1_bn.weight'] = from_numpy(name_weights['conv6_1/bn']['weight']) state_dict['conv6_1_bn.bias'] = from_numpy(name_weights['conv6_1/bn']['bias']) # conv6_2 state_dict['conv6_2.weight'] = from_numpy(name_weights['conv6_2']['weight']) state_dict['conv6_2.bias'] = from_numpy(name_weights['conv6_2']['bias']) # conv6_2_bn state_dict['conv6_2_bn.running_var'] = from_numpy(name_weights['conv6_2/bn']['running_var']) state_dict['conv6_2_bn.running_mean'] = from_numpy(name_weights['conv6_2/bn']['running_mean']) state_dict['conv6_2_bn.weight'] = from_numpy(name_weights['conv6_2/bn']['weight']) state_dict['conv6_2_bn.bias'] = from_numpy(name_weights['conv6_2/bn']['bias']) # upscore1 state_dict['upscore1.weight'] = from_numpy(name_weights['upscore1']['weight']) ############################ # conv7_1 state_dict['conv7_1.weight'] = from_numpy(name_weights['conv7_1']['weight']) state_dict['conv7_1.bias'] = from_numpy(name_weights['conv7_1']['bias']) # conv7_1_bn state_dict['conv7_1_bn.running_var'] = from_numpy(name_weights['conv7_1/bn']['running_var']) state_dict['conv7_1_bn.running_mean'] = from_numpy(name_weights['conv7_1/bn']['running_mean']) state_dict['conv7_1_bn.weight'] = from_numpy(name_weights['conv7_1/bn']['weight']) state_dict['conv7_1_bn.bias'] = from_numpy(name_weights['conv7_1/bn']['bias']) # conv7_2 state_dict['conv7_2.weight'] = from_numpy(name_weights['conv7_2']['weight']) state_dict['conv7_2.bias'] = from_numpy(name_weights['conv7_2']['bias']) # conv7_2_bn state_dict['conv7_2_bn.running_var'] = from_numpy(name_weights['conv7_2/bn']['running_var']) state_dict['conv7_2_bn.running_mean'] = from_numpy(name_weights['conv7_2/bn']['running_mean']) state_dict['conv7_2_bn.weight'] = from_numpy(name_weights['conv7_2/bn']['weight']) state_dict['conv7_2_bn.bias'] = from_numpy(name_weights['conv7_2/bn']['bias']) # upscore2 state_dict['upscore2.weight'] = from_numpy(name_weights['upscore2']['weight']) ############################ # conv8_1 state_dict['conv8_1.weight'] = from_numpy(name_weights['conv8_1']['weight']) state_dict['conv8_1.bias'] = from_numpy(name_weights['conv8_1']['bias']) # conv8_1_bn state_dict['conv8_1_bn.running_var'] = from_numpy(name_weights['conv8_1/bn']['running_var']) state_dict['conv8_1_bn.running_mean'] = from_numpy(name_weights['conv8_1/bn']['running_mean']) state_dict['conv8_1_bn.weight'] = from_numpy(name_weights['conv8_1/bn']['weight']) state_dict['conv8_1_bn.bias'] = from_numpy(name_weights['conv8_1/bn']['bias']) # conv8_2 state_dict['conv8_2.weight'] = from_numpy(name_weights['conv8_2']['weight']) state_dict['conv8_2.bias'] = from_numpy(name_weights['conv8_2']['bias']) # conv8_2_bn state_dict['conv8_2_bn.running_var'] = from_numpy(name_weights['conv8_2/bn']['running_var']) state_dict['conv8_2_bn.running_mean'] = from_numpy(name_weights['conv8_2/bn']['running_mean']) state_dict['conv8_2_bn.weight'] = from_numpy(name_weights['conv8_2/bn']['weight']) state_dict['conv8_2_bn.bias'] = from_numpy(name_weights['conv8_2/bn']['bias']) # upscore3 state_dict['upscore3.weight'] = from_numpy(name_weights['upscore3']['weight']) ############################ # conv9_1 state_dict['conv9_1.weight'] = from_numpy(name_weights['conv9_1']['weight']) state_dict['conv9_1.bias'] = from_numpy(name_weights['conv9_1']['bias']) # conv9_1_bn state_dict['conv9_1_bn.running_var'] = from_numpy(name_weights['conv9_1/bn']['running_var']) state_dict['conv9_1_bn.running_mean'] = from_numpy(name_weights['conv9_1/bn']['running_mean']) state_dict['conv9_1_bn.weight'] = from_numpy(name_weights['conv9_1/bn']['weight']) state_dict['conv9_1_bn.bias'] = from_numpy(name_weights['conv9_1/bn']['bias']) # conv9_2 state_dict['conv9_2.weight'] = from_numpy(name_weights['conv9_2']['weight']) state_dict['conv9_2.bias'] = from_numpy(name_weights['conv9_2']['bias']) # conv9_2_bn state_dict['conv9_2_bn.running_var'] = from_numpy(name_weights['conv9_2/bn']['running_var']) state_dict['conv9_2_bn.running_mean'] = from_numpy(name_weights['conv9_2/bn']['running_mean']) state_dict['conv9_2_bn.weight'] = from_numpy(name_weights['conv9_2/bn']['weight']) state_dict['conv9_2_bn.bias'] = from_numpy(name_weights['conv9_2/bn']['bias']) #######################3 #net_my_cc_loc state_dict['net_my_cc_loc.weight'] = from_numpy(name_weights['net_my_cc_loc']['weight']) state_dict['net_my_cc_loc.bias'] = from_numpy(name_weights['net_my_cc_loc']['bias']) # net_my_cc_conf state_dict['net_my_cc_conf.weight'] = from_numpy(name_weights['net_my_cc_conf']['weight']) state_dict['net_my_cc_conf.bias'] = from_numpy(name_weights['net_my_cc_conf']['bias']) # net_my_cc1_loc state_dict['net_my_cc1_loc.weight'] = from_numpy(name_weights['net_my_cc1_loc']['weight']) state_dict['net_my_cc1_loc.bias'] = from_numpy(name_weights['net_my_cc1_loc']['bias']) # net_my_cc1_conf state_dict['net_my_cc1_conf.weight'] = from_numpy(name_weights['net_my_cc1_conf']['weight']) state_dict['net_my_cc1_conf.bias'] = from_numpy(name_weights['net_my_cc1_conf']['bias']) #######################3 # net_my_rd_loc state_dict['net_my_rd_loc.weight'] = from_numpy(name_weights['net_my_rd_loc']['weight']) state_dict['net_my_rd_loc.bias'] = from_numpy(name_weights['net_my_rd_loc']['bias']) # net_my_rd_conf state_dict['net_my_rd_conf.weight'] = from_numpy(name_weights['net_my_rd_conf']['weight']) state_dict['net_my_rd_conf.bias'] = from_numpy(name_weights['net_my_rd_conf']['bias']) # net_my_rd1_loc state_dict['net_my_rd1_loc.weight'] = from_numpy(name_weights['net_my_rd1_loc']['weight']) state_dict['net_my_rd1_loc.bias'] = from_numpy(name_weights['net_my_rd1_loc']['bias']) # net_my_rd1_conf state_dict['net_my_rd1_conf.weight'] = from_numpy(name_weights['net_my_rd1_conf']['weight']) state_dict['net_my_rd1_conf.bias'] = from_numpy(name_weights['net_my_rd1_conf']['bias']) #######################3 # net_my_pd_loc state_dict['net_my_pd_loc.weight'] = from_numpy(name_weights['net_my_pd_loc']['weight']) state_dict['net_my_pd_loc.bias'] = from_numpy(name_weights['net_my_pd_loc']['bias']) # net_my_pd_conf state_dict['net_my_pd_conf.weight'] = from_numpy(name_weights['net_my_pd_conf']['weight']) state_dict['net_my_pd_conf.bias'] = from_numpy(name_weights['net_my_pd_conf']['bias']) # net_my_pd1_loc state_dict['net_my_pd1_loc.weight'] = from_numpy(name_weights['net_my_pd1_loc']['weight']) state_dict['net_my_pd1_loc.bias'] = from_numpy(name_weights['net_my_pd1_loc']['bias']) # net_my_pd1_conf state_dict['net_my_pd1_conf.weight'] = from_numpy(name_weights['net_my_pd1_conf']['weight']) state_dict['net_my_pd1_conf.bias'] = from_numpy(name_weights['net_my_pd1_conf']['bias']) #######################3 # net_my_vd_loc state_dict['net_my_vd_loc.weight'] = from_numpy(name_weights['net_my_vd_loc']['weight']) state_dict['net_my_vd_loc.bias'] = from_numpy(name_weights['net_my_vd_loc']['bias']) # net_my_vd_conf state_dict['net_my_vd_conf.weight'] = from_numpy(name_weights['net_my_vd_conf']['weight']) state_dict['net_my_vd_conf.bias'] = from_numpy(name_weights['net_my_vd_conf']['bias']) # net_my_vd1_loc state_dict['net_my_vd1_loc.weight'] = from_numpy(name_weights['net_my_vd1_loc']['weight']) state_dict['net_my_vd1_loc.bias'] = from_numpy(name_weights['net_my_vd1_loc']['bias']) # net_my_vd1_conf state_dict['net_my_vd1_conf.weight'] = from_numpy(name_weights['net_my_vd1_conf']['weight']) state_dict['net_my_vd1_conf.bias'] = from_numpy(name_weights['net_my_vd1_conf']['bias']) net.load_state_dict(state_dict) net.cuda() net.eval() torch.save(net.state_dict(), './net_my_model_20220420.pth')