這次主要看看修改網路結構後，acc、引數個數、運算量之間的關係。
模型均用keras編寫，sgd或者adam優化，4顯示卡並行訓練，batch_size=2000（效能會降低，但是訓練速度快）
下面先列舉訓練的各種網路
Net1：直接softmax分為10類
test accuracy 0.9169

Net2：2層全連線層
test accuracy 0.9843

Net3：標準卷積網路（本網路訓練adam比sgd結果要好）
test accuracy 0.9921

Net4 標準網路去掉maxpool，採用stride=(2,2)方式
test accuracy 0.9881

Net5 使用可分離卷積
test accuracy 0.9877

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(32, (3,3),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(64, (3,3),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = Flatten()(cnn)
    feature = Dense(1024,activation='relu')(cnn)
    feature = Dropout(0.5)(feature)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)
adam = Adam(lr=1e-4)
model.compile(optimizer=adam,loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 

Net6 Net5，使用可分離卷積卷積到1024
test accuracy 0.9715

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(32, (3,3),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(64, (3,3),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(1024, (7,7),padding='valid',activation='relu')(cnn)    
    cnn = Flatten()(cnn)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

adam = Adam(lr=1e-4)
model.compile(optimizer=adam,loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 

Net7 Net6使用改變學習率的sgd
test accuracy 0.9748

sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,loss='sparse_categorical_crossentropy',metrics=['accuracy'])
model_p = multi_gpu_model(model,4)
model_p.compile(optimizer=sgd,loss='sparse_categorical_crossentropy',metrics=['accuracy'])

def scheduler_200_81_122(epoch):
    if epoch < 121:
        return 0.01
    if epoch < 162:
        return 0.001
    return 0.0001

cb_lr = LearningRateScheduler(scheduler_200_81_122)
cbks = [cb_lr]

model_p.fit(x_train,y_train,batch_size=2000,epochs=200,
    callbacks=cbks,
    validation_data=(x_test, y_test))

Net8 Net7修改卷積核5*5
test accuracy 0.9788

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(32, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(64, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(1024, (7,7),padding='valid',activation='relu')(cnn)    
    cnn = Flatten()(cnn)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

Net9 Net8通道全部減小到50%
test accuracy 0.9735

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(32, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(512, (7,7),padding='valid',activation='relu')(cnn)    
    cnn = Flatten()(cnn)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

Net10 Net9通道全部減小到50%
test accuracy 0.9687

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)    
    cnn = Flatten()(cnn)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

Net11 2個Net10合併
test accuracy 0.9716

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    
    cnn = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)  
    cnn = Flatten()(cnn)
    
    cnn2 = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn2 = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn2)
    cnn2 = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn2)
    cnn2 = Flatten()(cnn2)
    
    cnn = concatenate([cnn,cnn2],axis=1)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(cnn) 
    model = Model(inputs=input_image, outputs=predict)

Net12 Net10和Net1修改版合併
test accuracy 0.9675

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    
    cnn = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)  
    cnn = Flatten()(cnn)
    
    F2 = Flatten()(input_image)
    F2 = Dense(256,activation='relu')(F2)
    
    cnn = concatenate([cnn,F2],axis=1)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(cnn) 
    model = Model(inputs=input_image, outputs=predict)

Net13 2個不同引數的Net10合併
test accuracy 0.9661

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    
    cnn = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)  
    cnn = Flatten()(cnn)
    
    cnn2 = SeparableConv2D(8, (3,3),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn2 = SeparableConv2D(16, (3,3),strides=(2,2),padding='same',activation='relu')(cnn2)
    cnn2 = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn2)
    cnn2 = Flatten()(cnn2)
    
    cnn = concatenate([cnn,cnn2],axis=1)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(cnn) 
    model = Model(inputs=input_image, outputs=predict)

Net14 Net10修改為77卷積
test accuracy 0.9723

Net15 Net14上修改第二層卷積為11 33 55合併
test accuracy 0.9759

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(8, (7,7),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn1 = Conv2D(16, (1,1),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn2 = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn3 = SeparableConv2D(16, (3,3),strides=(2,2),padding='same',activation='relu')(cnn)

    cnn = concatenate([cnn1,cnn2,cnn3],axis=3)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)
    cnn = Flatten()(cnn)

    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

Net16 Net14上修改最後為標準卷積方式
test accuracy 0.982

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    
    cnn = SeparableConv2D(8, (7,7),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (7,7),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = Conv2D(256, (7,7),padding='valid',activation='relu')(cnn)  
    cnn = Flatten()(cnn)

    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(cnn) 
    model = Model(inputs=input_image, outputs=predict)