【deeplearning.ai】第二門課：提升深層神經網路——正則化的程式設計作業
阿新 • • 發佈：2018-12-30
正則化的程式設計作業，包括無正則化情況、L2正則化、Dropout的程式設計實現，程式設計中用到的相關理論和公式請參考上一篇博文。
問題描述：原問題是判斷足球運動員是否頭球，在此省略問題背景，其實就是二分類問題。有以下型別的資料，藍點為一類，紅點為一類
匯入需要的擴充套件包，reg_utils.py及資料集在此下載
import numpy as np import matplotlib.pyplot as plt from reg_utils import sigmoid, relu, plot_decision_boundary, initialize_parameters, load_2D_dataset, predict_dec from reg_utils import compute_cost, predict, forward_propagation, backward_propagation, update_parameters import sklearn import sklearn.datasets import scipy.io from testCases import * %matplotlib inline plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots plt.rcParams['image.interpolation'] = 'nearest' plt.rcParams['image.cmap'] = 'gray'
train_X, train_Y, test_X, test_Y = load_2D_dataset()# 讀取資料
一、無正則化的模型實現
def model(X, Y, learning_rate = 0.3, num_iterations = 30000, print_cost = True, lambd = 0, keep_prob = 1): """ 輸入引數： X -- 輸入資料 Y -- 標籤，1代表藍點，0代表紅點 learning_rate -- 學習率 num_iterations -- 迭代次數 print_cost -- 如果為真，則每10000次迭代輸出cost lambd -- 正則化引數 keep_prob - dropout引數 返回： parameters -- 模型學習到的引數 """ grads = {} # costs = [] # 記錄cost m = X.shape[1] # 樣本的數目 layers_dims = [X.shape[0], 20, 3, 1] # 定義網路結構 # 引數初始化 parameters = initialize_parameters(layers_dims) # 迴圈，梯度下降 for i in range(0, num_iterations): # 前向傳播 if keep_prob == 1: a3, cache = forward_propagation(X, parameters) # 實使用不帶dropout的前向傳播 elif keep_prob < 1: a3, cache = forward_propagation_with_dropout(X, parameters, keep_prob) # 使用帶dropout的前向傳播 # 代價函式 if lambd == 0: cost = compute_cost(a3, Y) # 使用不帶正則化的cost計算函式 else: cost = compute_cost_with_regularization(a3, Y, parameters, lambd) # 使用帶正則化的cost計算函式 # 反向傳播 assert(lambd==0 or keep_prob==1) # it is possible to use both L2 regularization and dropout, # but this assignment will only explore one at a time if lambd == 0 and keep_prob == 1: grads = backward_propagation(X, Y, cache) elif lambd != 0: grads = backward_propagation_with_regularization(X, Y, cache, lambd) elif keep_prob < 1: grads = backward_propagation_with_dropout(X, Y, cache, keep_prob) # 更新引數 parameters = update_parameters(parameters, grads, learning_rate) # 每10000次迭代列印cost if print_cost and i % 10000 == 0: print("Cost after iteration {}: {}".format(i, cost)) if print_cost and i % 1000 == 0: costs.append(cost) # plot the cost plt.plot(costs) plt.ylabel('cost') plt.xlabel('iterations (x1,000)') plt.title("Learning rate =" + str(learning_rate)) plt.show() return parameters
在沒有任何正則化的情況下訓練這個模型：
parameters = model(train_X, train_Y)
print ("On the training set:")
predictions_train = predict(train_X, train_Y, parameters)
print ("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)plt.title("Model without regularization") axes = plt.gca() axes.set_xlim([-0.75,0.40]) axes.set_ylim([-0.75,0.65]) plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
cost曲線如下所示：
在訓練集上的準確率為0.947867298578，在測試集上的準確率為0.915
繪製出分類邊界如下所示。沒有正則化的情況下，訓練出現了過擬合。
二、L2正則化
def compute_cost_with_regularization(A3, Y, parameters, lambd):
"""
輸入引數：
A3 -- 前向傳播的輸出
Y -- 真實的標籤
parameters -- 模型引數
返回：
cost - 帶正則化損失函式的值
"""
m = Y.shape[1]
W1 = parameters["W1"]
W2 = parameters["W2"]
W3 = parameters["W3"]
# 不帶正則化項的cost
cross_entropy_cost = compute_cost(A3, Y)
# 正則化項
L2_regularization_cost = (np.sum(np.square(W1)) + np.sum(np.square(W2)) + np.sum(np.square(W3))) * lambd /(2 * m)
# 帶正則化項的cost
cost = cross_entropy_cost + L2_regularization_cost
return costdef backward_propagation_with_regularization(X, Y, cache, lambd):
"""
輸入引數：
X -- 輸入資料
Y -- 真實的標籤
cache -- 從forward_propagation()輸出的cache
lambd -- 正則化引數
返回：
gradients -- 權重和偏置的導數
"""
m = X.shape[1]
(Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache
dZ3 = A3 - Y
dW3 = 1./m * np.dot(dZ3, A2.T) + W3 * lambd/m
db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)
dA2 = np.dot(W3.T, dZ3)
dZ2 = np.multiply(dA2, np.int64(A2 > 0))
dW2 = 1./m * np.dot(dZ2, A1.T) + W2 * lambd/m
db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)
dA1 = np.dot(W2.T, dZ2)
dZ1 = np.multiply(dA1, np.int64(A1 > 0))
dW1 = 1./m * np.dot(dZ1, X.T) + W1 * lambd/m
db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)
gradients = {"dZ3": dZ3, "dW3": dW3, "db3": db3,"dA2": dA2,
"dZ2": dZ2, "dW2": dW2, "db2": db2, "dA1": dA1,
"dZ1": dZ1, "dW1": dW1, "db1": db1}
return gradients訓練此模型，得到cost曲線：
在訓練集上的準確率為0.938388625592，在測試集上的準確率為0.93，
繪製出的分類邊界如下：
三、Dropout
def forward_propagation_with_dropout(X, parameters, keep_prob = 0.5):
"""
輸入引數：
X -- 輸入資料
parameters -- 權重和偏置
keep_prob - 保留神經元的概率
返回：
A3 -- 網路的輸出
cache -- 計算反向傳播的cache
"""
np.random.seed(1)
W1 = parameters["W1"]
b1 = parameters["b1"]
W2 = parameters["W2"]
b2 = parameters["b2"]
W3 = parameters["W3"]
b3 = parameters["b3"]
Z1 = np.dot(W1, X) + b1
A1 = relu(Z1)
# dropout
D1 = np.random.rand(A1.shape[0], A1.shape[1]) # Step 1: initialize matrix D1 = np.random.rand(..., ...)
D1 = (D1 < keep_prob) # Step 2: convert entries of D1 to 0 or 1 (using keep_prob as the threshold)
A1 = np.multiply(A1, D1) # Step 3: shut down some neurons of A1
A1 = A1/keep_prob # Step 4: scale the value of neurons that haven't been shut down
Z2 = np.dot(W2, A1) + b2
A2 = relu(Z2)
# dropout
D2 = np.random.rand(A2.shape[0], A2.shape[1]) # Step 1: initialize matrix D2 = np.random.rand(..., ...)
D2 = (D2 < keep_prob) # Step 2: convert entries of D2 to 0 or 1 (using keep_prob as the threshold)
A2 = np.multiply(A2, D2) # Step 3: shut down some neurons of A2
A2 = A2/keep_prob # Step 4: scale the value of neurons that haven't been shut down
Z3 = np.dot(W3, A2) + b3
A3 = sigmoid(Z3)
cache = (Z1, D1, A1, W1, b1, Z2, D2, A2, W2, b2, Z3, A3, W3, b3)
return A3, cachedef backward_propagation_with_dropout(X, Y, cache, keep_prob):
"""
輸入引數：
X -- 輸入資料
Y -- 真實的標籤
cache -- 從forward_propagation_with_dropout()輸出的cache
keep_prob - 保留神經元的概率
返回：
gradients -- 權重、偏置的導數
"""
m = X.shape[1]
(Z1, D1, A1, W1, b1, Z2, D2, A2, W2, b2, Z3, A3, W3, b3) = cache
dZ3 = A3 - Y
dW3 = 1./m * np.dot(dZ3, A2.T)
db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)
dA2 = np.dot(W3.T, dZ3)
dA2 = np.multiply(dA2, D2) # Step 1: Apply mask D2 to shut down the same neurons as during the forward propagation
dA2 = dA2/keep_prob # Step 2: Scale the value of neurons that haven't been shut down
dZ2 = np.multiply(dA2, np.int64(A2 > 0))
dW2 = 1./m * np.dot(dZ2, A1.T)
db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)
dA1 = np.dot(W2.T, dZ2)
dA1 = np.multiply(dA1, D1) # Step 1: Apply mask D1 to shut down the same neurons as during the forward propagation
dA1 = dA1/keep_prob # Step 2: Scale the value of neurons that haven't been shut down
dZ1 = np.multiply(dA1, np.int64(A1 > 0))
dW1 = 1./m * np.dot(dZ1, X.T)
db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)
gradients = {"dZ3": dZ3, "dW3": dW3, "db3": db3,"dA2": dA2,
"dZ2": dZ2, "dW2": dW2, "db2": db2, "dA1": dA1,
"dZ1": dZ1, "dW1": dW1, "db1": db1}
return gradients訓練此模型，得到cost曲線：
在訓練集上的準確率為0.928909952607，在測試集上的準確率為0.95
繪製的分類邊界為：
從以上可以看出，正則化降低了訓練的準確率，因為它限制了網路擬合數據的能力，但提高了測試集的準確率。