'''
Created on 2017年9月13日

@author: weizhen
'''
import numpy as np
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

首先上來的是最簡單的sigmoid激勵函式，

至於為什麼選他做激勵函式，

1、因為這個函式能將定義域為(-inf,+inf)的值對映到(0,1)區間，便於計算(能夠消除量綱的影響)

2、這個函式的變化曲線不是特變陡峭，每一點處都可導

3、這個函式的導數為y(1-y)，即用他原來的函式值就可以求出其導數的值，便於計算

如果感覺寫得還行請大家搶個沙發啥的，小弟不勝感激。

誤差的反向傳遞求grad_xi

令

令

所以

令

則 

To Summary：

Tip3:上面公式中 的值是由輸出向量w_c與每一輪的變數值x相乘得來的,使用交叉熵函式作為損失函式主要是 為了恆量輸出向量與正確分佈的相似程度

那麼每次在誤差的反向傳播過程中，

如何將梯度下降方法應用到變數x和變數w_c上呢？

需要首先求出目標函式對於x和變數w_c的導數grad_xi和grad_wi

然後指定相應的步長d(可以對於x和變數w_c應用相同的步長)

然後使用梯度下降公式，來對x和w_c的值進行更新

x_i+1=x_i+d*grad_xi

w_i+1=w_i+d*gradwi

在將得到的兩個新的值代入到 交叉熵函式裡邊CrossEntropy(x_i+1,w_i+1)<Predefine Error 則停止迭代

否則繼續迭代。

上面所有的公式推導都是在求grad_xi,並且在這過程中需要使用到交叉熵關於  x和變數w_c隱函式的導數，也就是交叉熵對softmax函式求導，這時候可以進行相應的化簡

這裡邊有兩種情況

 當f_y=f_c時  $\frac{\partial Entropy}{\partial z_{y}}=z_{y}-1$ 即如果是正對的節點的誤差傳播，對後一節點的導數，可以用當前前向傳播傳播過程計算出來的值$z_{y}-1$來計算gradz

 當f_y不等於f_c時  $\frac{\partial Entropy}{\partial z_{y}}=z_{y}$ 即如果不是正對的節點的誤差傳播，對後一節點的導數，可以用當前前向傳播傳播過程計算出來的值$z_{y}$來計算gradz

如果實在不明白可以參考以下圖片,第一張圖片是交叉熵函式對x變數求偏導，第二張圖片是交叉熵函式對w變數求偏導

下面這張圖片是交叉熵損失函式對變數w求偏導，這裡將w=(w_c,w_y)

接下來是sigmoid函式的導數 

'''
Created on Sep 11, 2017

@author: p0079482
'''
def sigmoid_graf(f):
    """
        計算Sigmoid的梯度
    """
    f = f * (1 - f)
    
    return f

上面這個就沒啥可說的了，就是求神經元的導數

下面是softmax函式，能夠將[3,1,3]類似這種的輸出結果通過exp(3)/[exp(3)+exp(1)+exp(3)]這種形式對映到(0,1)的概率區間，

便於下一步我們進行交叉熵計算

下面的程式碼是數值求微分的方法，來檢驗上一步中計算的導數值和真實值的區別是否足夠小，即通過公式算出來的導數是否精確

'''
Created on Sep 11, 2017

@author: p0079482
'''
import random
import numpy as np
def gradcheck_naive(f,x):
    """ 對一個函式f求梯度檢驗
        -f 輸入x,然後輸出loss和梯度的函式
        -x 就是輸入
     """
     
    rndstate = random.getstate() 
      #獲取當前隨機數的環境狀態，以便下次再使用
    random.setstate(rndstate)
    fx,grad=f(x)
    h=1e-4
     
     #遍歷x的每一維
     #單個數組的迭代
     #nditer將輸入陣列視為只讀物件。要修改陣列元素，必須指定讀寫（ read-write）或只寫（write-only）模式
     
    it = np.nditer(x,flags=['multi_index'],op_flags=['readwrite'])
    while not it.finished:
         ix=it.multi_index
         old_val=x[ix]
         x[ix]=old_val-h
         random.setstate(rndstate)
         (fxh1,_)=f(x)
         
         x[ix]=old_val+h
         random.setstate(rndstate)
         (fxh2,_)=f(x)
         
         numgrad = (fxh2-fxh1)/(2*h)
         x[ix] = old_val
         
         #對比梯度
         reldiff = abs(numgrad-grad[ix])/max(1,abs(numgrad),abs(grad[ix]))
         if reldiff>1e-5:
             print("Gradient check failed.")
             print("First gradient error found at index %s"%str(ix))
             print("Your gradient: %f \t Numerical gradient: %f"%(grad[ix],numgrad))
             return
         it.iternext()  #Step to next dimension
    print("Gradient check passed!")
      
    
         
         
         

下面的程式碼是計算一個一層全連線網路，一層softmax層的程式。並且計算了梯度的反向傳播。

'''
Created on Sep 11, 2017

@author: p0079482
'''

"""寫出只有一個隱藏層其啟用函式為sigmoid的神經網路前向和後向傳播程式碼"""
import numpy as np
from q1_sigmoid import sigmoid
from q1_softmax import softmax
from q2_sigmoid import sigmoid_graf
def forward_backward_prop(data, labels, params, verbose=False):
    """2 個隱層的神經網路的前向運算和反向傳播"""
    
    if len(data.shape) >= 2:
        (N, _) = data.shape
    
    dimensions = [1] * 3
    dimensions[0]=N
    dimensions[1]=20
    dimensions[2]=labels.shape
    
    
    # ## 展開每一層神經網路的引數
    t = 0
    W1 = np.reshape(params[t:t + dimensions[0] * dimensions[1]], (dimensions[0], dimensions[1]))
    t += dimensions[0] * dimensions[1]
    b1 = np.reshape(params[t:t + dimensions[1]], (1, dimensions[1]))
    t += dimensions[1]
    W2 = np.reshape(params[t:t + dimensions[1] * dimensions[2]], (dimensions[1], dimensions[2]))
    t += dimensions[1] * dimensions[2]
    b2 = np.reshape(params[t:t + dimensions[2]], (1, dimensions[2]))
    
    # ##前向運算
    
    # 第一個隱層做內積
    a1 = sigmoid(data.dot(W1) + b1)
    # 第二個隱層做內積
    a2 = softmax(a1.dot(W2) + b2)
    
    cost = -np.sum(np.log(a2[labels == 1])) / N
    
    # ## 反向傳播
    
    # Calculate analytic gradient for the cross entropy function
    grad_a2 = (a2 - labels) / N
    
    # Backpropagate through the second latent layer
    gradW2 = np.dot(a1.T, grad_a2)
    gradb2 = np.sum(grad_a2, axis=0, keepdims=True)
    
    # Backpropagate through the first latent layer
    grad_a1 = np.dot(grad_a2, W2.T) * sigmoid_graf(a1)
    
    gradW1 = np.dot(data.T, grad_a1)
    gradb1 = np.sum(grad_a1, axis=0, keepdims=True)
    
    if verbose:  # Verbose mode for logging information
        print("W1 shape:{}".format(str(W1.shape)))
        print("W1 gradient shape: {}".format(str(gradW1.shape)))
        print("b1 shape: {}".format(str(b1.shape)))
        print("b1 gradient shape: {}".format(str(gradb1.shape)))
    
    # ## 梯度拼起來
    grad = np.concatenate((gradW1.flatten(), gradb1.flatten(), gradW2.flatten(), gradb2.flatten()))
    return cost, grad

    

下面的程式碼是隨機梯度下降，SGD真的是和普通的梯度下降演算法相比特別特別地塊和有效來著

'''
Created on Sep 12, 2017

@author: p0079482

實現隨機梯度下降
隨機梯度下降每1000輪，就儲存一下現在訓練得到的引數
'''
SAVE_PARAMS_EVERY = 1000

import glob
import os.path as op
import pickle  as pickle
import sys
import random

def load_saved_params():
    """
        載入之前的引數以免從頭開始訓練
    """
    st = 0
    for f in glob.glob("saved_params_*.npy"):
        iter = int(op.splitext(op.basename(f))[0].split("_")[2])
        if(iter > st):
            st = iter
    
    if st > 0:
        with open("saved_params_%d.npy" % st, "rb") as f:
            params = pickle.load(f)
            state = pickle.load(f)
        return st, params, state
    else:
        return st, None, None
    
def save_params(iter, params):
    with open("saved_params_%d.npy" % iter, "wb") as f:
        pickle.dump(params, f)
        pickle.dump(random.getstate(), f)

def sgd(f, x0, step, iterations, postprocessing=None, useSaved=False, PRINT_EVERY=10, ANNEAL_EVERY=20000):
    """隨機梯度下降
    輸入:
    f:需要最優化的函式
    x0:SGD的初始值
    step:SGD的步長
    iterations：總的迭代次數
    postprocessing:引數後處理(比如word2vec裡需要對詞向量做歸一化處理)
    PRINT_EVERY:指明多少次迭代以後輸出一下狀態
  輸出:
    x:SGD完成後的輸出引數
    """
    if useSaved:
        start_iter, oldx, state = load_saved_params()
        if start_iter > 0:
            x0 = oldx;
            step *= 0.5
        if state:
            random.setstate(state)
    else:
        start_iter = 0
    
    x = x0
    
    if not postprocessing:
        postprocessing = lambda x:x
    
    expcost = None
    
    for iter in range(start_iter + 1, iterations + 1):
        cost, grad = f(x)
        x = x - step * grad
        x = postprocessing(x)
        
        if iter % PRINT_EVERY == 0:
            print("iter#{},cost={}".format(iter, cost))
            sys.stdout.flush()
        
        if iter % SAVE_PARAMS_EVERY == 0 and useSaved:
            save_params(iter, x)
        
        if iter % ANNEAL_EVERY == 0:
            step *= 0.5
    return x

    

下面的程式碼是softmax函式的誤差反向傳播，

skip-gram模型的實現

'''
Created on Sep 11, 2017

@author: p0079482
'''
"""實現word2vec模型，並使用隨機梯度下降方法SGD訓練屬於自己的詞向量
      首先寫一個輔助函式對矩陣中的每一行進行歸一化，
      同樣在這個檔案中完成對softmax,
      負取樣損失函式
      以及梯度計算函式的實現
      然後完成面向skip-gram的梯度損失函式
"""
import numpy as np
import random

from q1_softmax import softmax
from q2_gradcheck import gradcheck_naive
from q2_sigmoid import sigmoid_graf
from q1_sigmoid import sigmoid

def normalizeRows(x):
    """
       行歸一化函式
    """
    N=x.shape[0]
    x/=np.sqrt(np.sum(x**2,axis=1)).reshape((N,1))+1e-30
    
    return x

def test_normalize_rows():
    print("Testing normalizeRows")
    x = normalizeRows(np.array([[3.0,4.0],[1,2]]))
    """結果應該是[[0.6,0.8],[0.4472136,0.89442719]]"""
    print(x)
    assert(np.amax(np.fabs(x-np.array([[0.6,0.8],[0.4472136,0.89442719]])))<= 1e-6)
    print(" ")

def softmaxCostAndGradient(predicted,target,outputVectors,dataset):
    """word2vec的Softmax損失函式"""
    
    """
       輸入:
       predicted：預測詞向量的numpy陣列
       target：目標詞的下標
       outputVectors：所有token的"output"向量(行形式)
       dataset：用來做負例取樣的，這裡其實沒用著
    
        輸出:
        cost:輸出的互熵損失
        gradPred:the gradient with respect to the predicted word vector
        grad:    the gradient with respect to all the other word vectors
    """
    probabilities = softmax(predicted.dot(outputVectors.T))
    cost = -np.log(probabilities[target])
    delta = probabilities
    delta[target] -= 1
    N = delta.shape[0]
    D = predicted.shape[0]
    grad = delta.reshape((N,1))*predicted.reshape((1,D))
    gradPred = (delta.reshape((1,N)).dot(outputVectors)).flatten()
    
    return cost,gradPred,grad

def negSamplingCostAndGradient(predicted,target,outputVectors,dataset,K=10):
    """
        Word2vec模型負例取樣後的損失函式和梯度
    """
    grad = np.zeros(outputVectors.shape)
    gradPred = np.zeros(predicted.shape)
    
    indices = [target]
    for k in range(K):
        newidx = dataset.sampleTokenIdx()
        while newidx == target:
            newidx = dataset.sampleTokenIdx()
        indices+=[newidx]
    
    labels = np.array([1]+[-1 for k in range(K)])
    vecs = outputVectors[indices,:]
    
    t=sigmoid(vecs.dot(predicted)*labels)
    cost = -np.sum(np.log(t))
    
    delta = labels*(t-1)
    gradPred = delta.reshape((1,K+1)).dot(vecs).flatten()
    gradtemp = delta.reshape((K+1,1)).dot(predicted.reshape((1,predicted.shape[0])))
    for k in range(K+1):
        grad[indices[k]]+=gradtemp[k,:]
    
    t = sigmoid(predicted.dot(outputVectors[target,:]))
    cost = -np.log(t)
    delta = t - 1
    
    gradPred+=delta*outputVectors[target,:]
    grad[target,:] += delta*predicted
    
    for k in range(K):
        idx = dataset.sampleTokenIdx()
        
        t = sigmoid(-predicted.dot(outputVectors[idx,:]))
        cost += -np.log(t)
        delta = 1-t
        
        gradPred += delta*outputVectors[idx,:]
        grad[idx,:]+=delta*predicted
    
    return cost,gradPred,grad

def skipgram(currentWord,C,contextWords,tokens,inputVectors,outputVectors,
             dataset,word2vecCostAndGradient=softmaxCostAndGradient):
    """
        Skip-gram model in word2vec
        
        skip-gram模型的實現
        輸入:
        currentWord:當前中心詞所對應的串
        C:上下文大小(詞窗大小)
        contextWords:最多2*C個詞
        tokens:對應詞向量中詞下標的字典
        inputVectors:"input" word vectors (as rows) for all tokens
        outputVectors: "output" word vectors (as rows) for all tokens
        word2vecCostAndGradient: the cost and gradient function for a prediction vector
                                 given the target word vectors,
                                 could be one of the two cost functions you implemented
                                 above
                                 
        輸出:
        cost:skip-gram模型算得的損失值
        grad:詞向量對應的梯度
    """
    currentI=tokens[currentWord]
    predicted = inputVectors[currentI,:]
    
    cost=0.0
    gradIn = np.zeros(inputVectors.shape)
    gradOut = np.zeros(outputVectors.shape)
    for cwd in contextWords:
        idx = tokens[cwd]
        cc,gp,gg = word2vecCostAndGradient(predicted,idx,outputVectors,dataset)
        cost+=cc
        gradOut += gg
        gradIn[currentI,:]+=gp
    return cost,gradIn,gradOut

def word2vec_sgd_wrapper(word2vecModel,tokens,wordVectors,dataset,C,word2vecCostAndGradient=softmaxCostAndGradient):
    batchsize = 50
    cost =0.0
    grad = np.zeros(wordVectors.shape)
    N=wordVectors.shape[0]
    splitIndex=int(N/2)
    inputVectors = wordVectors[:splitIndex,:]
    outputVectors = wordVectors[splitIndex:,:]
    for i in range(batchsize):
        C1 = random.randint(1,C)
        centerword,context=dataset.getRandomContext(C1)
        
        if word2vecModel == skipgram:
            denom =1
        else:
            denom =1
        
        c,gin,gout = word2vecModel(centerword,C1,context,tokens,inputVectors,outputVectors,dataset,word2vecCostAndGradient)
        cost+=c/batchsize/denom
        grad[:splitIndex,:]+=gin/batchsize/denom
        grad[splitIndex:,:]+=gout/batchsize/denom
     
    return cost, grad

def test_word2vec():
    """
         Interface to the dataset for negative sampling
    """
 
    dataset = type('dummy',(),{})()
    def dummySampleTokenIdx():
        return random.randint(0,4)
     
    def getRandomContext(C):
        tokens = ["a","b","c","d","e"]
        return tokens[random.randint(0,4)],[tokens[random.randint(0,4)] for i in range(2*C)]
     
    dataset.sampleTokenIdx = dummySampleTokenIdx
    dataset.getRandomContext = getRandomContext
     
    random.seed(31415)
    np.random.seed(9265)
    dummy_vectors = normalizeRows(np.random.randn(10,3))
    dummy_tokens = dict([("a",0),("b",1),("c",2),("d",3),("e",4)])
    print("===== Gradient check for skip-gram ======")
    #gradcheck_naive(lambda vec:word2vec_sgd_wrapper(skipgram,dummy_tokens,vec,dataset,5),dummy_vectors)
    gradcheck_naive(lambda vec:word2vec_sgd_wrapper(skipgram,dummy_tokens,vec,dataset,5,negSamplingCostAndGradient),dummy_vectors)
    print("\n===== Gradient check for CBOW")
    #gradcheck_naive(lambda vec:word2vec_sgd_wrapper(cbow,dummy_tokens,vec,dataset,5),dummy_vectors)
    #gradcheck_naive(lambda vec:word2vec_sgd_wrapper(cbow,dummy_tokens,vec,dataset,5,negSamplingCostAndGradient),dummy_vectors)
     
    print("\n===== Results ===")
    print(skipgram("c",3,["a","b","e","d","b","c"],dummy_tokens,dummy_vectors[:5,:],dummy_vectors[5:,:],dataset))
    #print(skipgram("c",1,["a","b"],dummy_tokens,dummy_vectors[:5,:],dummy_vectors[5:,:],dataset,negSamplingCostAndGradient))
    #print(cbow("a",2,["a","b","c","a"],dummy_tokens,dummy_vectors[:5,:],dummy_vectors[5:,:],dataset))
    #print(cbow("a",2,["a","b","a","c"],dummy_tokens,dummy_vectors[:5,:],dummy_vectors[5:,:],dataset,negSamplingCostAndGradient))
     
if __name__=="__main__":
    test_normalize_rows()
    test_word2vec()
    
        

下面的模型是word2vector模型的訓練，真正的純手寫版的，沒有利用任何包。

(看來我還是默默地再天朝搬磚吧，感覺一旦進了斯坦福都畢業不了)

'''
Created on 2017年9月17日

@author: weizhen
'''
import random
import numpy as np
from q3_word2vec import skipgram,word2vec_sgd_wrapper,negSamplingCostAndGradient
from q3_sgd import sgd,load_saved_params
from data_utils import StanfordSentiment
# Reset the random seed to make sure that everyone gets the same results
dataset = StanfordSentiment()
tokens = dataset.tokens()
nWords = len(tokens)

#going to train 10-dimensional vectors for this assignment
dimVectors = 10

#Context size
C=5
# Train word vectors (this could take a while!)

# Reset the random seed to make sure that everyone gets the same results
random.seed(31415)
np.random.seed(9265)
wordVectors = np.concatenate(((np.random.rand(nWords, dimVectors) - .5) / dimVectors, 
                              np.zeros((nWords, dimVectors))), axis=0)
wordVectors0 = sgd(lambda vec: word2vec_sgd_wrapper(skipgram, tokens, vec, dataset, C, negSamplingCostAndGradient), 
                   wordVectors, 0.3, 40000, None, True, PRINT_EVERY=10)
# sanity check: cost at convergence should be around or below 10

# sum the input and output word vectors

#st, wordVectors0, state2 = load_saved_params();
wordVectors = (wordVectors0[:int(nWords),:] + wordVectors0[int(nWords):,:])

print("\n=== For autograder ===")
checkWords = ["the", "a", "an", "movie", "ordinary", "but", "and"]
checkIdx = [tokens[word] for word in checkWords]
checkVecs = wordVectors[checkIdx, :]
print(checkVecs)

輸出的log如下所示

純手寫版的，真的是山外有山，人外有人呀。

既然tensorflow一行程式碼就行實現，那麼為啥還要，自己寫呢？

不為啥，就為了給自己裝逼用。哈哈哈