根據自己的理解寫的讀書筆記。

import collections
import math
import os
import random
import zipfile
import urllib
import numpy as np
import tensorflow as tf

#定義下載文字資料的函式
# url = 'http://mattmahoney.net/dc/'
#
# def maybe_download(filename,expected_bytes):
#     if not os.path.exists(filename):
#         filename,_ = urllib.request.urlretrieve(url + filename,filename)
 
#     statinfo = os.stat(filename)  #訪問一個檔案的詳細資訊。
#     if statinfo.st_size == expected_bytes:  #檔案大小(以位元組為單位)
#         print('Found and verified(驗證）',filename)
#     else:
#         print(statinfo.st_size)
#         raise Exception('Failed to verify(驗證）' + filename + 'Can you get to it with a browser(瀏覽器)?')
 
#     return filename
#
# filename = maybe_download('text8.zip',31344016)
filename = './text8.zip'
#解壓檔案，並將資料轉化成單詞的列表
def read_data(filename):
    with zipfile.ZipFile(filename) as f:
        #獲得名字列表，讀取成字串，編碼成'utf-8'，最後進行分割
data = tf.compat.as_str(f.read(f.namelist()[0])).split()
    return data

words = read_data(filename)
# print('Data size',len(words))
 
# print(words)
#建立詞彙表，將出現最多的50000個單詞作為詞彙表，放入字典中。
vocabulary_size = 50000
def build_dataset(words):
    count = [['UNK',-1]]
    count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
    # c=collections.Counter(words).most_common(10)
    # print(c)
    # count.extend(c)
    # print(count)  #[['UNK', -1], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764), ('in', 372201), ('a', 325873), ('to', 316376), ('zero', 264975), ('nine', 250430), ('two', 192644)]
dictionary = dict()#新建空字典
for word,_ in count:
        dictionary[word] = len(dictionary)
    # print(dictionary)  #{'UNK': 0, 'the': 1, 'of': 2, 'and': 3, 'one': 4, 'in': 5, 'a': 6, 'to': 7, 'zero': 8, 'nine': 9, 'two': 10}
data = list()
    unk_count = 0#未知單詞數量
for word in words:#單詞索引，不在字典中，則索引為0
if word in dictionary:
            index = dictionary[word]
        else:
            index = 0
unk_count += 1
data.append(index)
    count[0][1] = unk_count
    reverse_dictionary = dict(zip(dictionary.values(),dictionary.keys()))
    return data,count,dictionary,reverse_dictionary

data,count,dictionary,reverse_dictionary = build_dataset(words)

#刪除原始單詞列表，節約記憶體。列印詞彙表，瞭解詞頻
del words
# print('Most common words (+UNK)',count[:5])
# print('Sample data',data[:10],[reverse_dictionary[i] for i in data[:10]])
#以上程式碼為資料處理，得到單詞的詞頻和在字典中的索引
#skip-gram模式：從目標單詞反推語境
data_index = 0
#生成訓練用的batch資料
#batch_size為batch大小，num_skips為對每個單詞生成樣本數，skip_window為單詞最遠可以聯絡的距離
def generate_batch(batch_size,num_skips,skip_window):
    global data_index  #宣告全域性變數
assert batch_size % num_skips == 0#斷言batch_size是num_skips的整倍數
assert num_skips <= 2 * skip_window#斷言num_skips不大於skip_window的兩倍
batch = np.ndarray(shape=(batch_size),dtype=np.int32)#初始化為陣列
labels = np.ndarray(shape=(batch_size,1),dtype=np.int32)
    span = 2 * skip_window + 1  #對某個單詞建立相關樣本時會使用到的單詞數量
buffer = collections.deque(maxlen=span)  #建立最大容量為span的佇列，即雙向佇列
for _ in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    for i in range(batch_size // num_skips):#'//'取商的整數部分
target = skip_window
        targets_to_avoid = [skip_window]#因為要預測語境單詞，不包括目標單詞本身。所以需要一個避免列表
for j in range(num_skips):
            while target in targets_to_avoid:
                target = random.randint(0, span - 1)
            targets_to_avoid.append(target)
            batch[i * num_skips + j] = buffer[skip_window]
            labels[i * num_skips + j, 0] = buffer[target]
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    return batch,labels


# batch,labels = generate_batch(batch_size=8,num_skips=2,skip_window=1)
# print(batch)#[3081 3081   12   12    6    6  195  195]
# print(labels)#[[5234]
#              # [  12]
#              # [3081]
#              # [   6]
#              # [  12]
#              # [ 195]
#              # [   6]
#              # [   2]]
# for i in range(8):
#     print(batch[i],reverse_dictionary[batch[i]],'->',labels[i,0],reverse_dictionary[labels[i,0]])
batch_size = 128
embedding_size = 128#將單詞轉為稠密向量的維度，一般在50~1000範圍
skip_window = 1
num_skips = 2
valid_size = 16
valid_window = 100
valid_examples = np.random.choice(valid_window,valid_size,replace=False)#生成驗證資料，隨機抽取詞頻最高（前valid_window）的valid_size個單詞
num_sampled = 64#做負樣本的噪聲單詞數量
#定義skip-gram網路結構
graph = tf.Graph()
with graph.as_default():

    train_inputs = tf.placeholder(tf.int32,shape=[batch_size])
    train_labels = tf.placeholder(tf.int32,shape=[batch_size,1])
    valid_dataset = tf.constant(valid_examples,dtype=tf.int32)

#限定所有計算都在cpu上執行，因為接下來一些計算操作在GPU上可能還沒有實現
with tf.device('/cpu:0'):
        embeddings = tf.Variable(tf.random_uniform([vocabulary_size,embedding_size],-1.0,1.0))#隨機生成所有單詞的詞向量，單詞表大小50000，維度128
embed = tf.nn.embedding_lookup(embeddings,train_inputs)#查詢輸入train_inputs在embeddings裡對應的向量
        #用截斷正態分佈truncated_normal初始化NCE Loss中的權重引數nce_weights，並將其初始化為0
nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size,embedding_size],stddev=1.0/math.sqrt(embedding_size)))
        nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

    loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weights,biases=nce_biases,labels=train_labels,inputs=embed,num_sampled=num_sampled,num_classes=vocabulary_size))

    #優化器SGD，學習率1.0
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
    #先計算embeddings的平方，並按第二維降維到1，計算嵌入向量embeddings的L2範數
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings),1,keep_dims=True))
    #標準化embeddings
normalized_embeddings = embeddings/norm
    #查詢單詞的嵌入向量，並計算驗證單詞的嵌入向量與詞彙表中所有單詞的相似性
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,valid_dataset)
    #transpose_b=True  將b轉置
similarity = tf.matmul(valid_embeddings,normalized_embeddings,transpose_b=True)
    #初始化所有模型引數
init = tf.global_variables_initializer()

num_steps = 100001#迭代10萬次
with tf.Session(graph=graph) as session:
    init.run()
    print('Initialized')

    average_loss = 0
for step in range(num_steps):
        batch_inputs,batch_labels = generate_batch(batch_size,num_skips,skip_window)
        feed_dict = {train_inputs : batch_inputs,train_labels : batch_labels}

        _,loss_val = session.run([optimizer,loss],feed_dict=feed_dict)

        average_loss += loss_val

        if step % 2000 == 0:
            if step > 0:

                average_loss /= 2000
print('Average loss at step ',step,': ',average_loss)
            average_loss = 0
if step % 10000 == 0:
            sim = similarity.eval()
            for i in range(valid_size):
                valid_word = reverse_dictionary[valid_examples[i]]
                top_k = 8
nearest = (-sim[i, :]).argsort()[1:top_k+1]#argsort將陣列從小到大排列，並返回索引
log_str = 'Nearest to %s:' % valid_word
                for k in range(top_k):
                    close_word = reverse_dictionary[nearest[k]]
                    log_str = '%s %s,' % (log_str,close_word)
                print(log_str)
    final_embeddings = normalized_embeddings.eval()


from sklearn.manifold import TSNE#此降維演算法比PCA更高階，視覺化
import matplotlib.pyplot as plt

def plot_with_labels(low_dim_embs,labels,filename='tsne.png'):
    assert  low_dim_embs.shape[0] >= len(labels),'More labels than embeddings'
plt.figure(figsize=(18,18))
    for i,label in enumerate(labels):#enumerate列舉可遍歷、迭代（列表、字串）物件，加上索引
x,y = low_dim_embs[i,:]
        plt.scatter(x,y)#顯示散點圖
        #（工具書p242）annotate在圖上添加註釋，xy設定箭頭所指處的座標，xytext註釋內容座標，textcoords註釋內容座標的座標變換方式。
        #'offset points'以點為單位，相對於點xy的座標
        # ha='right'點在註釋右邊（right,center,left），va='bottom'點在註釋底部('top', 'bottom', 'center', 'baseline')
plt.annotate(label,xy=(x,y),xytext=(5,2),textcoords='offset points',ha='right',va='bottom')

    plt.savefig(filename)

#perplexity（混亂，複雜）與最近鄰數有關，一般在5~50，n_iter達到最優化所需的最大迭代次數，應當不少於250次
#init='pca'pca初始化比random穩定，n_components嵌入空間的維數（即降到2維，預設為2
tsne = TSNE(perplexity=30,n_components=2,init='pca',n_iter=5000)
plot_only = 100#顯示詞頻最高的一百個
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only,:])
labels = [reverse_dictionary[i] for i in range(plot_only)]
plot_with_labels(low_dim_embs,labels)

# plt.show()

筆記

tf.compat(compat相容性)

NAME

    tensorflow.python.util.compat - Functions for Python 2 vs. 3 compatibility.

DESCRIPTION

    ## Conversion routines

    In addition to the functions below, `as_str` converts an object to a `str`.

    @@as_bytes

    @@as_text

    @@as_str_any

    ## Types

    The compatibility module also provides the following types:

    * `bytes_or_text_types`

    * `complex_types`

    * `integral_types`

    * `real_types`

FUNCTIONS

    as_bytes(bytes_or_text, encoding=’utf-8’)

        Converts either bytes or unicode to `bytes`, using utf-8 encoding for text.

        Args:

          bytes_or_text: A `bytes`, `str`, or `unicode` object.

          encoding: A string indicating the charset for encoding unicode.

        Returns:

          A `bytes` object.

        Raises:

          TypeError: If `bytes_or_text` is not a binary or unicode string.

    as_str = as_text(bytes_or_text, encoding=’utf-8’)

        Returns the given argument as a unicode string.

        Args:

          bytes_or_text: A `bytes`, `str`, or `unicode` object.

          encoding: A string indicating the charset for decoding unicode.

        Returns:

          A `unicode` (Python 2) or `str` (Python 3) object.

        Raises:

          TypeError: If `bytes_or_text` is not a binary or unicode string.

    as_str_any(value)

        Converts to `str` as `str(value)`, but use `as_str` for `bytes`.

        Args:

          value: A object that can be converted to `str`.

        Returns:

          A `str` object.

    as_text(bytes_or_text, encoding=’utf-8’)

        Returns the given argument as a unicode string.

        Args:

          bytes_or_text: A `bytes`, `str`, or `unicode` object.

          encoding: A string indicating the charset for decoding unicode.

        Returns:

          A `unicode` (Python 2) or `str` (Python 3) object.

        Raises:

          TypeError: If `bytes_or_text` is not a binary or unicode string.

DATA

    bytes_or_text_types = (<class ‘bytes’>, <class ‘str’>)

    complex_types = (<class ‘numbers.Complex’>, <class ‘numpy.number’>)

    integral_types = (<class ‘numbers.Integral’>, <class ‘numpy.integer’>)

real_types = (<class ‘numbers.Real’>, <class ‘numpy.integer’>, <class …

zipfile.ZipFile.namelist(self)

Return a list of file names in the archive(檔案檔案).

zipfile.ZipFile.read

read(self, name, pwd=None)

    Return file bytes (as a string) for name.

split()通過指定分隔符對字串進行切片，如果引數num 有指定值，則僅分隔 num 個子字串

str.split(str=’’,num=string.count(str))

str – 分隔符，預設為所有的空字元，包括空格、換行(\n)、製表符(\t)等。

num – 分割次數。

collections.Counter

跟蹤值出現的次數，以字典形勢儲存，元素做key，其計數做value.

>>> c = collections.Counter(‘abcdeabcdabcaba’)

>>> c

Counter({‘a’: 5, ‘b’: 4, ‘c’: 3, ‘d’: 2, ‘e’: 1})

>>> c.most_common(3)

[(‘a’, 5), (‘b’, 4), (‘c’, 3)]

most_common (List the n most common elements) 從多到少返回一個有前n多的元素的列表（list），如果n被忽略或者為none，返回所有元素，相同數量的元素次序任意。

collections.deque

使用list儲存資料時，按索引訪問元素很快，但是插入和刪除元素就很慢了，因為list是線性儲存，資料量大的時候，插入和刪除效率很低。

deque是為了高效實現插入和刪除操作的雙向列表，適合用於佇列和棧：

append(…)

 |      Add an element to the right side of the deque.

 |  

appendleft(…)

 |      Add an element to the left side of the deque.

tf.nn.embedding_lookup

embedding_lookup(params, ids, partition_strategy=’mod’, name=None, validate_indices=True, max_norm=None)

Looks up `ids` in a list of embedding tensors. 查詢輸入ids在params嵌入向量列表中的位置

norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings),1,keep_dims=True))

axis=1

keep_dims: If true, retains reduced dimensions with length 1.

Input_tensor：被降維的張量必須其數據型別必須被預先指定。reduction_indices：降維的維度如果為None(default)，則所有維度都要降維。keep_dims：如果keep_dims為true，則降維的尺寸將保留為1 name：降維操作的名字。返回一個降維後的張量。

計算L2範數