tensorflow實現Word2Vec(找到目標英文單詞的相近詞)
根據自己的理解寫的讀書筆記。
import collections import math import os import random import zipfile import urllib import numpy as np import tensorflow as tf #定義下載文字資料的函式 # url = 'http://mattmahoney.net/dc/' # # def maybe_download(filename,expected_bytes): # if not os.path.exists(filename): # filename,_ = urllib.request.urlretrieve(url + filename,filename)# statinfo = os.stat(filename) #訪問一個檔案的詳細資訊。 # if statinfo.st_size == expected_bytes: #檔案大小(以位元組為單位) # print('Found and verified(驗證)',filename) # else: # print(statinfo.st_size) # raise Exception('Failed to verify(驗證)' + filename + 'Can you get to it with a browser(瀏覽器)?')# return filename # # filename = maybe_download('text8.zip',31344016) filename = './text8.zip' #解壓檔案,並將資料轉化成單詞的列表 def read_data(filename): with zipfile.ZipFile(filename) as f: #獲得名字列表,讀取成字串,編碼成'utf-8',最後進行分割 data = tf.compat.as_str(f.read(f.namelist()[0])).split() return data words = read_data(filename) # print('Data size',len(words))# print(words) #建立詞彙表,將出現最多的50000個單詞作為詞彙表,放入字典中。 vocabulary_size = 50000 def build_dataset(words): count = [['UNK',-1]] count.extend(collections.Counter(words).most_common(vocabulary_size - 1)) # c=collections.Counter(words).most_common(10) # print(c) # count.extend(c) # print(count) #[['UNK', -1], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764), ('in', 372201), ('a', 325873), ('to', 316376), ('zero', 264975), ('nine', 250430), ('two', 192644)] dictionary = dict()#新建空字典 for word,_ in count: dictionary[word] = len(dictionary) # print(dictionary) #{'UNK': 0, 'the': 1, 'of': 2, 'and': 3, 'one': 4, 'in': 5, 'a': 6, 'to': 7, 'zero': 8, 'nine': 9, 'two': 10} data = list() unk_count = 0#未知單詞數量 for word in words:#單詞索引,不在字典中,則索引為0 if word in dictionary: index = dictionary[word] else: index = 0 unk_count += 1 data.append(index) count[0][1] = unk_count reverse_dictionary = dict(zip(dictionary.values(),dictionary.keys())) return data,count,dictionary,reverse_dictionary data,count,dictionary,reverse_dictionary = build_dataset(words) #刪除原始單詞列表,節約記憶體。列印詞彙表,瞭解詞頻 del words # print('Most common words (+UNK)',count[:5]) # print('Sample data',data[:10],[reverse_dictionary[i] for i in data[:10]]) #以上程式碼為資料處理,得到單詞的詞頻和在字典中的索引 #skip-gram模式:從目標單詞反推語境 data_index = 0 #生成訓練用的batch資料 #batch_size為batch大小,num_skips為對每個單詞生成樣本數,skip_window為單詞最遠可以聯絡的距離 def generate_batch(batch_size,num_skips,skip_window): global data_index #宣告全域性變數 assert batch_size % num_skips == 0#斷言batch_size是num_skips的整倍數 assert num_skips <= 2 * skip_window#斷言num_skips不大於skip_window的兩倍 batch = np.ndarray(shape=(batch_size),dtype=np.int32)#初始化為陣列 labels = np.ndarray(shape=(batch_size,1),dtype=np.int32) span = 2 * skip_window + 1 #對某個單詞建立相關樣本時會使用到的單詞數量 buffer = collections.deque(maxlen=span) #建立最大容量為span的佇列,即雙向佇列 for _ in range(span): buffer.append(data[data_index]) data_index = (data_index + 1) % len(data) for i in range(batch_size // num_skips):#'//'取商的整數部分 target = skip_window targets_to_avoid = [skip_window]#因為要預測語境單詞,不包括目標單詞本身。所以需要一個避免列表 for j in range(num_skips): while target in targets_to_avoid: target = random.randint(0, span - 1) targets_to_avoid.append(target) batch[i * num_skips + j] = buffer[skip_window] labels[i * num_skips + j, 0] = buffer[target] buffer.append(data[data_index]) data_index = (data_index + 1) % len(data) return batch,labels # batch,labels = generate_batch(batch_size=8,num_skips=2,skip_window=1) # print(batch)#[3081 3081 12 12 6 6 195 195] # print(labels)#[[5234] # # [ 12] # # [3081] # # [ 6] # # [ 12] # # [ 195] # # [ 6] # # [ 2]] # for i in range(8): # print(batch[i],reverse_dictionary[batch[i]],'->',labels[i,0],reverse_dictionary[labels[i,0]]) batch_size = 128 embedding_size = 128#將單詞轉為稠密向量的維度,一般在50~1000範圍 skip_window = 1 num_skips = 2 valid_size = 16 valid_window = 100 valid_examples = np.random.choice(valid_window,valid_size,replace=False)#生成驗證資料,隨機抽取詞頻最高(前valid_window)的valid_size個單詞 num_sampled = 64#做負樣本的噪聲單詞數量 #定義skip-gram網路結構 graph = tf.Graph() with graph.as_default(): train_inputs = tf.placeholder(tf.int32,shape=[batch_size]) train_labels = tf.placeholder(tf.int32,shape=[batch_size,1]) valid_dataset = tf.constant(valid_examples,dtype=tf.int32) #限定所有計算都在cpu上執行,因為接下來一些計算操作在GPU上可能還沒有實現 with tf.device('/cpu:0'): embeddings = tf.Variable(tf.random_uniform([vocabulary_size,embedding_size],-1.0,1.0))#隨機生成所有單詞的詞向量,單詞表大小50000,維度128 embed = tf.nn.embedding_lookup(embeddings,train_inputs)#查詢輸入train_inputs在embeddings裡對應的向量 #用截斷正態分佈truncated_normal初始化NCE Loss中的權重引數nce_weights,並將其初始化為0 nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size,embedding_size],stddev=1.0/math.sqrt(embedding_size))) nce_biases = tf.Variable(tf.zeros([vocabulary_size])) loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weights,biases=nce_biases,labels=train_labels,inputs=embed,num_sampled=num_sampled,num_classes=vocabulary_size)) #優化器SGD,學習率1.0 optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) #先計算embeddings的平方,並按第二維降維到1,計算嵌入向量embeddings的L2範數 norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings),1,keep_dims=True)) #標準化embeddings normalized_embeddings = embeddings/norm #查詢單詞的嵌入向量,並計算驗證單詞的嵌入向量與詞彙表中所有單詞的相似性 valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,valid_dataset) #transpose_b=True 將b轉置 similarity = tf.matmul(valid_embeddings,normalized_embeddings,transpose_b=True) #初始化所有模型引數 init = tf.global_variables_initializer() num_steps = 100001#迭代10萬次 with tf.Session(graph=graph) as session: init.run() print('Initialized') average_loss = 0 for step in range(num_steps): batch_inputs,batch_labels = generate_batch(batch_size,num_skips,skip_window) feed_dict = {train_inputs : batch_inputs,train_labels : batch_labels} _,loss_val = session.run([optimizer,loss],feed_dict=feed_dict) average_loss += loss_val if step % 2000 == 0: if step > 0: average_loss /= 2000 print('Average loss at step ',step,': ',average_loss) average_loss = 0 if step % 10000 == 0: sim = similarity.eval() for i in range(valid_size): valid_word = reverse_dictionary[valid_examples[i]] top_k = 8 nearest = (-sim[i, :]).argsort()[1:top_k+1]#argsort將陣列從小到大排列,並返回索引 log_str = 'Nearest to %s:' % valid_word for k in range(top_k): close_word = reverse_dictionary[nearest[k]] log_str = '%s %s,' % (log_str,close_word) print(log_str) final_embeddings = normalized_embeddings.eval() from sklearn.manifold import TSNE#此降維演算法比PCA更高階,視覺化 import matplotlib.pyplot as plt def plot_with_labels(low_dim_embs,labels,filename='tsne.png'): assert low_dim_embs.shape[0] >= len(labels),'More labels than embeddings' plt.figure(figsize=(18,18)) for i,label in enumerate(labels):#enumerate列舉可遍歷、迭代(列表、字串)物件,加上索引 x,y = low_dim_embs[i,:] plt.scatter(x,y)#顯示散點圖 #(工具書p242)annotate在圖上添加註釋,xy設定箭頭所指處的座標,xytext註釋內容座標,textcoords註釋內容座標的座標變換方式。 #'offset points'以點為單位,相對於點xy的座標 # ha='right'點在註釋右邊(right,center,left),va='bottom'點在註釋底部('top', 'bottom', 'center', 'baseline') plt.annotate(label,xy=(x,y),xytext=(5,2),textcoords='offset points',ha='right',va='bottom') plt.savefig(filename) #perplexity(混亂,複雜)與最近鄰數有關,一般在5~50,n_iter達到最優化所需的最大迭代次數,應當不少於250次 #init='pca'pca初始化比random穩定,n_components嵌入空間的維數(即降到2維,預設為2 tsne = TSNE(perplexity=30,n_components=2,init='pca',n_iter=5000) plot_only = 100#顯示詞頻最高的一百個 low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only,:]) labels = [reverse_dictionary[i] for i in range(plot_only)] plot_with_labels(low_dim_embs,labels) # plt.show()
筆記
tf.compat(compat相容性)
NAME
tensorflow.python.util.compat - Functions for Python 2 vs. 3 compatibility.
DESCRIPTION
## Conversion routines
In addition to the functions below, `as_str` converts an object to a `str`.
@@as_bytes
@@as_text
@@as_str_any
## Types
The compatibility module also provides the following types:
* `bytes_or_text_types`
* `complex_types`
* `integral_types`
* `real_types`
FUNCTIONS
as_bytes(bytes_or_text, encoding='utf-8')
Converts either bytes or unicode to `bytes`, using utf-8 encoding for text.
Args:
bytes_or_text: A `bytes`, `str`, or `unicode` object.
encoding: A string indicating the charset for encoding unicode.
Returns:
A `bytes` object.
Raises:
TypeError: If `bytes_or_text` is not a binary or unicode string.
as_str = as_text(bytes_or_text, encoding='utf-8')
Returns the given argument as a unicode string.
Args:
bytes_or_text: A `bytes`, `str`, or `unicode` object.
encoding: A string indicating the charset for decoding unicode.
Returns:
A `unicode` (Python 2) or `str` (Python 3) object.
Raises:
TypeError: If `bytes_or_text` is not a binary or unicode string.
as_str_any(value)
Converts to `str` as `str(value)`, but use `as_str` for `bytes`.
Args:
value: A object that can be converted to `str`.
Returns:
A `str` object.
as_text(bytes_or_text, encoding='utf-8')
Returns the given argument as a unicode string.
Args:
bytes_or_text: A `bytes`, `str`, or `unicode` object.
encoding: A string indicating the charset for decoding unicode.
Returns:
A `unicode` (Python 2) or `str` (Python 3) object.
Raises:
TypeError: If `bytes_or_text` is not a binary or unicode string.
DATA
bytes_or_text_types = (<class 'bytes'>, <class 'str'>)
complex_types = (<class 'numbers.Complex'>, <class 'numpy.number'>)
integral_types = (<class 'numbers.Integral'>, <class 'numpy.integer'>)
real_types = (<class 'numbers.Real'>, <class 'numpy.integer'>, <class ...
zipfile.ZipFile.namelist(self)
Return a list of file names in the archive(檔案檔案).
zipfile.ZipFile.read
read(self, name, pwd=None)
Return file bytes (as a string) for name.
split()通過指定分隔符對字串進行切片,如果引數num 有指定值,則僅分隔 num 個子字串
str.split(str=’’,num=string.count(str))
str -- 分隔符,預設為所有的空字元,包括空格、換行(\n)、製表符(\t)等。
num -- 分割次數。
collections.Counter
跟蹤值出現的次數,以字典形勢儲存,元素做key,其計數做value.
>>> c = collections.Counter('abcdeabcdabcaba')
>>> c
Counter({'a': 5, 'b': 4, 'c': 3, 'd': 2, 'e': 1})
>>> c.most_common(3)
[('a', 5), ('b', 4), ('c', 3)]
most_common (List the n most common elements) 從多到少返回一個有前n多的元素的列表(list),如果n被忽略或者為none,返回所有元素,相同數量的元素次序任意。
collections.deque
使用list儲存資料時,按索引訪問元素很快,但是插入和刪除元素就很慢了,因為list是線性儲存,資料量大的時候,插入和刪除效率很低。
deque是為了高效實現插入和刪除操作的雙向列表,適合用於佇列和棧:
append(...)
| Add an element to the right side of the deque.
|
appendleft(...)
| Add an element to the left side of the deque.
tf.nn.embedding_lookup
embedding_lookup(params, ids, partition_strategy='mod', name=None, validate_indices=True, max_norm=None)
Looks up `ids` in a list of embedding tensors. 查詢輸入ids在params嵌入向量列表中的位置
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings),1,keep_dims=True))
axis=1
keep_dims: If true, retains reduced dimensions with length 1.
Input_tensor:被降維的張量必須其數據型別必須被預先指定。reduction_indices:降維的維度如果為None(default),則所有維度都要降維。keep_dims:如果keep_dims為true,則降維的尺寸將保留為1 name:降維操作的名字。返回一個降維後的張量。
計算L2範數