PTB資料集上用迴圈神經網路實現語言建模
阿新 • • 發佈:2019-02-17
一:PTB文字資料集是語言模型學習中目前最廣泛的資料集,tensorflow對ptb資料集是支援的,首先要下載資料集http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz,資料集中我們只需要利用data中的ptb.test.txt, ptb.train.txt. ptb.valid.txt三個資料檔案,這三個資料檔案是已經經過預處理的,包含10000個不同的詞語和語句結束識別符號。
二:tensorflow提供了ptb_raw_data函式來讀取原始資料,並將原始資料中的單詞轉化為單詞ID。雖然rnn可以接受任意長度的序列,但是訓練時需要將序列按照某個固定的長度來截斷,注意這裡是在訓練的時候,測試時可以為任意長度,tensorflow是利用ptb_iterator函式來實現的:
import reader
DATA_PATH="/home/yang/PycharmProjects/nlp/simple-examples/data"
train_data,valid_data,test_data,_=reader.ptb_raw_data(DATA_PATH)
result=reader.ptb_iterator(train_data,4,5)
x,y=result.next()
print "X:",x
print "Y:",y
X: [[ 6 709 947 1557 1]
[5966 31 8066 36 501]
[ 8 3 3 8 554]
[ 58 245 4168 214 5]]
Y: [[ 709 947 1557 1 1028]
[ 31 8066 36 501 5]
[ 3 3 8 554 16]
[ 245 4168 214 5 23]]
ptb_iterator函式的功能會將一個長序列劃分為batch_size段,batch_size就是batch有多少個,num_step就是每個batch大小是多少,這裡batch_size=4,num_step=5,
三:接下來我們開始使用迴圈神經網路實現語言模型:
1:
在這之前我們先來建一個reader.py,因為我們要匯入import reader函式來實現很多word_id等等等的功能
import os
import numpy as np
import tensorflow as tf
def _read_words(filename):
with tf.gfile.GFile(filename, "r") as f:
return f.read().replace("\n", "<eos>").split()
def _build_vocab(filename):
data = _read_words(filename)
counter = collections.Counter(data)
count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
words, _ = list(zip(*count_pairs))
word_to_id = dict(zip(words, range(len(words))))
return word_to_id
def _file_to_word_ids(filename, word_to_id):
data = _read_words(filename)
return [word_to_id[word] for word in data]
def ptb_raw_data(data_path=None):
train_path = os.path.join(data_path, "ptb.train.txt")
valid_path = os.path.join(data_path, "ptb.valid.txt")
test_path = os.path.join(data_path, "ptb.test.txt")
word_to_id = _build_vocab(train_path)
train_data = _file_to_word_ids(train_path, word_to_id)
valid_data = _file_to_word_ids(valid_path, word_to_id)
test_data = _file_to_word_ids(test_path, word_to_id)
vocabulary = len(word_to_id)
return train_data, valid_data, test_data, vocabulary
def ptb_iterator(raw_data, batch_size, num_steps):
raw_data = np.array(raw_data, dtype=np.int32)
data_len = len(raw_data)
batch_len = data_len // batch_size
data = np.zeros([batch_size, batch_len], dtype=np.int32)
for i in range(batch_size):
data[i] = raw_data[batch_len * i:batch_len * (i + 1)]
epoch_size = (batch_len - 1) // num_steps
if epoch_size == 0:
raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
for i in range(epoch_size):
x = data[:, i*num_steps:(i+1)*num_steps]
y = data[:, i*num_steps+1:(i+1)*num_steps+1]
yield (x, y)
2:
現在我們語言模型開始開工:
import numpy as np
import tensorflow as tf
import reader
from tensorflow.contrib import rnn
DATA_PATH = "/home/yang/PycharmProjects/nlp/simple-examples/data"
HIDDEN_SIZE = 200
NUM_LAYERS = 2
VOCAB_SIZE = 10000
LEARNING_RATE = 1.0
TRAIN_BATCH_SIZE = 20
TRAIN_NUM_STEP = 35
EVAL_BATCH_SIZE = 1
EVAL_NUM_STEP = 1
NUM_EPOCH = 2
KEEP_PROB = 0.5
MAX_GRAD_NORM = 5
def LstmCell(is_training):
lstm_cell = rnn.BasicLSTMCell(HIDDEN_SIZE, reuse=tf.get_variable_scope().reuse)
if is_training:
lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=KEEP_PROB)
return lstm_cell
class PTBModel(object):
def __init__(self, is_training, batch_size, num_steps):
self.batch_size = batch_size
self.num_steps = num_steps
self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps])
self.targets = tf.placeholder(tf.int32, [batch_size, num_steps])
cell = rnn.MultiRNNCell([LstmCell(is_training) for _ in range(NUM_LAYERS)])
self.initial_state = cell.zero_state(batch_size, tf.float32)
embedding = tf.get_variable("embedding", [VOCAB_SIZE, HIDDEN_SIZE])
inputs = tf.nn.embedding_lookup(embedding, self.input_data)
if is_training:
inputs = tf.nn.dropout(inputs, KEEP_PROB)
outputs = []
state = self.initial_state
with tf.variable_scope("RNN"):
for time_step in range(num_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
cell_output, state = cell(inputs[:, time_step, :], state)
outputs.append(cell_output)
output = tf.reshape(tf.concat(outputs, 1), [-1, HIDDEN_SIZE])
weight = tf.get_variable("weight", [HIDDEN_SIZE, VOCAB_SIZE])
bias = tf.get_variable("bias", [VOCAB_SIZE])
logits = tf.matmul(output, weight) + bias
loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
[logits],
[tf.reshape(self.targets, [-1])],
[tf.ones([batch_size * num_steps], dtype=tf.float32)]
)
self.cost = tf.reduce_sum(loss) / batch_size
self.final_state = state
if not is_training:
return
trainable_variables = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, trainable_variables), MAX_GRAD_NORM)
optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE)
self.train_op = optimizer.apply_gradients(zip(grads, trainable_variables))
def run_epoch(session, model, data, train_op, output_log):
total_costs = 0.0
iters = 0
state = session.run(model.initial_state)
for step, (x, y) in enumerate(reader.ptb_iterator(data, model.batch_size, model.num_steps)):
cost, state, _ = session.run([model.cost, model.final_state, train_op],
{model.input_data: x, model.targets: y,
model.initial_state: state})
total_costs += cost
iters += model.num_steps
if output_log and step % 100 == 0:
print("after % step ,perplexity is %.3f" % (step, np.exp(total_costs / iters)))
return np.exp(total_costs / iters)
def main(_):
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
initializer = tf.random_uniform_initializer(-0.05, 0.05)
with tf.variable_scope("language_model", reuse=None, initializer=initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
with tf.variable_scope("language_model", reuse=True, initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
with tf.Session() as session:
tf.global_variables_initializer().run()
for i in range(NUM_EPOCH):
print("In iteration:%d" % (i + 1))
run_epoch(session, train_model, train_data, train_model.train_op, True)
valid_perplexity = run_epoch(session, eval_model, valid_data, tf.no_op(), False)
print("Epoch: %d Validation Perplexity : %.3f" % (i + 1, valid_perplexity))
test_perplexity = run_epoch(session, eval_model, test_data, tf.no_op(), False)
print("Test Perplexity:%.3f" % test_perplexity)
if __name__ == "__main__":
tf.app.run()
程式碼我就不去講解了,程式碼是簡單的關鍵是原理我們必須得弄明白:
In iteration:1
after 0tep ,perplexity is 10016.963
Epoch: 1 Validation Perplexity : 2214.258
In iteration:2
after 0tep ,perplexity is 6029.708
Epoch: 2 Validation Perplexity : 426.626
Test Perplexity:426.631
這裡需要知道自然語言模型效果好壞的評價指標就是複雜度(perplxity)。perplxity值刻畫的就是某個語言模型估計的一句話出現的概率。perplxity的值越小模型越好,也就是這句話出現的概率越高越好,出現這句話的概率和perplxitty是成反比例的。
這裡剛開始perplxity的值為10016.多,出現某一句話的概率一定很小的,但我們Epoch等於2的時候維426點多,此時的概率明顯比10016點多色時候概率要大的多。
二:tensorflow提供了ptb_raw_data函式來讀取原始資料,並將原始資料中的單詞轉化為單詞ID。雖然rnn可以接受任意長度的序列,但是訓練時需要將序列按照某個固定的長度來截斷,注意這裡是在訓練的時候,測試時可以為任意長度,tensorflow是利用ptb_iterator函式來實現的:
import reader
DATA_PATH="/home/yang/PycharmProjects/nlp/simple-examples/data"
train_data,valid_data,test_data,_=reader.ptb_raw_data(DATA_PATH)
result=reader.ptb_iterator(train_data,4,5)
x,y=result.next()
print "X:",x
print "Y:",y
X: [[ 6 709 947 1557 1]
[5966 31 8066 36 501]
[ 8 3 3 8 554]
[ 58 245 4168 214 5]]
Y: [[ 709 947 1557 1 1028]
[ 31 8066 36 501 5]
[ 3 3 8 554 16]
[ 245 4168 214 5 23]]
ptb_iterator函式的功能會將一個長序列劃分為batch_size段,batch_size就是batch有多少個,num_step就是每個batch大小是多少,這裡batch_size=4,num_step=5,
三:接下來我們開始使用迴圈神經網路實現語言模型:
1:
在這之前我們先來建一個reader.py,因為我們要匯入import reader函式來實現很多word_id等等等的功能
import os
import numpy as np
import tensorflow as tf
def _read_words(filename):
with tf.gfile.GFile(filename, "r") as f:
return f.read().replace("\n", "<eos>").split()
def _build_vocab(filename):
data = _read_words(filename)
counter = collections.Counter(data)
count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
words, _ = list(zip(*count_pairs))
word_to_id = dict(zip(words, range(len(words))))
return word_to_id
def _file_to_word_ids(filename, word_to_id):
data = _read_words(filename)
return [word_to_id[word] for word in data]
def ptb_raw_data(data_path=None):
train_path = os.path.join(data_path, "ptb.train.txt")
valid_path = os.path.join(data_path, "ptb.valid.txt")
test_path = os.path.join(data_path, "ptb.test.txt")
word_to_id = _build_vocab(train_path)
train_data = _file_to_word_ids(train_path, word_to_id)
valid_data = _file_to_word_ids(valid_path, word_to_id)
test_data = _file_to_word_ids(test_path, word_to_id)
vocabulary = len(word_to_id)
return train_data, valid_data, test_data, vocabulary
def ptb_iterator(raw_data, batch_size, num_steps):
raw_data = np.array(raw_data, dtype=np.int32)
data_len = len(raw_data)
batch_len = data_len // batch_size
data = np.zeros([batch_size, batch_len], dtype=np.int32)
for i in range(batch_size):
data[i] = raw_data[batch_len * i:batch_len * (i + 1)]
epoch_size = (batch_len - 1) // num_steps
if epoch_size == 0:
raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
for i in range(epoch_size):
x = data[:, i*num_steps:(i+1)*num_steps]
y = data[:, i*num_steps+1:(i+1)*num_steps+1]
yield (x, y)
2:
現在我們語言模型開始開工:
import numpy as np
import tensorflow as tf
import reader
from tensorflow.contrib import rnn
DATA_PATH = "/home/yang/PycharmProjects/nlp/simple-examples/data"
HIDDEN_SIZE = 200
NUM_LAYERS = 2
VOCAB_SIZE = 10000
LEARNING_RATE = 1.0
TRAIN_BATCH_SIZE = 20
TRAIN_NUM_STEP = 35
EVAL_BATCH_SIZE = 1
EVAL_NUM_STEP = 1
NUM_EPOCH = 2
KEEP_PROB = 0.5
MAX_GRAD_NORM = 5
def LstmCell(is_training):
lstm_cell = rnn.BasicLSTMCell(HIDDEN_SIZE, reuse=tf.get_variable_scope().reuse)
if is_training:
lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=KEEP_PROB)
return lstm_cell
class PTBModel(object):
def __init__(self, is_training, batch_size, num_steps):
self.batch_size = batch_size
self.num_steps = num_steps
self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps])
self.targets = tf.placeholder(tf.int32, [batch_size, num_steps])
cell = rnn.MultiRNNCell([LstmCell(is_training) for _ in range(NUM_LAYERS)])
self.initial_state = cell.zero_state(batch_size, tf.float32)
embedding = tf.get_variable("embedding", [VOCAB_SIZE, HIDDEN_SIZE])
inputs = tf.nn.embedding_lookup(embedding, self.input_data)
if is_training:
inputs = tf.nn.dropout(inputs, KEEP_PROB)
outputs = []
state = self.initial_state
with tf.variable_scope("RNN"):
for time_step in range(num_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
cell_output, state = cell(inputs[:, time_step, :], state)
outputs.append(cell_output)
output = tf.reshape(tf.concat(outputs, 1), [-1, HIDDEN_SIZE])
weight = tf.get_variable("weight", [HIDDEN_SIZE, VOCAB_SIZE])
bias = tf.get_variable("bias", [VOCAB_SIZE])
logits = tf.matmul(output, weight) + bias
loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
[logits],
[tf.reshape(self.targets, [-1])],
[tf.ones([batch_size * num_steps], dtype=tf.float32)]
)
self.cost = tf.reduce_sum(loss) / batch_size
self.final_state = state
if not is_training:
return
trainable_variables = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, trainable_variables), MAX_GRAD_NORM)
optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE)
self.train_op = optimizer.apply_gradients(zip(grads, trainable_variables))
def run_epoch(session, model, data, train_op, output_log):
total_costs = 0.0
iters = 0
state = session.run(model.initial_state)
for step, (x, y) in enumerate(reader.ptb_iterator(data, model.batch_size, model.num_steps)):
cost, state, _ = session.run([model.cost, model.final_state, train_op],
{model.input_data: x, model.targets: y,
model.initial_state: state})
total_costs += cost
iters += model.num_steps
if output_log and step % 100 == 0:
print("after % step ,perplexity is %.3f" % (step, np.exp(total_costs / iters)))
return np.exp(total_costs / iters)
def main(_):
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
initializer = tf.random_uniform_initializer(-0.05, 0.05)
with tf.variable_scope("language_model", reuse=None, initializer=initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
with tf.variable_scope("language_model", reuse=True, initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
with tf.Session() as session:
tf.global_variables_initializer().run()
for i in range(NUM_EPOCH):
print("In iteration:%d" % (i + 1))
run_epoch(session, train_model, train_data, train_model.train_op, True)
valid_perplexity = run_epoch(session, eval_model, valid_data, tf.no_op(), False)
print("Epoch: %d Validation Perplexity : %.3f" % (i + 1, valid_perplexity))
test_perplexity = run_epoch(session, eval_model, test_data, tf.no_op(), False)
print("Test Perplexity:%.3f" % test_perplexity)
if __name__ == "__main__":
tf.app.run()
程式碼我就不去講解了,程式碼是簡單的關鍵是原理我們必須得弄明白:
In iteration:1
after 0tep ,perplexity is 10016.963
Epoch: 1 Validation Perplexity : 2214.258
In iteration:2
after 0tep ,perplexity is 6029.708
Epoch: 2 Validation Perplexity : 426.626
Test Perplexity:426.631
這裡需要知道自然語言模型效果好壞的評價指標就是複雜度(perplxity)。perplxity值刻畫的就是某個語言模型估計的一句話出現的概率。perplxity的值越小模型越好,也就是這句話出現的概率越高越好,出現這句話的概率和perplxitty是成反比例的。
這裡剛開始perplxity的值為10016.多,出現某一句話的概率一定很小的,但我們Epoch等於2的時候維426點多,此時的概率明顯比10016點多色時候概率要大的多。