[ MOOC課程學習 ] 人工智慧實踐：Tensorflow筆記_CH6_2 製作資料集
阿新 • • 發佈：2019-02-09
製作資料集
- tfrecords 檔案：
(1)tfrecords: 是一種二進位制檔案,可先將圖片和標籤製作成該格式的檔案。使用 tfrecords 進行資料讀取,會提高記憶體利用率。
(2)tf.train.Example: 用來儲存訓練資料。訓練資料的特徵用鍵值對的形式表示。
如:‘ img_raw ’ :值 ‘ label ’ :值
值是 Byteslist/FloatList/Int64List
(3)SerializeToString( ): 把資料序列化成字串儲存。 程式碼
mnist_generateds.py
mnist_forward.py
mnist_backward.py
mnist_test.py
mnist_app.py(1) 資料集生成讀取檔案 mnist_generateds.py
#coding:utf-8 import tensorflow as tf import numpy as np from PIL import Image import os image_train_path='./mnist_data_jpg/mnist_train_jpg_60000/' label_train_path='./mnist_data_jpg/mnist_train_jpg_60000.txt' tfRecord_train='./data/mnist_train.tfrecords' image_test_path='./mnist_data_jpg/mnist_test_jpg_10000/'label_test_path='./mnist_data_jpg/mnist_test_jpg_10000.txt' tfRecord_test='./data/mnist_test.tfrecords' data_path='./data' resize_height = 28 resize_width = 28 def write_tfRecord(tfRecordName, image_path, label_path): writer = tf.python_io.TFRecordWriter(tfRecordName) num_pic = 0 f = open(label_path, 'r') contents = f.readlines() f.close() for content in contents: value = content.split() img_path = image_path + value[0] img = Image.open(img_path) img_raw = img.tobytes() labels = [0] * 10 labels[int(value[1])] = 1 example = tf.train.Example(features=tf.train.Features(feature={ 'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw])), 'label': tf.train.Feature(int64_list=tf.train.Int64List(value=labels)) })) writer.write(example.SerializeToString()) num_pic += 1 print ("the number of picture:", num_pic) writer.close() print("write tfrecord successful") def generate_tfRecord(): isExists = os.path.exists(data_path) if not isExists: os.makedirs(data_path) print 'The directory was created successfully' else: print 'directory already exists' write_tfRecord(tfRecord_train, image_train_path, label_train_path) write_tfRecord(tfRecord_test, image_test_path, label_test_path) def read_tfRecord(tfRecord_path): filename_queue = tf.train.string_input_producer([tfRecord_path], shuffle=True) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example(serialized_example, features={ 'label': tf.FixedLenFeature([10], tf.int64), 'img_raw': tf.FixedLenFeature([], tf.string) }) img = tf.decode_raw(features['img_raw'], tf.uint8) img.set_shape([784]) img = tf.cast(img, tf.float32) * (1. / 255) label = tf.cast(features['label'], tf.float32) return img, label def get_tfrecord(num, isTrain=True): if isTrain: tfRecord_path = tfRecord_train else: tfRecord_path = tfRecord_test img, label = read_tfRecord(tfRecord_path) img_batch, label_batch = tf.train.shuffle_batch([img, label], batch_size = num, num_threads = 2, capacity = 1000, min_after_dequeue = 700) return img_batch, label_batch def main(): generate_tfRecord() if __name__ == '__main__': main()1)
filename_queue = tf.train.string_input_producer([tfRecord_path])# 該函式會生成一個先入先出的佇列,檔案閱讀器會使用它來讀取資料。 tf.train.string_input_producer( string_tensor, # 儲存影象和標籤資訊的 TFRecord 檔名列表 num_epochs=None, # 迴圈讀取的輪數(可選) shuffle=True, # 布林值(可選),如果為 True,則在每輪隨機打亂讀取順序 seed=None, # 隨機讀取時設定的種子(可選) capacity=32, # 設定佇列容量 shared_name=None, # (可選) 如果設定,該佇列將在多個會話中以給定名稱共享。所有具有此佇列的裝置都可以通過 shared_name 訪問它。在分散式設定中使用這種方法意味著每個名稱只能被訪問此操作的其中一個會話看到。 name=None, # 操作的名稱(可選) cancel_op=None # 取消佇列 )2)
_, serialized_example = reader.read(filename_queue)features = tf.parse_single_example(serialized_example,features={ 'img_raw': tf.FixedLenFeature([ ], tf.string) , 'label': tf.FixedLenFeature([10], tf.int64)}) # 把讀出的每個樣本儲存在 serialized_example 中進行解序列化, # 標籤和圖片的鍵名應該和製作 tfrecords 的鍵名相同,其中標籤給出幾分類。# 該函式可以將 tf.train.Example 協議記憶體塊(protocol buffer)解析為張量。 tf.parse_single_example( serialized, # 一個標量字串張量 features, # 一個字典對映功能鍵 FixedLenFeature 或 VarLenFeature值,也就是在協議記憶體塊中儲存的 name=None, example_names=None # 標量字串聯的名稱(可選) )3)
img_batch, label_batch = tf.train.shuffle_batch()# 這個函式隨機讀取一個 batch 的資料。 tf.train.shuffle_batch( tensors, # 待亂序處理的列表中的樣本(影象和標籤) batch_size, # 從佇列中提取的新批量大小 capacity, # 佇列中元素的最大數量 min_after_dequeue, # 出隊後佇列中的最小數量元素,用於確保元素的混合級別 num_threads=1, # 排列 tensors 的執行緒數 seed=None, # 用於佇列內的隨機洗牌 enqueue_many=False, # tensor 中的每個張量是否是一個例子 shapes=None, # 每個示例的形狀 allow_smaller_final_batch=False, # (可選)布林值。 如果為 True,則在佇列中剩餘數量不足時允許最終批次更小。 shared_name=None, # (可選)如果設定,該佇列將在多個會話中以給定名稱共享。 name=None # 操作的名稱(可選) )(2) mnist_forward.py
import tensorflow as tf INPUT_NODE = 784 OUTPUT_NODE = 10 LAYER1_NODE = 500 def get_weight(shape, regularizer): w = tf.Variable(tf.truncated_normal(shape,stddev=0.1)) if regularizer != None: tf.add_to_collection('losses', tf.contrib.layers.l2_regularizer(regularizer)(w)) return w def get_bias(shape): b = tf.Variable(tf.zeros(shape)) return b def forward(x, regularizer): w1 = get_weight([INPUT_NODE, LAYER1_NODE], regularizer) b1 = get_bias([LAYER1_NODE]) y1 = tf.nn.relu(tf.matmul(x, w1) + b1) w2 = get_weight([LAYER1_NODE, OUTPUT_NODE], regularizer) b2 = get_bias([OUTPUT_NODE]) y = tf.matmul(y1, w2) + b2 return y(3) 反向傳播檔案修改圖片標籤獲取的介面 mnist_backward.py
import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data import mnist_forward import os import mnist_generateds#1 BATCH_SIZE = 200 LEARNING_RATE_BASE = 0.1 LEARNING_RATE_DECAY = 0.99 REGULARIZER = 0.0001 STEPS = 50000 MOVING_AVERAGE_DECAY = 0.99 MODEL_SAVE_PATH="./model/" MODEL_NAME="mnist_model" train_num_examples = 60000#2 def backward(): x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE]) y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE]) y = mnist_forward.forward(x, REGULARIZER) global_step = tf.Variable(0, trainable=False) ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1)) cem = tf.reduce_mean(ce) loss = cem + tf.add_n(tf.get_collection('losses')) learning_rate = tf.train.exponential_decay( LEARNING_RATE_BASE, global_step, train_num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step) ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step, ema_op]): train_op = tf.no_op(name='train') saver = tf.train.Saver() img_batch, label_batch = mnist_generateds.get_tfrecord(BATCH_SIZE, isTrain=True)#3 with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) coord = tf.train.Coordinator()#4 threads = tf.train.start_queue_runners(sess=sess, coord=coord)#5 for i in range(STEPS): xs, ys = sess.run([img_batch, label_batch])#6 _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys}) if i % 1000 == 0: print("After %d training step(s), loss on training batch is %g." % (step, loss_value)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step) coord.request_stop()#7 coord.join(threads)#8 def main(): backward()#9 if __name__ == '__main__': main()1) 關鍵操作:利用多執行緒提高圖片和標籤的批獲取效率
方法:將批獲取的操作放到執行緒協調器開啟和關閉之間
2) 開啟執行緒協調器:coord = tf.train.Coordinator( ) threads = tf.train.start_queue_runners(sess=sess, coord=coord)# 這個函式將會啟動輸入佇列的執行緒,填充訓練樣本到佇列中,以便出隊操作可以從佇列中拿到樣本。 # 這種情況下最好配合使用一個 tf.train.Coordinator ,這樣可以在發生錯誤的情況下正確地關閉這些執行緒。 tf.train.start_queue_runners( sess=None, # 用於執行佇列操作的會話。 預設為預設會話。 coord=None, # 可選協調器,用於協調啟動的執行緒。 daemon=True, # 守護程序,執行緒是否應該標記為守護程序,這意味著它們不會阻止程式退出。 start=True, # 設定為 False 只建立執行緒,不啟動它們。 collection=tf.GraphKeys.QUEUE_RUNNERS #指定圖集合以獲取啟動佇列的GraphKey。預設為GraphKeys.QUEUE_RUNNERS。 )關閉執行緒協調器:
coord.request_stop( ) coord.join(threads)(4) mnist_test.py
#coding:utf-8 import time import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data import mnist_forward import mnist_backward import mnist_generateds TEST_INTERVAL_SECS = 5 TEST_NUM = 10000#1 def test(): with tf.Graph().as_default() as g: x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE]) y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE]) y = mnist_forward.forward(x, None) ema = tf.train.ExponentialMovingAverage(mnist_backward.MOVING_AVERAGE_DECAY) ema_restore = ema.variables_to_restore() saver = tf.train.Saver(ema_restore) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) img_batch, label_batch = mnist_generateds.get_tfrecord(TEST_NUM, isTrain=False)#2 while True: with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(mnist_backward.MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] coord = tf.train.Coordinator()#3 threads = tf.train.start_queue_runners(sess=sess, coord=coord)#4 xs, ys = sess.run([img_batch, label_batch])#5 accuracy_score = sess.run(accuracy, feed_dict={x: xs, y_: ys}) print("After %s training step(s), test accuracy = %g" % (global_step, accuracy_score)) coord.request_stop()#6 coord.join(threads)#7 else: print('No checkpoint file found') return time.sleep(TEST_INTERVAL_SECS) def main(): test()#8 if __name__ == '__main__': main()(5) mnist_app.py
#coding:utf-8 import tensorflow as tf import numpy as np from PIL import Image import mnist_backward import mnist_forward def restore_model(testPicArr): with tf.Graph().as_default() as tg: x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE]) y = mnist_forward.forward(x, None) preValue = tf.argmax(y, 1) variable_averages = tf.train.ExponentialMovingAverage(mnist_backward.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(mnist_backward.MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) preValue = sess.run(preValue, feed_dict={x:testPicArr}) return preValue else: print("No checkpoint file found") return -1 def pre_pic(picName): img = Image.open(picName) reIm = img.resize((28,28), Image.ANTIALIAS) im_arr = np.array(reIm.convert('L')) threshold = 50 for i in range(28): for j in range(28): im_arr[i][j] = 255 - im_arr[i][j] if (im_arr[i][j] < threshold): im_arr[i][j] = 0 else: im_arr[i][j] = 255 nm_arr = im_arr.reshape([1, 784]) nm_arr = nm_arr.astype(np.float32) img = np.multiply(nm_arr, 1.0/255.0) return nm_arr #img def application(): testNum = input("input the number of test pictures:") for i in range(testNum): testPic = raw_input("the path of test picture:") testPicArr = pre_pic(testPic) preValue = restore_model(testPicArr) print "The prediction number is:", preValue def main(): application() if __name__ == '__main__': main()
