記錄Highway Network的程式碼實現部分，以備日後使用。。。

Highway Network

1.作用or概念部分

2.程式碼實現部分

def highway(input_, size, scope='highway',reuse=None,layer_size=1, bias=-2, f=tf.nn.tanh):
    """Highway Network (cf. http://arxiv.org/abs/1505.00387).
    t = sigmoid(Wy + b)
    z = t * g(Wy + b) + (1 - t) * y
    where g is nonlinearity, t is transform gate, and (1 - t) is carry gate.
    """
 


    output=input_
    with tf.variable_scope(scope,reuse=reuse):
        for idx in range(layer_size):
            output = tf.tanh(dense(input_,size,scope='dense1'))
            transform_gate = tf.sigmoid(dense(input_,size,scope='dense2'))
            carry_gate = 1. - transform_gate
            output = transform_gate * output + carry_gate * input_
    return
 
 output

def dense(inputs, output_size, bias=True,seq_len=None,max_len=None,mode='mul',scope='dense'):
    with tf.variable_scope(scope):
        input_size = int(inputs.shape[-1])
        W = tf.Variable(tf.random_uniform([input_size, output_size], -0.1, 0.1))
        #W = tf.get_variable("W",
        #                [input_size,output_size],
 

        #                dtype = tf.float32,
        #                regularizer=regularizer,
        #                initializer =  initializer())
        if bias:
            b = tf.Variable(tf.random_uniform([output_size], -0.05, 0.05))
            #b = tf.get_variable("b",
            #            [output_size],
            #            regularizer=regularizer,
            #            initializer = tf.zeros_initializer())
        else:
            b = 0
        outputs = tf.matmul(tf.reshape(inputs, (-1, input_size)), W) + b
        outputs = tf.reshape(outputs, \
                             tf.concat([tf.shape(inputs)[:-1], [output_size]], 0)
                             )
        if seq_len != None:
            outputs = Mask(outputs, seq_len, max_len,mode=mode)
        return outputs