客户端码农学习ML —— 全连接神经网络识别手写图片Mnist

# 首先准备样本共4个，有特征x1、x2，x0是bias，固定值1。
x0 = np.array([1, 1, 1, 1], dtype=np.float32)
x1 = np.array([0, 0, 1, 1], dtype=np.float32)
x2 = np.array([0, 1, 0, 1], dtype=np.float32)

# 然后准备目标值target
if arg_type == 'AND':
    target = np.array([0, 0, 0, 1], dtype=np.float32)
else arg_type == 'OR':
    target = np.array([0, 1, 1, 1], dtype=np.float32)

# 把特征和目标值转成矩证以便下面使用
m_x = np.vstack((x0, x1, x2)).T
m_target = np.matrix(target).T

v_w = tf.Variable(np.zeros((3, 1)), dtype=tf.float32)
h_x = tf.placeholder(shape=[None, 3], dtype=tf.float32)
h_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

z = tf.matmul(h_x, v_w)
loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=z, labels=h_target)
loss = tf.reduce_mean(loss)
optimizer = tf.train.GradientDescentOptimizer(200)
train = optimizer.minimize(loss)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

loss_vec = []

for step in range(401):
    feed_dict = {h_x: m_x, h_target: m_target}
    sess.run(train, feed_dict)
    loss_vec.append(sess.run(loss, feed_dict=feed_dict))
    if step % 100 == 0:
        print('step=%d W=%s loss=%s' % (
            step, sess.run(v_w).ravel(), sess.run(loss, feed_dict=feed_dict)))

_w = sess.run(v_w).ravel()
sess.close()

print(f'Train data size: {mnist.train.num_examples}')
print(f'Validation data size: {mnist.validation.num_examples}')
print(f'Test data size: {mnist.test.num_examples}')

INPUT_NODE = 28 * 28  # 输入节点数量
OUTPUT_NODE = 10  # 输出节点数量

LAYER1_NODE = 500 # 第一个隐藏层节点数量

# 定义一个获取权重的方法
def get_weight_variable(shape, regularizer):
    weights = tf.get_variable('weights', shape, initializer=tf.truncated_normal_initializer(stddev=0.1))
    if regularizer is not None:
        tf.add_to_collection(tf.GraphKeys.LOSSES, regularizer(weights))
    return weights

# 定义计算图，输入层 矩阵乘 隐藏层权重，激活函数Relu
def inference(input_tensor, regularizer):
    with tf.variable_scope('layer1'):
        weights = get_weight_variable([INPUT_NODE, LAYER1_NODE], regularizer)
        biases = tf.get_variable("biases", [LAYER1_NODE], initializer=tf.constant_initializer(0))
        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights) + biases)

    with tf.variable_scope('layer2'):
        weights = get_weight_variable([LAYER1_NODE, OUTPUT_NODE], regularizer)
        biases = tf.get_variable("biases", [OUTPUT_NODE], initializer=tf.constant_initializer(0))
        output_layer = tf.matmul(layer1, weights) + biases

    return output_layer

def train(mnist):
    x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='x-input')
    y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='y-input')

    regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE) # 使用l2正则化
    y = mnist_inference.inference(x, regularizer) # 使用刚刚定义的计算图
    global_step = tf.Variable(0, trainable=False, name='globalStep')

    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
    variables_averages_op = variable_averages.apply(tf.trainable_variables())
    average_y = y

    cross_entroy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
    cross_entroy_mean = tf.reduce_mean(cross_entroy)
    loss = cross_entroy_mean + tf.add_n(tf.get_collection(tf.GraphKeys.LOSSES))

	# 使用指数衰减的学习率方法，这样初期学习率可略高一点提高训练速度，后面会自动慢慢的降低学习率
    learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step
                                               , mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY)

    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)

    # train_op = tf.group(train_step, variables_averages_op)
    with tf.control_dependencies([train_step, variables_averages_op]):
        train_op = tf.no_op(name='train')


    saver = tf.train.Saver() # 每1000轮迭代保存一次模型训练结果，以便恢复训练或者使用这些模型评估训练成果
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for i in range(TRAINING_STEPS):
            xs, ys = mnist.train.next_batch(BATCH_SIZE)
            _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})
            if i % 1000 == 0:
                print(f'After {step} training step(s), loss on training is {loss_value}')
                saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
        print(f'After last {step} training step(s), loss on training is {loss_value}')
        saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)

x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='y-input')
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}

# 同样要定义计算图，直接用训练里定义好的方法
y = mnist_inference.inference(x, None)

# 判断预测值与样本里目标值是否相等
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# saver = tf.train.Saver()

variable_averages = tf.train.ExponentialMovingAverage(mnist_train.MOVING_AVERAGE_DECAY)
variable_to_restore = variable_averages.variables_to_restore()
print(f'variable_to_restore={variable_to_restore}')
saver = tf.train.Saver(variable_to_restore)

while True:
    with tf.Session() as sess:
    	 # 从训练代码里保存的模型文件中读取参数并运行预测方法
        ckpt = tf.train.get_checkpoint_state(mnist_train.MODEL_SAVE_PATH)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
            global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
            accuracy_score = sess.run(accuracy, feed_dict=validate_feed)
            print(f'After {global_step} training step(s), validation accuracy = {accuracy_score}')
        else:
            print('No checkpoint file found')
    time.sleep(EVAL_INTERVAL_SECS)