序列数据的处理
电影评论
import tensorflow as tf from tensorflow import keras from tensorflow.keras import layers data = keras.datasets.imdb # 导入数据 max_word = 10000 # 取前面10000条数据 (x_train, y_train), (x_test, y_test) = data.load_data(num_words = max_word) # 文本训练成密集向量是最好的编码方式 # 将所有的数据都填充到同一长度 x_train = keras.preprocessing.sequence.pad_sequences(x_train, 300) x_test = keras.preprocessing.sequence.pad_sequences(x_test, 300) model = keras.models.Sequential() model.add(layers.Embedding(10000, 50, input_length=300)) # model.add(layers.Flatten()) # 把数据展平 model.add(layers.GlobalAveragePooling1D()) # 全局平均池化,同上同等效果,降维 # 缩小规模,减少参数,一定程度解决过拟合问题 model.add(layers.Dense(128, activation='relu')) model.add(layers.Dropout(0.5)) model.add(layers.Dense(1, activation='sigmoid')) # 优化器选择 optimizer=tf.train.AdadeltaOptimizer() 也可以 # keras 自带优化器,有多种 lr==learing rate model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001), loss='binary_crossentropy', metrics=['acc']) history = model.fit(x_train, y_train, epochs=15, batch_size=256, validation_data=(x_test, y_test)) import matplotlib.pyplot as plt %matplotlib inline plt.plot(history.epoch, history.history['loss'], 'r') plt.plot(history.epoch, history.history['val_loss'], 'g--') plt.plot(history.epoch, history.history['acc'], 'r') plt.plot(history.epoch, history.history['val_acc'], 'g--')
解决过拟合问题:
1、dropout
2、添加 l1、l2 正则化
Eager 模式:
命令式编程环境,可以立即评估操作产生的结果,无需构建计算图。
Eager 模式极大的方便我们使用 Tensor Flow、调试模型,增加了网络调试的灵活程度和 Tensor Flow 对于初学者的友好。可以叫它 tensorflow 的交互模式。
用于编写自定义的循环与序列,计算梯度下降算法:
# eager模式与张量 import tensorflow as tf tf.executing_eagerly() # 验证eager模式是否运行,true表示正在运行 x = [[2,]] m = tf.matmul(x, x) # 直接计算结果得到的是一个张量 print(m) m.numpy() # tensor的numpy方法可以返回numpy所对应的ndarray值,也就是对应的numpy值 a = tf.constant([[1, 2], [3, 4]]) # 建立一个常量 a a.numpy() b = tf.add(a, 1) b import numpy as np d = np.array([[5,6],[7,8]]) d (a + b).numpy() g = tf.convert_to_tensor(10) g float(g) c = tf.multiply(a, b) c num = tf.convert_to_tensor(10) num for i in range(num.numpy()): i = tf.constant(i) if (i%2) == 0: print('even') else: print('odd')
变量与自动微分运算
import tensorflow as tf tf.executing_eagerly() # 验证eager模式是否运行,true表示正在运行 v = tf.Variable(0.0) # 创建一个变量 (v + 1).numpy() v.assign(5) # 直接改变变量的值 v v.assign_add(1) # 加法运算 v.read_value() # 读取变量值,读取结果是tensor # 自动求解微分、梯度 # 用tape把计算运算过程记录下来 w = tf.Variable([[1.0]]) # 建立上下文管理器,跟踪变量的运算,计算梯度 with tf.GradientTape() as t: loss = w * w grad = t.gradient(loss, w) # 求解微分、梯度 grad # 用常量记录 w = tf.constant(3.0) with tf.GradientTape() as t: t.watch(w) loss = w * w dloss_dw = t.gradient(loss,w) dloss_dwView Code
在 GradientTape() 记录跟踪变量常量的时候,数据类型一定要设为 float 类型,否则无法计算梯度值。
GradientTape() 只能记录一次运算,在调用 gradient 之后,空间会释放掉,不能同时计算多个微分值。
如果需要同时计算多个微分值,需要设置 GradientTape 中参数 persistent=True
# 同时计算多个微分值,需要设置参数 persistent=True w = tf.constant(3.0) with tf.GradientTape(persistent=True) as t: t.watch(w) y = w * w z = y * y dy_dw = t.gradient(y,w) dy_dw dz_dw = t.gradient(z,w) dz_dw
自定义序列训练过程
tensorflow 2.0 既提供了封装好的 tf.keras API,可以使用 fit 模式训练一个模型,也可以自己定义设计训练模型
import tensorflow as tf (train_image,train_labels),_ = tf.keras.datasets.mnist.load_data() # (test_image,test_labels) 暂时不需要使用,用一个占位符 _ 代替 train_image.shape train_image = tf.expand_dims(train_image, -1) # 扩增维度,本来是一维的,扩增到最后的维度 train_image.shape train_image = tf.cast(train_image/255, tf.float32) # 归一化与改变数据类型(要进行梯度运算) train_labels = tf.cast(train_labels, tf.int64) # 预处理,改变数据类型 dataset = tf.data.Dataset.from_tensor_slices((train_image, train_labels)) dataset dataset = dataset.shuffle(10000).batch(32) dataset # 不规定输入的数据类型,需要用None代替 model = tf.keras.Sequential([ tf.keras.layers.Conv2D(16, [3,3], activation='relu', input_shape=(None,None,1)), tf.keras.layers.Conv2D(32, [3,3], activation='relu'), tf.keras.layers.GlobalMaxPooling2D(), tf.keras.layers.Dense(10) ]) # 优化器 optimizer = tf.keras.optimizers.Adam() # 多分类交叉熵损失值 loss_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # SparseCategoricalCrossentropy 是一种可调用方法 # sparse_categorical_crossentropy 需要把正确的和预测的结果以参数的形式填进去 # from_logits表示激活状态,上面模型中并没有激活,参数为True # iter 是生成器 features, labels = next(iter(dataset)) predictions = model(features) tf.argmax(predictions, axis=1) labels def loss(model, x, y): y_ = model(x) return loss_func(y, y_) # 一个批次数据的训练过程 def train_step(model, images, labels): with tf.GradientTape() as t: loss_step = loss(model, images, labels) grads = t.gradient(loss_step, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables)) def train(): for epoch in range(10): for (batch, (images, labels)) in enumerate(dataset): train_step(model, images, labels) print('Epoch{} is finished'.format(epoch)) train()View Code
tf.keras.metrics 汇总计算模块
使用 tf.keras.metrics 汇总计算模块时,先初始化一个对象,然后对对象进行操作
简单的使用如下:
# 初始化对象 # 计算均值 m = tf.keras.metrics.Mean('acc') m(10) m(20) # 保存上一次的状态,进行效果累加 m.result().numpy() # 显示结果 m.reset_states() # 重置指标计算模块的状态 m.result() a = tf.keras.metrics.SparseCategoricalAccuracy('acc') # 计算准确率 a(labels, model(features)) # 把前面导入的数据代入进去
不仅可以计算均值,还可以计算模块内集成的各种计算方法。
把模块运用到自定义循环中:
train_loss = tf.keras.metrics.Mean('train_loss') train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy('train_accuracy') def train_step(model, images, labels): with tf.GradientTape() as t: # 计算损失函数 pred = model(images) loss_step = loss_func(labels, pred) grads = t.gradient(loss_step, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables)) train_loss(loss_step) train_accuracy(labels, pred) def train(): for epoch in range(10): for (batch, (images, labels)) in enumerate(dataset): train_step(model, images, labels) print('Epoch{} loss is {}, accuracy is {}'.format(epoch, train_loss.result(), train_accuracy.result())) train_loss.reset_states() train_accuracy.reset_states() train()
import tensorflow as tf tf.executing_eagerly() # 验证eager模式是否运行,true表示正在运行 (train_image,train_labels), (test_image, test_labels) = tf.keras.datasets.mnist.load_data() # (test_image,test_labels) 暂时不需要使用,用一个占位符 _ 代替 train_image.shape train_image = tf.expand_dims(train_image, -1) # 扩增维度,本来是一维的,扩增到最后的维度 test_image = tf.expand_dims(test_image, -1) train_image.shape train_image = tf.cast(train_image/255, tf.float32) # 归一化与改变数据类型(要进行梯度运算) test_image = tf.cast(test_image/255, tf.float32) train_labels = tf.cast(train_labels, tf.int64) # 预处理,改变数据类型 test_labels = tf.cast(test_labels, tf.int64) dataset = tf.data.Dataset.from_tensor_slices((train_image, train_labels)) test_dataset = tf.data.Dataset.from_tensor_slices((test_image, test_labels)) dataset = dataset.shuffle(10000).batch(32) test_dataset = test_dataset.batch(32) # 不规定输入的数据类型,需要用None代替 model = tf.keras.Sequential([ tf.keras.layers.Conv2D(16, [3,3], activation='relu', input_shape=(None,None,1)), tf.keras.layers.Conv2D(32, [3,3], activation='relu'), tf.keras.layers.GlobalMaxPooling2D(), tf.keras.layers.Dense(10) ]) # 优化器 optimizer = tf.keras.optimizers.Adam() # 多分类交叉熵损失值 loss_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # SparseCategoricalCrossentropy 是一种可调用方法 # sparse_categorical_crossentropy 需要把正确的和预测的结果以参数的形式填进去 # from_logits表示激活状态,上面模型中并没有激活,参数为True def loss(model, x, y): y_ = model(x) return loss_func(y, y_) train_loss = tf.keras.metrics.Mean('train_loss') train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy('train_accuracy') test_loss = tf.keras.metrics.Mean('test_loss') test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy('test_accuracy') def train_step(model, images, labels): with tf.GradientTape() as t: # 计算损失函数 pred = model(images) loss_step = loss_func(labels, pred) grads = t.gradient(loss_step, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables)) train_loss(loss_step) train_accuracy(labels, pred) def test_step(model, images, labels): pred = model(images) loss_step = loss_func(labels, pred) test_loss(loss_step) test_accuracy(labels, pred) def train(): for epoch in range(10): for (batch, (images, labels)) in enumerate(dataset): train_step(model, images, labels) print('Epoch{} loss is {}, accuracy is {}'.format(epoch, train_loss.result(), train_accuracy.result())) for (batch, (images, labels)) in enumerate(test_dataset): test_step(model, images, labels) print('Epoch{} test_loss is {}, test_accuracy is {}'.format(epoch, test_loss.result(), test_accuracy.result())) train_loss.reset_states() train_accuracy.reset_states() test_loss.reset_states() test_accuracy.reset_states() train()View Code
Tensorboard
Tensorboard 通过读取 TensorFlow 的事件文件来运行。TensorFlow 的事件文件包括了在 TensorFlow 运行中涉及到的主要数据。
一、通过 tf.keras 回调函数使用 tensorboard
import tensorflow as tf import datetime (train_image,train_labels), (test_image, test_labels) = tf.keras.datasets.mnist.load_data() # (test_image,test_labels) 暂时不需要使用,用一个占位符 _ 代替 train_image = tf.expand_dims(train_image, -1) # 扩增维度,本来是一维的,扩增到最后的维度 test_image = tf.expand_dims(test_image, -1) train_image = tf.cast(train_image/255, tf.float32) # 归一化与改变数据类型(要进行梯度运算) test_image = tf.cast(test_image/255, tf.float32) train_labels = tf.cast(train_labels, tf.int64) # 预处理,改变数据类型 test_labels = tf.cast(test_labels, tf.int64) dataset = tf.data.Dataset.from_tensor_slices((train_image, train_labels)) test_dataset = tf.data.Dataset.from_tensor_slices((test_image, test_labels)) dataset = dataset.repeat().shuffle(60000).batch(128) test_dataset = test_dataset.repeat().batch(128) # 不规定输入的数据类型,需要用None代替 model = tf.keras.Sequential([ tf.keras.layers.Conv2D(16, [3,3], activation='relu', input_shape=(None,None,1)), tf.keras.layers.Conv2D(32, [3,3], activation='relu'), tf.keras.layers.GlobalMaxPooling2D(), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy']) import os log_dir = os.path.join('logs', datetime.datetime.Now().strftime("%Y%m%d-%H%M%s")) tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir, histogram_freq=1) model.fit( dataset, epochs = 5, steps_per_epoch = 60000//128, validation_data = test_dataset, validation_steps = 10000//128, callbacks = [tensorboard_callback] )View Code
二、认识 tensorboard 界面
显示 tensorboard 界面,有两种方法,一个是在 notebook 显示,一个是在浏览器显示
%load_ext tensorboard %matplotlib inline %tensorboard --logdir logs # 在notebook打开tensorboard界面
在浏览器打开界面。需要在命令行运行 tensorboard --logdir logs
目录 logs 需要在当前目录下
记录自定义标量:
重新调整回归模型并记录自定义学习率。实现过程:
使用创建文件编写器 tf.summary.creat_file_writer()。
定义自定义学习率功能。这将被传递给 keras LearningRateScheduler 回调。
在学习率功能内,用于 tf.summary.scalar() 记录自定义学习速率。
将 LearningRateScheduler 回调传递给 Model.fit()。
通常,要记录自定义标量,需要使用 tf.summary.scalar() 文件编写器。文件编写器负责将此运行的数据写入指定的目录,并在使用时隐式使用 tf.summary.scalar()。
import os log_dir = os.path.join('logs', datetime.datetime.Now().strftime("%Y%m%d-%H%M%s")) tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir, histogram_freq=1) file_writer = tf.summary.create_file_writer(log_dir + '/lr') # 设置文件编写器 file_writer.set_as_default() # 设置为默认文件编写器 def lr_sche(epoch): learning_rate = 0.2 if epoch>5: learning_rate = 0.02 if epoch>10: learning_rate = 0.01 if epoch>20: learning_rate = 0.005 tf.summary.scalar('learning_rate', data=learning_rate, step=epoch) return learning_rate lr_callback = tf.keras.callbacks.LearningRateScheduler(lr_sche) model.fit( dataset, epochs = 25, steps_per_epoch = 60000//128, validation_data = test_dataset, validation_steps = 10000//128, callbacks = [tensorboard_callback, lr_callback] ) %load_ext tensorboard %matplotlib inline %tensorboard --logdir logs # 在notebook打开tensorboard界面View Code
四、在自定义循环中使用 tensorboard
在自定义训练中使用 Tensorboard
optimizer = tf.keras.optimizers.Adam() loss_func = tf.keras.losses.SparseCategoricalCrossentropy() def loss(model, x, y): y_ = model(x) return loss_func(y, y_) train_loss = tf.keras.metrics.Mean('train_loss') train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy('train_accuracy') test_loss = tf.keras.metrics.Mean('test_loss') test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy('test_accuracy') def train_step(model, images, labels): with tf.GradientTape() as t: # 计算损失函数 pred = model(images) loss_step = loss_func(labels, pred) grads = t.gradient(loss_step, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables)) train_loss(loss_step) train_accuracy(labels, pred) def test_step(model, images, labels): pred = model(images) loss_step = loss_func(labels, pred) test_loss(loss_step) test_accuracy(labels, pred) current_time = datetime.datetime.Now().strftime("%Y%m%d-%H%M%s") train_log_dir = 'logs/gradient_tape' + current_time + '/train' test_log_dir = 'logs/gradient_tape' + current_time + '/test' train_writer = tf.summary.create_file_writer(train_log_dir) test_writer = tf.summary.create_file_writer(test_log_dir) def train(): for epoch in range(10): for (batch, (images, labels)) in enumerate(dataset): train_step(model, images, labels) with train_writer.as_default(): tf.summary.scalar('train_loss', train_loss.result(), step=epoch) tf.summary.scalar('train_acc', train_accuracy.result(), step=epoch) for (batch, (images, labels)) in enumerate(test_dataset): test_step(model, images, labels) print('*', end='') with test_writer.as_default(): tf.summary.scalar('test_loss', test_loss.result(), step=epoch) tf.summary.scalar('test_acc', test_accuracy.result(), step=epoch) train_loss.reset_states() train_accuracy.reset_states() test_loss.reset_states() test_accuracy.reset_states() train()View Code