问题描述
我正在尝试加载 tfrecords 以提高训练速度。我的 tfrecord 包含音频文件的提取特征及其相应的标签。为了重现相同的结果，我生成了一个大小为 (50,50) 和标签 (0 或 1) 的随机 NumPy 数组，并保存为 tfrecord 并读取它们以进行训练。
tensorflow 版本：2.3.0
这是我的示例代码，
生成TFrecord
from pathlib import Path
import tensorflow as tf
import numpy as np
import os
import shutil
from tensorflow.keras.optimizers import SGD,Adadelta,Adam,Nadam,RMSprop
from tensorflow.keras.layers.experimental import preprocessing
from tensorflow.keras.layers import Conv1D,Conv2D,Dense,Flatten,MaxPool1D,MaxPool2D,Dropout,Batchnormalization,Input,MaxPooling1D,Activation,Concatenate,SeparableConv1D
from tensorflow.keras.layers import GlobalAveragePooling1D,GlobalMaxPooling1D,Bidirectional,GRU,LSTM,Timedistributed,ConvLSTM2D,SimpleRNN,AveragePooling1D
from tensorflow.keras.layers import Reshape,Lambda,Dot,softmax,LocallyConnected1D,Layernormalization,add
from tensorflow.keras.models import Model,Sequential,load_model
def _bytes_feature(value):
"""Returns a bytes_list from a string / byte."""
# If the value is an eager tensor BytesList won't unpack a string from an EagerTensor.
if isinstance(value,type(tf.constant(0))):
value = value.numpy()
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _float_feature(value):
"""Returns a float_list from a float / double."""
return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
def _int64_feature(value):
"""Returns an int64_list from a bool / enum / int / uint."""
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def create_example(label,audio_feature):
feature = {
'label': _int64_feature(label),'audio_feature': _bytes_feature(audio_feature),}
# Create a Features message using tf.train.Example.
example_proto = tf.train.Example(features=tf.train.Features(feature=feature))
return example_proto
def serialize_example(audio_feature,label):
feature = {
'audio_feature': _bytes_feature(audio_feature),'label': _int64_feature(label),}
# Create a Features message using tf.train.Example.
example_proto = tf.train.Example(features=tf.train.Features(feature=feature))
return example_proto.SerializetoString()
def genrate_tf_records(tf_records_dir,length):
cnt = 1
if os.path.exists(tf_records_dir):
shutil.rmtree(tf_records_dir)
print(f"Removed old directory...({tf_records_dir})")
print("Creating new tf_record directory...")
Path(tf_records_dir).mkdir(parents = True,exist_ok = True)
file_path = os.path.join(tf_records_dir,"tfdata.tfrecord")
with tf.io.TFRecordWriter(file_path) as writer:
for fn in range(length):
audio_feature,label = np.random.randn(50,50),np.random.choice([0,1])
serialized_example = serialize_example(tf.io.serialize_tensor(audio_feature),label)
writer.write(serialized_example)
tf_records_train_dir ="./tf_features_aug_train/"
tf_records_val_dir ="./tf_features_aug_val/"
genrate_tf_records(tf_records_train_dir,length=1000)
genrate_tf_records(tf_records_val_dir,length=100)
读取TFrecord
AUTOTUNE = tf.data.experimental.AUTOTUNE
def _parse_batch(record_batch):
feature_description = {
'audio_feature': tf.io.FixedLenFeature((),tf.string),'label': tf.io.FixedLenFeature((),tf.int64),}
example = tf.io.parse_example(record_batch,feature_description)
audio_feature = tf.io.parse_tensor(example['audio_feature'],out_type = tf.float64)
#audio_feature = tf.reshape(audio_feature,(input_shape[0],input_shape[1]))
label = example['label']
return audio_feature,label
def get_dataset_from_tfrecords(tfrecords_dir='tfrecords',mode='train',n_epochs=1):
# List all *.tfrecord files for the selected split
files_ds = tf.data.Dataset.list_files(str(tfrecords_dir)+"*.tfrecord")
# disregard data order in favor of reading speed
ignore_order = tf.data.Options()
ignore_order.experimental_deterministic = False
files_ds = files_ds.with_options(ignore_order)
ds = tf.data.TFRecordDataset(files_ds,num_parallel_reads=AUTOTUNE)
# Parse a batch into a dataset of [audio,label] pairs
ds = ds.map(lambda x: _parse_batch(x))
for data,label in ds.take(1):
print(np.shape(data.numpy()))
print(label.numpy())
return ds.prefetch(buffer_size=AUTOTUNE)
train_ds = get_dataset_from_tfrecords(tfrecords_dir=tf_records_train_dir,mode='train')
val_ds = get_dataset_from_tfrecords(tfrecords_dir=tf_records_val_dir,mode='val')
我能够加载保存的 tfrecords 并解析数据。但是当我将 TensorFlow 对象传递给 model.fit() 时，我们得到了一个错误。
创建模型
class Model_Creator():
def getmodel(self,model_name,input_shape,numclass):
#import pdb;pdb.set_trace()
if model_name in dir(self) and callable(getattr(self,model_name)):
print(model_name,'from ACK.py')
model = getattr(self,model_name)(input_shape,numclass)
else:
print(model_name,'from ign_utils/models_audio_ign.py')
model = Model_Creator2().getmodel( model_name,numclass)
print('Created ',model_name)
return model
def cnn_model(self,numclass):
datainp = Input(shape=input_shape)
x=datainp
#x = Dropout(0.1)(x)
x = Conv1D(filters=32,kernel_size=3,strides=1,activation='relu',padding='same')(x)
x = Layernormalization(axis=2)(x)
x = MaxPool1D(strides=2)(x)
#x = Dropout(0.1)(x)
x = GlobalAveragePooling1D()(x)
x = Dropout(0.1)(x)
x = Dense(numclass,activation='softmax')(x)
output_model = Model(inputs=datainp,outputs=x)
return output_model
modelcreator = Model_Creator()
model = modelcreator.getmodel(model_name = 'cnn_model',input_shape=(50,numclass=2)
model.summary()
model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['accuracy'])
运行模型
model.fit(train_ds,validation_data = val_ds,epochs=10)
当我运行这个时，我收到错误，
ValueError: 维度 0 的切片索引 0 越界。对于'{{节点 strided_slice}} = StridedSlice[索引=DT_INT32，T=DT_INT32， begin_mask=0,ellipsis_mask=0,end_mask=0,new_axis_mask=0,shrink_axis_mask=1]（形状，strided_slice/stack，strided_slice/stack_1， strided_slice/stack_2)' 输入形状：[0],[1],[1] 和 计算输入张量：input[1] =,input[2] =,input[3] = .
我不明白为什么会出现这个错误。我可以从 tfrecords 读取和解析数据，但无法将其用于训练（model.fit()）。
P.S：我已经编写了易于复制的代码。这两天我一直在坚持。期待一些帮助。提前致谢。
解决方法
暂无找到可以解决该程序问题的有效方法，小编努力寻找整理中！
如果你已经找到好的解决方法，欢迎将解决方案带上本链接一起发送给小编。
小编邮箱:dio#foxmail.com (将#修改为@）