Python keras.regularizers 模块,l2() 实例源码
我们从Python开源项目中,提取了以下50个代码示例,用于说明如何使用keras.regularizers.l2()。
def cnn_word_model(self):
embed_input = Input(shape=(self.opt['max_sequence_length'], self.opt['embedding_dim'],))
outputs = []
for i in range(len(self.kernel_sizes)):
output_i = Conv1D(self.opt['filters_cnn'], kernel_size=self.kernel_sizes[i], activation=None,
kernel_regularizer=l2(self.opt['regul_coef_conv']), padding='same')(embed_input)
output_i = Batchnormalization()(output_i)
output_i = Activation('relu')(output_i)
output_i = GlobalMaxPooling1D()(output_i)
outputs.append(output_i)
output = concatenate(outputs, axis=1)
output = Dropout(rate=self.opt['dropout_rate'])(output)
output = Dense(self.opt['dense_dim'],
kernel_regularizer=l2(self.opt['regul_coef_dense']))(output)
output = Batchnormalization()(output)
output = Activation('relu')(output)
output = Dropout(rate=self.opt['dropout_rate'])(output)
output = Dense(1, kernel_regularizer=l2(self.opt['regul_coef_dense']))(output)
output = Batchnormalization()(output)
act_output = Activation('sigmoid')(output)
model = Model(inputs=embed_input, outputs=act_output)
return model
def test_activity_regularization():
from keras.engine import Input, Model
layer = core.ActivityRegularization(l1=0.01, l2=0.01)
# test in functional API
x = Input(shape=(3,))
z = core.Dense(2)(x)
y = layer(z)
model = Model(input=x, output=y)
model.compile('rmsprop', 'mse', mode='FAST_COMPILE')
model.predict(np.random.random((2, 3)))
# test serialization
model_config = model.get_config()
model = Model.from_config(model_config)
model.compile('rmsprop', 'mse')
def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4):
''' SubpixelConvolutional Upscaling (factor = 2)
Args:
ip: keras tensor
nb_filters: number of layers
type: can be 'upsampling','subpixel','deconv'. Determines type of upsampling performed
weight_decay: weight decay factor
Returns: keras tensor,after applying upsampling operation.
'''
if type == 'upsampling':
x = UpSampling2D()(ip)
elif type == 'subpixel':
x = Conv2D(nb_filters, (3, 3), activation='relu', padding='same', kernel_regularizer=l2(weight_decay),
use_bias=False, kernel_initializer='he_normal')(ip)
x = SubPixelUpscaling(scale_factor=2)(x)
x = Conv2D(nb_filters, kernel_initializer='he_normal')(x)
else:
x = Conv2DTranspose(nb_filters, strides=(2, 2),
kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(ip)
return x
def _conv_bn_relu(**conv_params): # ???????BN
"""Helper to build a conv -> BN -> relu block
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides", (1, 1))
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
padding = conv_params.setdefault("padding", "same")
kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(input):
conv = Conv2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer)(input)
return _bn_relu(conv)
return f
def _bn_relu_conv(**conv_params): # ????????BN
"""Helper to build a BN -> relu -> conv block.
This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides", l2(1.e-4))
def f(input):
activation = _bn_relu(input)
return Conv2D(filters=filters,
kernel_regularizer=kernel_regularizer)(activation)
return f
def _shortcut(input, residual): # ??????????????????y=F(x)+x
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width,height)
# Should be int if network architecture is correctly configured.
input_shape = K.int_shape(input)
residual_shape = K.int_shape(residual)
stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(0.0001))(input)
return add([shortcut, residual])
def basic_block(filters, init_strides=(1, is_first_block_of_first_layer=False): # ????????????
"""Basic 3 X 3 convolution blocks for use on resnets with layers <= 34.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
"""
def f(input):
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
conv1 = Conv2D(filters=filters, kernel_size=(3,
strides=init_strides,
padding="same",
kernel_initializer="he_normal",
kernel_regularizer=l2(1e-4))(input)
else:
conv1 = _bn_relu_conv(filters=filters,
strides=init_strides)(input)
residual = _bn_relu_conv(filters=filters, 3))(conv1)
return _shortcut(input, residual)
return f
def bottleneck(filters, is_first_block_of_first_layer=False):
"""Bottleneck architecture for > 34 layer resnet.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
Returns:
A final conv layer of filters * 4
"""
def f(input):
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
conv_1_1 = Conv2D(filters=filters, kernel_size=(1,
strides=init_strides,
padding="same",
kernel_initializer="he_normal",
kernel_regularizer=l2(1e-4))(input)
else:
conv_1_1 = _bn_relu_conv(filters=filters,
strides=init_strides)(input)
conv_3_3 = _bn_relu_conv(filters=filters, 3))(conv_1_1)
residual = _bn_relu_conv(filters=filters * 4, 1))(conv_3_3)
return _shortcut(input, residual)
return f
def lstm_word_model(self):
embed_input = Input(shape=(self.opt['max_sequence_length'],))
output = Bidirectional(LSTM(self.opt['units_lstm'], activation='tanh',
kernel_regularizer=l2(self.opt['regul_coef_lstm']),
dropout=self.opt['dropout_rate']))(embed_input)
output = Dropout(rate=self.opt['dropout_rate'])(output)
output = Dense(self.opt['dense_dim'],
kernel_regularizer=l2(self.opt['regul_coef_dense']))(output)
output = Batchnormalization()(output)
act_output = Activation('sigmoid')(output)
model = Model(inputs=embed_input, outputs=act_output)
return model
def create_actor_network(self, state_size, action_dim):
"""Create actor network."""
print ("[MESSAGE] Build actor network.""")
S = Input(shape=state_size)
h_0 = Conv2D(32, padding="same",
kernel_regularizer=l2(0.0001),
activation="relu")(S)
h_1 = Conv2D(32,
activation="relu")(h_0)
h_1 = AveragePooling2D(2, 2)(h_1)
h_1 = Flatten()(h_1)
h_1 = Dense(600, activation="relu")(h_1)
A = Dense(action_dim, activation="softmax")(h_1)
model = Model(inputs=S, outputs=A)
return model, model.trainable_weights, S
def test_dense():
from keras import regularizers
from keras import constraints
layer_test(core.Dense,
kwargs={'output_dim': 3},
input_shape=(3, 2))
layer_test(core.Dense,
kwargs={'output_dim': 3,
'W_regularizer': regularizers.l2(0.01),
'b_regularizer': regularizers.l1(0.01),
'activity_regularizer': regularizers.activity_l2(0.01),
'W_constraint': constraints.Maxnorm(1),
'b_constraint': constraints.Maxnorm(1)}, 2))
def test_maxout_dense():
from keras import regularizers
from keras import constraints
layer_test(core.MaxoutDense, 2))
layer_test(core.MaxoutDense, 2))
def test_timedistributeddense():
from keras import regularizers
from keras import constraints
layer_test(core.TimedistributedDense,
kwargs={'output_dim': 2, 'input_length': 2}, 2, 3))
layer_test(core.TimedistributedDense, 3))
def _conv_bn_relu(**conv_params):
"""Helper to build a conv -> BN -> relu residual unit activation function.
This is the original resnet v1 scheme in https://arxiv.org/abs/1512.03385
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides", 1))
dilation_rate = conv_params.setdefault("dilation_rate", 1))
conv_name = conv_params.setdefault("conv_name", None)
bn_name = conv_params.setdefault("bn_name", None)
relu_name = conv_params.setdefault("relu_name", None)
kernel_initializer = conv_params.setdefault("kernel_initializer", l2(1.e-4))
def f(x):
x = Conv2D(filters=filters,
strides=strides,
dilation_rate=dilation_rate,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer,
name=conv_name)(x)
return _bn_relu(x, bn_name=bn_name, relu_name=relu_name)
return f
def _bn_relu_conv(**conv_params):
"""Helper to build a BN -> relu -> conv residual unit with full pre-activation function.
This is the resnet v2 scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides", l2(1.e-4))
def f(x):
activation = _bn_relu(x, relu_name=relu_name)
return Conv2D(filters=filters,
dilation_rate=dilation_rate,
kernel_regularizer=kernel_regularizer,
name=conv_name)(activation)
return f
def _bn_relu_conv(**conv_params):
"""Helper to build a BN -> relu -> conv block.
This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides",
kernel_regularizer=kernel_regularizer)(activation)
return f
def _shortcut(input, residual):
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width, residual])
def basic_block(filters, is_first_block_of_first_layer=False):
"""Basic 3 X 3 convolution blocks for use on resnets with layers <= 34.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
"""
def f(input):
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
conv1 = Conv2D(filters=filters, residual)
return f
def gen_Model(num_units, actfn='linear', reg_coeff=0.0, last_act='softmax'):
''' Generate a neural network model of approporiate architecture
Args:
num_units: architecture of network in the format [n1,n2,...,nL]
actfn: activation function for hidden layers ('relu'/'sigmoid'/'linear'/'softmax')
reg_coeff: L2-regularization coefficient
last_act: activation function for final layer ('relu'/'sigmoid'/'linear'/'softmax')
Output:
model: Keras sequential model with appropriate fully-connected architecture
'''
model = Sequential()
for i in range(1, len(num_units)):
if i == 1 and i < len(num_units) - 1:
model.add(Dense(input_dim=num_units[0], output_dim=num_units[i], activation=actfn,
W_regularizer=Reg.l2(l=reg_coeff), init='glorot_normal'))
elif i == 1 and i == len(num_units) - 1:
model.add(Dense(input_dim=num_units[0], activation=last_act, init='glorot_normal'))
elif i < len(num_units) - 1:
model.add(Dense(output_dim=num_units[i], init='glorot_normal'))
elif i == len(num_units) - 1:
model.add(Dense(output_dim=num_units[i], init='glorot_normal'))
return model
def cnn(height, width):
question_input = Input(shape=(height, width, name='question_input')
conv1_Q = Conv2D(512, (2, 320), activation='sigmoid', padding='valid',
kernel_regularizer=regularizers.l2(0.01),
kernel_initializer=initializers.random_normal(mean=0.0, stddev=0.02))(question_input)
Max1_Q = MaxPooling2D((29, strides=(1, padding='valid')(conv1_Q)
F1_Q = Flatten()(Max1_Q)
Drop1_Q = Dropout(0.25)(F1_Q)
predictQ = Dense(32, stddev=0.02))(Drop1_Q)
prediction2 = Dropout(0.25)(predictQ)
predictions = Dense(1, activation='relu')(prediction2)
model = Model(inputs=[question_input],
outputs=predictions)
model.compile(loss='mean_squared_error',
optimizer=Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0))
# model.compile(loss='mean_squared_error',
# optimizer='nadam')
return model
def test_keras_import(self):
# Conv 1D
model = Sequential()
model.add(LocallyConnected1D(32, 3, kernel_regularizer=regularizers.l2(0.01),
bias_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l2(0.01), kernel_constraint='max_norm',
bias_constraint='max_norm', input_shape=(10, 16)))
model.build()
self.keras_param_test(model, 1, 12)
# Conv 2D
model = Sequential()
model.add(LocallyConnected2D(32, 16, 14)
# ********** Recurrent Layers **********
def test_keras_import(self):
model = Sequential()
model.add(Batchnormalization(center=True, scale=True, beta_regularizer=regularizers.l2(0.01),
gamma_regularizer=regularizers.l2(0.01),
beta_constraint='max_norm', gamma_constraint='max_norm',
input_shape=(10, 16)))
model.build()
json_string = Model.to_json(model)
with open(os.path.join(settings.BASE_DIR, 'media', 'test.json'), 'w') as out:
json.dump(json.loads(json_string), out, indent=4)
sample_file = open(os.path.join(settings.BASE_DIR, 'r')
response = self.client.post(reverse('keras-import'), {'file': sample_file})
response = json.loads(response.content)
layerId = sorted(response['net'].keys())
self.assertEqual(response['result'], 'success')
self.assertEqual(response['net'][layerId[0]]['info']['type'], 'Scale')
self.assertEqual(response['net'][layerId[1]]['info']['type'], 'Batchnorm')
# ********** Noise Layers **********
def feature_extractor(FLAGS, suffix = ""):
weights = FLAGS.weights if FLAGS.weights != "random" else None
if FLAGS.model == "vgg16":
from keras.applications.vgg16 import VGG16
feature_extractor = VGG16(weights = weights)
remove_last_layer(feature_extractor)
elif FLAGS.model == "vgg19":
from keras.applications.vgg19 import VGG19
feature_extractor = VGG19(weights = weights)
remove_last_layer(feature_extractor)
elif FLAGS.model == "resnet50":
from keras.applications.resnet50 import resnet50
feature_extractor = resnet50(weights = weights)
remove_last_layer(feature_extractor)
else:
raise NotImplementedError
feature_extractor.name = FLAGS.model + suffix
if FLAGS.regularizer == "l2":
add_regularizer(feature_extractor)
elif FLAGS.regularizer != "none":
raise NotImplementedError
return feature_extractor
def keepsize_256(nx, ny, noise, depth, n_filters=64, l2_reg=1e-7):
"""
Deep residual network that keeps the size of the input throughout the whole network
"""
def residual(inputs, n_filters):
x = ReflectionPadding2D()(inputs)
x = Conv2D(n_filters, kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(x)
x = Batchnormalization()(x)
x = Activation(activation)(x)
x = ReflectionPadding2D()(x)
x = Conv2D(n_filters, kernel_regularizer=l2(l2_reg))(x)
x = Batchnormalization()(x)
x = add([x, inputs])
return x
inputs = Input(shape=(nx, 1))
x = GaussianNoise(noise)(inputs)
x = ReflectionPadding2D()(x)
x = Conv2D(n_filters, kernel_regularizer=l2(l2_reg))(x)
x0 = Activation(activation)(x)
x = residual(x0, n_filters)
for i in range(depth-1):
x = residual(x, n_filters)
x = ReflectionPadding2D()(x)
x = Conv2D(n_filters, kernel_regularizer=l2(l2_reg))(x)
x = Batchnormalization()(x)
x = add([x, x0])
# Upsampling for superresolution
x = UpSampling2D()(x)
x = ReflectionPadding2D()(x)
x = Conv2D(4*n_filters, kernel_regularizer=l2(l2_reg))(x)
x = Activation(activation)(x)
final = Conv2D(1, kernel_regularizer=l2(l2_reg))(x)
return Model(inputs=inputs, outputs=final)
def prep_model(inputs, N, s0pad, s1pad, c, granlevels=1):
# LSTM
lstm = LSTM(N, return_sequences=True, implementation=2,
kernel_regularizer=l2(c['l2reg']), recurrent_regularizer=l2(c['l2reg']),
bias_regularizer=l2(c['l2reg']))
x1 = inputs[0]
x2 = inputs[1]
h1 = lstm(x1)
h2 = lstm(x2)
W_x = Dense(N, kernel_initializer='glorot_uniform', use_bias=True,
kernel_regularizer=l2(c['l2reg']))
W_h = Dense(N, kernel_initializer='orthogonal',
kernel_regularizer=l2(c['l2reg']))
sigmoid = Activation('sigmoid')
a1 = multiply([x1, sigmoid( add([W_x(x1), W_h(h1)]) )])
a2 = multiply([x2, sigmoid( add([W_x(x2), W_h(h2)]) )])
# Averaging
avg = Lambda(function=lambda x: K.mean(x, axis=1),
output_shape=lambda shape: (shape[0], ) + shape[2:])
gran1 = avg(a1)
gran2 = avg(a2)
return [gran1, gran2], N
def prep_model(inputs, c):
Nc, outputs = B.cnnsum_input(inputs, siamese=c['cnnsiamese'],
dropout=c['dropout'], l2reg=c['l2reg'],
cnninit=c['cnninit'], cnnact=c['cnnact'], cdim=c['cdim'])
# Projection
if c['project']:
outputs = Dense(int(N*c['pdim']), kernal_regularizer=l2(c['l2reg']), activation=c['pact'])(outputs)
# model.add_shared_node(name='proj',inputs=['e0s_','e1s_'],outputs=['e0p','e1p'],
# layer=Dense(input_dim=Nc,output_dim=int(N*c['pdim']),
# W_regularizer=l2(c['l2reg']),activation=c['pact']))
# This dropout is controversial; it might be harmful to apply,
# or at least isn't a clear win.
# model.add_shared_node(name='projdrop',inputs=['e0p',outputs=['e0p_','e1p_'],
# layer=Dropout(c['dropout'],input_shape=(N,)))
# return ('e0p_','e1p_')
return outputs, N
def prep_model(inputs, c):
outputs = B.rnn_input(inputs,
dropout=c['dropout'], dropoutfix_inp=c['dropoutfix_inp'], dropoutfix_rec=c['dropoutfix_rec'],
sdim=c['sdim'],
rnnbidi=c['rnnbidi'], rnn=c['rnn'], rnnact=c['rnnact'], rnninit=c['rnninit'],
rnnbidi_mode=c['rnnbidi_mode'], rnnlevels=c['rnnlevels'])
# Projection
if c['project']:
proj = Dense(int(N*c['pdim']), activation=c['pact'], kernel_regularizer=l2(c['l2reg']), name='proj')
e0p = proj(outputs[0])
e1p = proj(outputs[1])
N = N*c['pdim']
return [e0p, e1p], N
else:
return [outputs[0], outputs[1]], N
#input_dim=int(N*c['sdim'])
def _conv_bn_relu(**conv_params):
"""Helper to build a conv -> BN -> relu block
"""
nb_filter = conv_params["nb_filter"]
nb_row = conv_params["nb_row"]
nb_col = conv_params["nb_col"]
subsample = conv_params.setdefault("subsample", 1))
init = conv_params.setdefault("init", "he_normal")
border_mode = conv_params.setdefault("border_mode", "same")
W_regularizer = conv_params.setdefault("W_regularizer", l2(1.e-4))
def f(input):
conv = Convolution2D(nb_filter=nb_filter, nb_row=nb_row, nb_col=nb_col, subsample=subsample,
init=init, border_mode=border_mode, W_regularizer=W_regularizer)(input)
return _bn_relu(conv)
return f
def _bn_relu_conv(**conv_params):
"""Helper to build a BN -> relu -> conv block.
This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf
"""
nb_filter = conv_params["nb_filter"]
nb_row = conv_params["nb_row"]
nb_col = conv_params["nb_col"]
subsample = conv_params.setdefault("subsample", l2(1.e-4))
def f(input):
activation = _bn_relu(input)
return Convolution2D(nb_filter=nb_filter, W_regularizer=W_regularizer)(activation)
return f
def _shortcut(input,height)
# Should be int if network architecture is correctly configured.
input_shape = K.int_shape(input)
residual_shape = K.int_shape(residual)
stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Convolution2D(nb_filter=residual_shape[CHANNEL_AXIS],
nb_row=1, nb_col=1,
subsample=(stride_width,
init="he_normal", border_mode="valid",
W_regularizer=l2(0.0001))(input)
return merge([shortcut, residual], mode="sum")
def ResidualBlock1D_helper(layers, kernel_size, filters, final_stride=1):
def f(_input):
basic = _input
for ln in range(layers):
#basic = Batchnormalization()( basic ) # triggers kNown keras bug w/ Timedistributed: https://github.com/fchollet/keras/issues/5221
basic = ELU()(basic)
basic = Conv1D(filters,
kernel_regularizer=l2(1.e-4), padding='same')(basic)
# note that this strides without averaging
return AveragePooling1D(pool_size=1, strides=final_stride)(Add()([_input, basic]))
return f
def createModel(w=None,h=None):
# Input placeholder
original = Input(shape=(w, h, 4), name='icon_goes_here')
# Model layer stack
x = original
x = Convolution2D(64, 4, border_mode='same', b_regularizer=l2(0.1))(x)
x = Convolution2D(64, b_regularizer=l2(0.1))(x)
x = AveragePooling2D((2, border_mode='valid')(x)
x = Convolution2D(16, b_regularizer=l2(0.1))(x)
x = Convolution2D(4, b_regularizer=l2(0.1))(x)
downscaled = x
# Compile model
hintbot = Model(input=original, output=downscaled)
hintbot.compile(optimizer='adam', loss='mean_squared_error')
# Train
if (os.path.isfile(load_weights_filepath)):
hintbot.load_weights(load_weights_filepath)
return hintbot
def _model(self, input_shape):
self.model.add(Dense(self.hidden[0],
input_shape=(input_shape[1],),
kernel_regularizer=l2(self.wd),
kernel_initializer=self.ki))
if self.bn:
self.model.add(Batchnormalization(axis=1))
self.model.add(Activation(self.activation))
for i in self.hidden[1:]:
self.model.add(Dense(i, kernel_regularizer=l2(self.wd),
kernel_initializer=self.ki))
if self.bn:
self.model.add(Batchnormalization(axis=1))
self.model.add(Activation(self.activation))
self.model.add(Dense(self.N, activation='softmax',
kernel_regularizer=l2(self.wd),
kernel_initializer=self.ki))
def _shortcut(input, residual, weight_decay=.0001, dropout=.0):
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width,height)
# Should be int if network architecture is correctly configured.
# !!! The dropout argument is just a place holder.
# !!! It shall not be applied to identity mapping.
stride_width = input._keras_shape[ROW_AXIS] // residual._keras_shape[ROW_AXIS]
stride_height = input._keras_shape[COL_AXIS] // residual._keras_shape[COL_AXIS]
equal_channels = residual._keras_shape[CHANNEL_AXIS] == input._keras_shape[CHANNEL_AXIS]
shortcut = input
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Convolution2D(nb_filter=residual._keras_shape[CHANNEL_AXIS],
W_regularizer=l2(weight_decay))(input)
return merge([shortcut, mode="sum")
# Builds a residual block with repeating bottleneck blocks.
def test_activity_regularization():
from keras.engine import Input, 'mse')
def test_maxout_dense():
from keras import regularizers
from keras import constraints
layer_test(core.MaxoutDense, 2))
def test_timedistributeddense():
from keras import regularizers
from keras import constraints
layer_test(core.TimedistributedDense, 3))
def _bn_relu_conv(**conv_params):
"""Helper to build a BN -> relu -> conv block.
This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf
"""
nb_filter = conv_params["nb_filter"]
nb_row = conv_params["nb_row"]
nb_col = conv_params["nb_col"]
subsample = conv_params.setdefault("subsample",1))
init = conv_params.setdefault("init", W_regularizer=W_regularizer)(activation)
return f
def basic_block(nb_filter, init_subsample=(1, is_first_block_of_first_layer=False):
"""Basic 3 X 3 convolution blocks for use on resnets with layers <= 34.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
"""
def f(input):
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
conv1 = Convolution2D(nb_filter=nb_filter,
nb_row=3, nb_col=3,
subsample=init_subsample, border_mode="same",
W_regularizer=l2(0.0001))(input)
else:
conv1 = _bn_relu_conv(nb_filter=nb_filter, nb_row=3,
subsample=init_subsample)(input)
residual = _bn_relu_conv(nb_filter=nb_filter, nb_col=3)(conv1)
return _shortcut(input, residual)
return f
def bottleneck(nb_filter, is_first_block_of_first_layer=False):
"""Bottleneck architecture for > 34 layer resnet.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
Returns:
A final conv layer of nb_filter * 4
"""
def f(input):
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
conv_1_1 = Convolution2D(nb_filter=nb_filter,
W_regularizer=l2(0.0001))(input)
else:
conv_1_1 = _bn_relu_conv(nb_filter=nb_filter, nb_row=1,
subsample=init_subsample)(input)
conv_3_3 = _bn_relu_conv(nb_filter=nb_filter, nb_col=3)(conv_1_1)
residual = _bn_relu_conv(nb_filter=nb_filter * 4, nb_col=1)(conv_3_3)
return _shortcut(input, residual)
return f
def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4):
''' Apply Batchnorm,Relu 1x1,Conv2D,optional compression,dropout and Maxpooling2D
Args:
ip: keras tensor
nb_filter: number of filters
compression: calculated as 1 - reduction. Reduces the number of feature maps
in the transition block.
dropout_rate: dropout rate
weight_decay: weight decay factor
Returns: keras tensor,after applying batch_norm,relu-conv,dropout,maxpool
'''
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
x = Batchnormalization(axis=concat_axis, epsilon=1.1e-5)(ip)
x = Activation('relu')(x)
x = Conv2D(int(nb_filter * compression), use_bias=False,
kernel_regularizer=l2(weight_decay))(x)
x = AveragePooling2D((2, 2))(x)
return x
def conv_block(ip, dropout_rate=None, weight_decay=1E-4):
''' Apply Batchnorm,Relu 3x3,optional dropout
Args:
ip: Input keras tensor
nb_filter: number of filters
dropout_rate: dropout rate
weight_decay: weight decay factor
Returns: keras tensor with batch_norm,relu and convolution2d added
'''
x = Activation('relu')(ip)
x = Convolution2D(nb_filter, init="he_uniform", bias=False,
W_regularizer=l2(weight_decay))(x)
if dropout_rate:
x = Dropout(dropout_rate)(x)
return x
def transition_block(ip,optional dropout and Maxpooling2D
Args:
ip: keras tensor
nb_filter: number of filters
dropout_rate: dropout rate
weight_decay: weight decay factor
Returns: keras tensor,maxpool
'''
concat_axis = 1 if K.image_dim_ordering() == "th" else -1
x = Convolution2D(nb_filter,
W_regularizer=l2(weight_decay))(ip)
if dropout_rate:
x = Dropout(dropout_rate)(x)
x = AveragePooling2D((2, 2))(x)
x = Batchnormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
beta_regularizer=l2(weight_decay))(x)
return x
def build(self):
dim_data = self.size_of_input_data_dim
nb_time_step = self.size_of_input_timesteps
financial_time_series_input = Input(shape=(nb_time_step, dim_data))
lstm_layer_1 = LSTM(output_dim=nb_hidden_units, dropout_U=dropout, dropout_W=dropout, inner_activation='sigmoid',
W_regularizer=l2(l2_norm_alpha), b_regularizer=l2(l2_norm_alpha),
return_sequences=True)
lstm_layer_2 = LSTM(output_dim=nb_hidden_units,
return_sequences=True)
h1 = lstm_layer_1(financial_time_series_input)
h2 = lstm_layer_2(h1)
time_series_predictions = TimedistributedDense(1)(h2)
self.model = Model(financial_time_series_input, time_series_predictions,
name="deep rnn for financial time series forecasting")