Skip to content

Commit

Permalink
Clean up model code
Browse files Browse the repository at this point in the history
  • Loading branch information
fschaeffler93 committed May 21, 2019
1 parent 0024633 commit 9b048e5
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 108 deletions.
18 changes: 6 additions & 12 deletions model/group_pointcloud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@ def __init__(self, out_channels, name):
super(VFELayer, self).__init__()
self.units = int(out_channels / 2)
with tf.variable_scope(name, reuse=tf.AUTO_REUSE) as scope:
self.dense = tf.layers.Dense(
self.units, tf.nn.relu, name='dense', _reuse=tf.AUTO_REUSE, _scope=scope)
self.batch_norm = tf.layers.BatchNormalization(
name='batch_norm', fused=True, _reuse=tf.AUTO_REUSE, _scope=scope)
self.dense = tf.layers.Dense(self.units, tf.nn.relu, name='dense', _reuse=tf.AUTO_REUSE, _scope=scope)
self.batch_norm = tf.layers.BatchNormalization(name='batch_norm', fused=True, _reuse=tf.AUTO_REUSE, _scope=scope)

def apply(self, inputs, mask, training):
# [K, T, 7] tensordot [7, units] = [K, T, units]
Expand Down Expand Up @@ -49,21 +47,18 @@ def __init__(self, training, batch_size, name=''):
# scalar
self.batch_size = batch_size
# [ΣK, 35/45, 7]
self.feature = tf.placeholder(
tf.float32, [None, cfg.VOXEL_POINT_COUNT, 7], name='feature')
self.feature = tf.placeholder(tf.float32, [None, cfg.VOXEL_POINT_COUNT, 7], name='feature')
# [ΣK]
self.number = tf.placeholder(tf.int64, [None], name='number')
# [ΣK, 4], each row stores (batch, d, h, w)
self.coordinate = tf.placeholder(
tf.int64, [None, 4], name='coordinate')
self.coordinate = tf.placeholder(tf.int64, [None, 4], name='coordinate')

with tf.variable_scope(name, reuse=tf.AUTO_REUSE) as scope:
self.vfe1 = VFELayer(32, 'VFE-1')
self.vfe2 = VFELayer(128, 'VFE-2')

# boolean mask [K, T, 2 * units]
mask = tf.not_equal(tf.reduce_max(
self.feature, axis=2, keep_dims=True), 0)
mask = tf.not_equal(tf.reduce_max(self.feature, axis=2, keep_dims=True), 0)
x = self.vfe1.apply(self.feature, mask, self.training)
x = self.vfe2.apply(x, mask, self.training)

Expand All @@ -72,7 +67,6 @@ def __init__(self, training, batch_size, name=''):

# car: [N * 10 * 400 * 352 * 128]
# pedestrian/cyclist: [N * 10 * 200 * 240 * 128]
self.outputs = tf.scatter_nd(
self.coordinate, voxelwise, [self.batch_size, 10, cfg.INPUT_HEIGHT, cfg.INPUT_WIDTH, 128])
self.outputs = tf.scatter_nd(self.coordinate, voxelwise, [self.batch_size, 10, cfg.INPUT_HEIGHT, cfg.INPUT_WIDTH, 128])


30 changes: 10 additions & 20 deletions model/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ def __init__(self,
# hyper parameters and status
self.cls = cls
self.single_batch_size = single_batch_size
self.learning_rate = tf.Variable(
float(learning_rate), trainable=False, dtype=tf.float32)
self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=tf.float32)
self.global_step = tf.Variable(1, trainable=False)
self.epoch = tf.Variable(0, trainable=False)
self.epoch_add_op = self.epoch.assign(self.epoch + 1)
Expand Down Expand Up @@ -64,10 +63,8 @@ def __init__(self,
with tf.device('/gpu:{}'.format(dev)), tf.name_scope('gpu_{}'.format(dev)):
# must use name scope here since we do not want to create new variables
# graph
feature = FeatureNet(
training=self.is_train, batch_size=self.single_batch_size)
rpn = MiddleAndRPN(
input=feature.outputs, alpha=self.alpha, beta=self.beta, training=self.is_train)
feature = FeatureNet(training=self.is_train, batch_size=self.single_batch_size)
rpn = MiddleAndRPN(input=feature.outputs, alpha=self.alpha, beta=self.beta, training=self.is_train)
tf.get_variable_scope().reuse_variables()
# input
self.vox_feature.append(feature.feature)
Expand All @@ -76,8 +73,7 @@ def __init__(self,
self.targets.append(rpn.targets)
self.pos_equal_one.append(rpn.pos_equal_one)
self.pos_equal_one_sum.append(rpn.pos_equal_one_sum)
self.pos_equal_one_for_reg.append(
rpn.pos_equal_one_for_reg)
self.pos_equal_one_for_reg.append(rpn.pos_equal_one_for_reg)
self.neg_equal_one.append(rpn.neg_equal_one)
self.neg_equal_one_sum.append(rpn.neg_equal_one_sum)
# output
Expand All @@ -95,8 +91,7 @@ def __init__(self,
self.cls_neg_loss = rpn.cls_neg_loss_rec
self.params = tf.trainable_variables()
gradients = tf.gradients(self.loss, self.params)
clipped_gradients, gradient_norm = tf.clip_by_global_norm(
gradients, max_gradient_norm)
clipped_gradients, gradient_norm = tf.clip_by_global_norm(gradients, max_gradient_norm)

self.delta_output.append(delta_output)
self.prob_output.append(prob_output)
Expand All @@ -110,8 +105,7 @@ def __init__(self,
# self.xxxloss is only the loss for the lowest tower
with tf.device('/gpu:{}'.format(self.avail_gpus[0])):
self.grads = average_gradients(self.tower_grads)
self.update = [self.opt.apply_gradients(
zip(self.grads, self.params), global_step=self.global_step)]
self.update = [self.opt.apply_gradients(zip(self.grads, self.params), global_step=self.global_step)]
self.gradient_norm = tf.group(*self.gradient_norm)

self.update.extend(self.extra_update_ops)
Expand All @@ -122,12 +116,9 @@ def __init__(self,

self.anchors = cal_anchors()
# for predict and image summary
self.rgb = tf.placeholder(
tf.uint8, [None, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH, 3])
self.bv = tf.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.INPUT_HEIGHT, cfg.BV_LOG_FACTOR * cfg.INPUT_WIDTH, 3])
self.bv_heatmap = tf.placeholder(tf.uint8, [
None, cfg.BV_LOG_FACTOR * cfg.FEATURE_HEIGHT, cfg.BV_LOG_FACTOR * cfg.FEATURE_WIDTH, 3])
self.rgb = tf.placeholder(tf.uint8, [None, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH, 3])
self.bv = tf.placeholder(tf.uint8, [None, cfg.BV_LOG_FACTOR * cfg.INPUT_HEIGHT, cfg.BV_LOG_FACTOR * cfg.INPUT_WIDTH, 3])
self.bv_heatmap = tf.placeholder(tf.uint8, [None, cfg.BV_LOG_FACTOR * cfg.FEATURE_HEIGHT, cfg.BV_LOG_FACTOR * cfg.FEATURE_WIDTH, 3])
self.boxes2d = tf.placeholder(tf.float32, [None, 4])
self.boxes2d_scores = tf.placeholder(tf.float32, [None])

Expand All @@ -137,8 +128,7 @@ def __init__(self,
self.boxes2d, self.boxes2d_scores, max_output_size=cfg.RPN_NMS_POST_TOPK, iou_threshold=cfg.RPN_NMS_THRESH)

# summary and saver
self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2,
max_to_keep=10, pad_step_number=True, keep_checkpoint_every_n_hours=1.0)
self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2, max_to_keep=10, pad_step_number=True, keep_checkpoint_every_n_hours=1.0)

self.train_summary = tf.summary.merge([
tf.summary.scalar('train/loss', self.loss),
Expand Down
113 changes: 37 additions & 76 deletions model/rpn.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,87 +17,57 @@ def __init__(self, input, alpha=1.5, beta=1, sigma=3, training=True, name=''):
self.input = input
self.training = training
# groundtruth(target) - each anchor box, represent as △x, △y, △z, △l, △w, △h, rotation
self.targets = tf.placeholder(
tf.float32, [None, cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 14])
self.targets = tf.placeholder(tf.float32, [None, cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 14])
# postive anchors equal to one and others equal to zero(2 anchors in 1 position)
self.pos_equal_one = tf.placeholder(
tf.float32, [None, cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 2])
self.pos_equal_one = tf.placeholder(tf.float32, [None, cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 2])
self.pos_equal_one_sum = tf.placeholder(tf.float32, [None, 1, 1, 1])
self.pos_equal_one_for_reg = tf.placeholder(
tf.float32, [None, cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 14])
self.pos_equal_one_for_reg = tf.placeholder(tf.float32, [None, cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 14])
# negative anchors equal to one and others equal to zero
self.neg_equal_one = tf.placeholder(
tf.float32, [None, cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 2])
self.neg_equal_one = tf.placeholder(tf.float32, [None, cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH, 2])
self.neg_equal_one_sum = tf.placeholder(tf.float32, [None, 1, 1, 1])

with tf.variable_scope('MiddleAndRPN_' + name):
# convolutinal middle layers
temp_conv = ConvMD(3, 128, 64, 3, (2, 1, 1),
(1, 1, 1), self.input, name='conv1')
temp_conv = ConvMD(3, 64, 64, 3, (1, 1, 1),
(0, 1, 1), temp_conv, name='conv2')
temp_conv = ConvMD(3, 64, 64, 3, (2, 1, 1),
(1, 1, 1), temp_conv, name='conv3')
temp_conv = ConvMD(3, 128, 64, 3, (2, 1, 1), (1, 1, 1), self.input, name='conv1')
temp_conv = ConvMD(3, 64, 64, 3, (1, 1, 1), (0, 1, 1), temp_conv, name='conv2')
temp_conv = ConvMD(3, 64, 64, 3, (2, 1, 1), (1, 1, 1), temp_conv, name='conv3')
temp_conv = tf.transpose(temp_conv, perm=[0, 2, 3, 4, 1])
temp_conv = tf.reshape(
temp_conv, [-1, cfg.INPUT_HEIGHT, cfg.INPUT_WIDTH, 128])
temp_conv = tf.reshape(temp_conv, [-1, cfg.INPUT_HEIGHT, cfg.INPUT_WIDTH, 128])

# rpn
# block1:
temp_conv = ConvMD(2, 128, 128, 3, (2, 2), (1, 1),
temp_conv, training=self.training, name='conv4')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv5')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv6')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv7')
deconv1 = Deconv2D(128, 256, 3, (1, 1), (0, 0),
temp_conv, training=self.training, name='deconv1')
temp_conv = ConvMD(2, 128, 128, 3, (2, 2), (1, 1), temp_conv, training=self.training, name='conv4')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv5')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv6')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv7')
deconv1 = Deconv2D(128, 256, 3, (1, 1), (0, 0), temp_conv, training=self.training, name='deconv1')

# block2:
temp_conv = ConvMD(2, 128, 128, 3, (2, 2), (1, 1),
temp_conv, training=self.training, name='conv8')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv9')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv10')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv11')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv12')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv13')
deconv2 = Deconv2D(128, 256, 2, (2, 2), (0, 0),
temp_conv, training=self.training, name='deconv2')
temp_conv = ConvMD(2, 128, 128, 3, (2, 2), (1, 1), temp_conv, training=self.training, name='conv8')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv9')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv10')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv11')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv12')
temp_conv = ConvMD(2, 128, 128, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv13')
deconv2 = Deconv2D(128, 256, 2, (2, 2), (0, 0), temp_conv, training=self.training, name='deconv2')

# block3:
temp_conv = ConvMD(2, 128, 256, 3, (2, 2), (1, 1),
temp_conv, training=self.training, name='conv14')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv15')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv16')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv17')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv18')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1),
temp_conv, training=self.training, name='conv19')
deconv3 = Deconv2D(256, 256, 4, (4, 4), (0, 0),
temp_conv, training=self.training, name='deconv3')
temp_conv = ConvMD(2, 128, 256, 3, (2, 2), (1, 1), temp_conv, training=self.training, name='conv14')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv15')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv16')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv17')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv18')
temp_conv = ConvMD(2, 256, 256, 3, (1, 1), (1, 1), temp_conv, training=self.training, name='conv19')
deconv3 = Deconv2D(256, 256, 4, (4, 4), (0, 0), temp_conv, training=self.training, name='deconv3')

# final:
temp_conv = tf.concat([deconv3, deconv2, deconv1], -1)
# Probability score map, scale = [None, 200/100, 176/120, 2]
p_map = ConvMD(2, 768, 2, 1, (1, 1), (0, 0), temp_conv,
training=self.training, activation=False, bn=False, name='conv20')
p_map = ConvMD(2, 768, 2, 1, (1, 1), (0, 0), temp_conv, training=self.training, activation=False, bn=False, name='conv20')
# Regression(residual) map, scale = [None, 200/100, 176/120, 14]
r_map = ConvMD(2, 768, 14, 1, (1, 1), (0, 0),
temp_conv, training=self.training, activation=False, bn=False, name='conv21')
r_map = ConvMD(2, 768, 14, 1, (1, 1), (0, 0), temp_conv, training=self.training, activation=False, bn=False, name='conv21')
# softmax output for positive anchor and negative anchor, scale = [None, 200/100, 176/120, 1]
self.p_pos = tf.sigmoid(p_map)
#self.p_pos = tf.nn.softmax(p_map, dim=3)
self.output_shape = [cfg.FEATURE_HEIGHT, cfg.FEATURE_WIDTH]

self.cls_pos_loss = (-self.pos_equal_one * tf.log(self.p_pos + small_addon_for_BCE)) / self.pos_equal_one_sum
Expand All @@ -108,8 +78,7 @@ def __init__(self, input, alpha=1.5, beta=1, sigma=3, training=True, name=''):
self.cls_neg_loss_rec = tf.reduce_sum( self.cls_neg_loss )


self.reg_loss = smooth_l1(r_map * self.pos_equal_one_for_reg, self.targets *
self.pos_equal_one_for_reg, sigma) / self.pos_equal_one_sum
self.reg_loss = smooth_l1(r_map * self.pos_equal_one_for_reg, self.targets * self.pos_equal_one_for_reg, sigma) / self.pos_equal_one_sum
self.reg_loss = tf.reduce_sum(self.reg_loss)

self.loss = tf.reduce_sum(self.cls_loss + self.reg_loss)
Expand All @@ -122,11 +91,9 @@ def smooth_l1(deltas, targets, sigma=3.0):
sigma2 = sigma * sigma
diffs = tf.subtract(deltas, targets)
smooth_l1_signs = tf.cast(tf.less(tf.abs(diffs), 1.0 / sigma2), tf.float32)

smooth_l1_option1 = tf.multiply(diffs, diffs) * 0.5 * sigma2
smooth_l1_option2 = tf.abs(diffs) - 0.5 / sigma2
smooth_l1_add = tf.multiply(smooth_l1_option1, smooth_l1_signs) + \
tf.multiply(smooth_l1_option2, 1 - smooth_l1_signs)
smooth_l1_add = tf.multiply(smooth_l1_option1, smooth_l1_signs) + tf.multiply(smooth_l1_option2, 1 - smooth_l1_signs)
smooth_l1 = smooth_l1_add

return smooth_l1
Expand All @@ -139,16 +106,13 @@ def ConvMD(M, Cin, Cout, k, s, p, input, training=True, activation=True, bn=True
if(M == 2):
paddings = (np.array(temp_p)).repeat(2).reshape(4, 2)
pad = tf.pad(input, paddings, "CONSTANT")
temp_conv = tf.layers.conv2d(
pad, Cout, k, strides=s, padding="valid", reuse=tf.AUTO_REUSE, name=scope)
temp_conv = tf.layers.conv2d(pad, Cout, k, strides=s, padding="valid", reuse=tf.AUTO_REUSE, name=scope)
if(M == 3):
paddings = (np.array(temp_p)).repeat(2).reshape(5, 2)
pad = tf.pad(input, paddings, "CONSTANT")
temp_conv = tf.layers.conv3d(
pad, Cout, k, strides=s, padding="valid", reuse=tf.AUTO_REUSE, name=scope)
temp_conv = tf.layers.conv3d(pad, Cout, k, strides=s, padding="valid", reuse=tf.AUTO_REUSE, name=scope)
if bn:
temp_conv = tf.layers.batch_normalization(
temp_conv, axis=-1, fused=True, training=training, reuse=tf.AUTO_REUSE, name=scope)
temp_conv = tf.layers.batch_normalization(temp_conv, axis=-1, fused=True, training=training, reuse=tf.AUTO_REUSE, name=scope)
if activation:
return tf.nn.relu(temp_conv)
else:
Expand All @@ -160,14 +124,11 @@ def Deconv2D(Cin, Cout, k, s, p, input, training=True, bn=True, name='deconv'):
paddings = (np.array(temp_p)).repeat(2).reshape(4, 2)
pad = tf.pad(input, paddings, "CONSTANT")
with tf.variable_scope(name) as scope:
temp_conv = tf.layers.conv2d_transpose(
pad, Cout, k, strides=s, padding="SAME", reuse=tf.AUTO_REUSE, name=scope)
temp_conv = tf.layers.conv2d_transpose(pad, Cout, k, strides=s, padding="SAME", reuse=tf.AUTO_REUSE, name=scope)
if bn:
temp_conv = tf.layers.batch_normalization(
temp_conv, axis=-1, fused=True, training=training, reuse=tf.AUTO_REUSE, name=scope)
temp_conv = tf.layers.batch_normalization(temp_conv, axis=-1, fused=True, training=training, reuse=tf.AUTO_REUSE, name=scope)
return tf.nn.relu(temp_conv)


if(__name__ == "__main__"):
m = MiddleAndRPN(tf.placeholder(
tf.float32, [None, 10, cfg.INPUT_HEIGHT, cfg.INPUT_WIDTH, 128]))
m = MiddleAndRPN(tf.placeholder(tf.float32, [None, 10, cfg.INPUT_HEIGHT, cfg.INPUT_WIDTH, 128]))

0 comments on commit 9b048e5

Please sign in to comment.