bump

cond_vaegan.py

@@ -4,7 +4,7 @@
 import deeppy as dp
 import deeppy.expr as expr
 
-from vaegan import KLDivergence, NegativeGradient, SquareError
+from vaegan import KLDivergence, NegativeGradient, ScaleGradient, SquareError, WeightedParameter
 
 
 class AppendSpatially(expr.base.Binary):
@@ -45,21 +45,30 @@ def __call__(self, x, y):
         return x
 
 
-class ConditionalVAEGAN(dp.base.Model):
+class ConditionalVAEGAN(dp.base.Model, dp.base.CollectionMixin):
     def __init__(self, encoder, sampler, generator, discriminator, mode,
-                 reconstruct_error=None):
+                 reconstruct_error=None, vae_grad_scale=1.0):
         self.encoder = encoder
         self.sampler = sampler
-        self.generator = generator
         self.mode = mode
         self.discriminator = discriminator
+        self.vae_grad_scale = vae_grad_scale
         self.eps = 1e-4
         if reconstruct_error is None:
             reconstruct_error = SquareError()
         self.reconstruct_error = reconstruct_error
+        generator.params = [p.parent if isinstance(p, WeightedParameter) else p
+                            for p in generator.params]
         if self.mode == 'vaegan':
+            generator.params = [WeightedParameter(p, vae_grad_scale)
+                                for p in generator.params]
             self.generator_neg = deepcopy(generator)
             self.generator_neg.params = [p.share() for p in generator.params]
+        if self.mode == 'gan':
+            generator.params = [WeightedParameter(p, -1.0)
+                                for p in generator.params]
+        self.generator = generator
+        self.collection = [self.encoder, self.sampler, self.generator, self.discriminator]
 
     def _embed_expr(self, x, y):
         h_enc = self.encoder(x, y)
@@ -81,24 +90,20 @@ def setup(self, x_shape, y_shape):
             z, z_mu, z_log_sigma, z_eps = self.sampler(h_enc)
             self.kld = KLDivergence()(z_mu, z_log_sigma)
             x_tilde = self.generator(z, self.y_src)
-#            if self.mode == 'vaegan':
-#                x_tilde = ScaleGradient()(x_tilde)
             self.logpxz = self.reconstruct_error(x_tilde, self.x_src)
-            loss = self.kld + expr.sum(self.logpxz)
+            loss = 0.5*self.kld + expr.sum(self.logpxz)
 
         if self.mode in ['gan', 'vaegan']:
             y = self.y_src
             if self.mode == 'gan':
                 z = self.sampler.samples()
                 x_tilde = self.generator(z, y)
-                x_tilde = NegativeGradient()(x_tilde)
                 gen_size = batch_size
             elif self.mode == 'vaegan':
-                z = NegativeGradient()(z)
+                z = ScaleGradient(0.0)(z)
                 z = expr.Concatenate(axis=0)(z, z_eps)
                 y = expr.Concatenate(axis=0)(y, self.y_src)
                 x_tilde = self.generator_neg(z, y)
-                x_tilde = NegativeGradient()(x_tilde)
                 gen_size = batch_size*2
             x = expr.Concatenate(axis=0)(self.x_src, x_tilde)
             y = expr.Concatenate(axis=0)(y, self.y_src)

cond_vaegan_cifar.py

@@ -16,10 +16,11 @@ def affine(n_out, gain):
     return expr.nnet.Affine(n_out=n_out, weights=dp.AutoFiller(gain))
 
 
-def conv(n_filters, filter_size, gain=1.0):
+def conv(n_filters, filter_size, stride=1, gain=1.0):
     return expr.nnet.Convolution(
-        n_filters=n_filters, strides=(1, 1), weights=dp.AutoFiller(gain),
-        filter_shape=(filter_size, filter_size), border_mode='same',
+        n_filters=n_filters, strides=(stride, stride),
+        weights=dp.AutoFiller(gain), filter_shape=(filter_size, filter_size),
+        border_mode='same',
     )
 
 
@@ -84,23 +85,22 @@ def model_expressions(img_shape):
         conv(n_channels, 3, gain=gain),
     ])
     discriminator = cond_vaegan.ConditionalSequential([
-        conv(32, 5, gain=gain),
-        pool(),
+        conv(32, 5, stride=2, gain=gain),
         expr.nnet.ReLU(),
         expr.nnet.SpatialDropout(0.2),
-        conv(64, 5, gain=gain),
-        pool(),
+        conv(64, 5, stride=2, gain=gain),
         expr.nnet.ReLU(),
         expr.nnet.SpatialDropout(0.2),
         conv(96, 3, gain=gain),
-        expr.nnet.ReLU(),
-        expr.nnet.SpatialDropout(0.2),
         expr.Reshape((-1, 96*8*8)),
-        expr.Concatenate(axis=1),
-        affine(n_discriminator, gain),
-        expr.nnet.ReLU(),
-        expr.nnet.Dropout(0.5),
-        affine(1, gain),
+#        expr.nnet.ReLU(),
+#        expr.nnet.SpatialDropout(0.2),
+#        expr.Reshape((-1, 96*8*8)),
+#        expr.Concatenate(axis=1),
+#        affine(n_discriminator, gain),
+##        expr.nnet.ReLU(),
+##        expr.nnet.Dropout(0.5),
+##        affine(1, gain),
         expr.nnet.Sigmoid(),
     ])
     return encoder, sampler, generator, discriminator
@@ -112,7 +112,7 @@ def clip_range(imgs):
 
 def run():
     mode = 'gan'
-    experiment_name = mode
+    experiment_name = mode + '_stride_local_discrimination'
     filename = 'savestates/cifar_cond_' + experiment_name + '.pickle'
     in_filename = filename
     in_filename = None
@@ -181,9 +181,12 @@ def plot():
 
     # Train network
     runs = [
-        (150, dp.RMSProp(learn_rate=0.1)),
-        (150, dp.RMSProp(learn_rate=0.08)),
+#        (10, dp.RMSProp(learn_rate=0.08)),
+#        (25, dp.RMSProp(learn_rate=0.12)),
+#        (100, dp.RMSProp(learn_rate=0.1)),
+        (150, dp.RMSProp(learn_rate=0.075)),
         (150, dp.RMSProp(learn_rate=0.06)),
+        (150, dp.RMSProp(learn_rate=0.05)),
         (150, dp.RMSProp(learn_rate=0.04)),
         (25, dp.RMSProp(learn_rate=0.01)),
     ]

cond_vaegan_mnist.py

@@ -21,15 +21,19 @@ def model_expressions(img_shape):
     sigma = 0.001
     n_in = np.prod(img_shape)
     n_encoder = 1024
-    n_hidden = 64
-    n_generator = 1024
-    n_discriminator = 1024
+    n_hidden = 32
+    n_generator = 2048
+    n_discriminator = 2048
 
     encoder = cond_vaegan.ConditionalSequential([
         expr.Concatenate(axis=1),
         affine(n_encoder, gain),
+        expr.nnet.BatchNormalization(),
         expr.nnet.ReLU(),
+        expr.nnet.Dropout(0.5),
+        expr.Concatenate(axis=1),
         affine(n_encoder, gain),
+        expr.nnet.BatchNormalization(),
         expr.nnet.ReLU(),
     ])
     sampler = vaegan.NormalSampler(
@@ -46,18 +50,21 @@ def model_expressions(img_shape):
         affine(n_generator, gain),
         expr.nnet.BatchNormalization(),
         expr.nnet.ReLU(),
+        expr.Concatenate(axis=1),
         affine(n_in, gain),
         expr.nnet.Sigmoid(),
     ])
     discriminator = cond_vaegan.ConditionalSequential([
-        expr.nnet.Dropout(0.5),
         expr.Concatenate(axis=1),
         affine(n_discriminator, gain),
         expr.nnet.ReLU(),
-        expr.nnet.Dropout(0.5),
+        expr.nnet.Dropout(0.25),
         expr.Concatenate(axis=1),
         affine(n_discriminator, gain),
+        expr.nnet.BatchNormalization(),
         expr.nnet.ReLU(),
+        expr.nnet.Dropout(0.25),
+        expr.Concatenate(axis=1),
         affine(1, gain),
         expr.nnet.Sigmoid(),
 
@@ -71,9 +78,10 @@ def to_b01c(imgs_flat, img_shape):
 
 
 def run():
-    mode = 'gan'
-    experiment_name = mode
-    filename = 'savestates/mnist_cond_' + experiment_name + '.pickle'
+    mode = 'vaegan'
+    vae_grad_scale = 0.025
+    experiment_name = mode + 'scale_%.5f' % vae_grad_scale
+    filename = 'savestates/mnist_' + experiment_name + '.pickle'
     in_filename = filename
     in_filename = None
     print('experiment_name', experiment_name)
@@ -95,6 +103,7 @@ def run():
     x_train = np.reshape(x_train, (x_train.shape[0], -1))
     x_test = np.reshape(x_test, (x_test.shape[0], -1))
 
+
     # Setup network
     if in_filename is None:
         print('Creating new model')
@@ -111,11 +120,12 @@ def run():
         generator=generator,
         discriminator=discriminator,
         mode=mode,
-        reconstruct_error=expr.nnet.BinaryCrossEntropy()
+        reconstruct_error=expr.nnet.BinaryCrossEntropy(),
+        vae_grad_scale=vae_grad_scale,
     )
 
     # Prepare network inputs
-    batch_size = 64
+    batch_size = 128
     train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size,
                                      epoch_size=250)
 
@@ -136,6 +146,7 @@ def run():
 
     def plot():
         model.phase = 'test'
+        model.sampler.batch_size=100
         examples_z = model.embed(examples, examples_y)
         examples_recon = model.reconstruct(examples_z, examples_y)
         recon_video.append(img_tile(to_b01c(examples_recon, img_shape)))
@@ -144,12 +155,13 @@ def plot():
         model.setup(**train_input.shapes)
         model.phase = 'train'
 
+
     # Train network
     runs = [
-        (50, dp.RMSProp(learn_rate=0.3)),
-        (150, dp.RMSProp(learn_rate=0.1)),
-        (5, dp.RMSProp(learn_rate=0.05)),
+        (75, dp.RMSProp(learn_rate=0.075)),
+        (25, dp.RMSProp(learn_rate=0.05)),
         (5, dp.RMSProp(learn_rate=0.01)),
+        (5, dp.RMSProp(learn_rate=0.005)),
     ]
     try:
         for n_epochs, learn_rule in runs:
@@ -167,12 +179,71 @@ def plot():
         expressions = encoder, sampler, generator, discriminator
         pickle.dump(expressions, f)
 
+    model.phase = 'test'
+    batch_size = 128
+    model.sampler.batch_size=128
+    z = []
+    i = 0
+    z = model.embed(x_train, y_train)
+    print(z.shape)
+    z_mean = np.mean(z, axis=0)
+    z_std = np.std(z, axis=0)
+    z_cov = np.cov(z.T)
+    print(np.mean(z_mean), np.std(z_mean))
+    print(np.mean(z_std), np.std(z_std))
+    print(z_mean.shape, z_std.shape, z_cov.shape)
+
+
+    raw_input('\n\ngenerate latent space video?\n')
     print('Generating latent space video')
     walk_video = Video('plots/mnist_' + experiment_name + '_walk.mp4')
-    for z in random_walk(samples_z, 500, step_std=0.15):
+    for z in random_walk(samples_z, 500, n_dir_steps=10, mean=z_mean, std=z_cov):
         samples = model.reconstruct(z, samples_y)
         walk_video.append(img_tile(to_b01c(samples, img_shape)))
 
 
+
+    print('Generating AdversarialMNIST dataset')
+    _, y_train, _, y_test = dataset.arrays(dp_dtypes=True)
+    n = 0
+    batch_size = 512
+    advmnist_size = 1e6
+    x_advmnist = np.empty((advmnist_size, 28*28))
+    y_advmnist = np.empty((advmnist_size,))
+    while n < advmnist_size:
+        samples_z = np.random.multivariate_normal(mean=z_mean, cov=z_cov,
+                                                  size=batch_size)
+        samples_z = samples_z.astype(dp.float_)
+        start_idx = n % len(y_train)
+        stop_idx = (n + batch_size) % len(y_train)
+        if start_idx > stop_idx:
+            samples_y = np.concatenate([y_train[start_idx:], y_train[:stop_idx]])
+        else:
+            samples_y = y_train[start_idx:stop_idx]
+        y_advmnist[n:n+batch_size] = samples_y[:advmnist_size-n]
+        samples_y = one_hot(samples_y, n_classes).astype(dp.float_)
+        samples = model.reconstruct(samples_z, samples_y)
+        x_advmnist[n:n+batch_size] = samples[:advmnist_size-n]
+        n += batch_size
+
+
+    x_train = x_advmnist
+    y_train = y_advmnist
+    import sklearn.neighbors
+    clf = sklearn.neighbors.KNeighborsClassifier(n_neighbors=1, algorithm='brute', n_jobs=-1)
+    clf.fit(x_train, y_train)
+    print('KNN predict')
+    step = 2500
+    errors = []
+    i = 0
+    while i < len(x_test):
+        print(i)
+        errors.append(clf.predict(x_test[i:i+step]) != y_test[i:i+step])
+        i += step
+    error = np.mean(errors)
+    print('Test error rate: %.4f' % error)
+
+    print('DONE ' + experiment_name)
+
 if __name__ == '__main__':
     run()

util.py

@@ -6,7 +6,9 @@
 
 
 def img_tile(imgs):
-    return dp.misc.img_tile(dp.misc.img_stretch(imgs))
+    if imgs.dtype not in [np.int_, np.uint8]:
+        imgs = dp.misc.img_stretch(imgs)
+    return dp.misc.img_tile(imgs)
 
 
 def plot_img(img, title, filename=None):
@@ -26,13 +28,19 @@ def one_hot(labels, n_classes):
     return onehot
 
 
-def random_walk(start_pos, n_steps, step_std):
+def random_walk(start_pos, n_steps, n_dir_steps=10, mean=0.0, std=1.0):
     pos = np.copy(start_pos)
     for i in range(n_steps):
-        if i % 10 == 0:
-            step = np.random.normal(scale=step_std, size=pos.shape)
-            sign_change = np.logical_and(np.abs(pos) > 0.7,
-                                         np.sign(pos) == np.sign(step))
-            step[sign_change] *= -1
+        if i % n_dir_steps == 0:
+            if isinstance(mean, float):
+                next_point = np.random.normal(
+                    scale=std, loc=mean, size=pos.shape
+                )
+            else:
+                next_point = np.random.multivariate_normal(
+                    mean=mean, cov=std, size=pos.shape[0]
+                )
+            step = (next_point - pos)
+            step /= n_dir_steps
         pos += step
         yield pos