organization

Moved old results images into a folder, moved new correlation images + files into folder, commented some code

Author: Joanna

.DS_Store

@@ -111,10 +111,6 @@
     end
 end
 
-%% Jaccard
-
-%% PCA?
-
 %% Plotting: ssimvals (comparing digits across manips)
 
 ys = zeros(8, 10);
@@ -193,7 +189,6 @@
 end
 
 %% Plotting: correlations (dataset to dataset, z-score per dataset, opt. abs val)
-% the next section is more meaningful, btw
 
 for i = 1:10
     im = figure(i);
@@ -208,7 +203,7 @@
     yticklabels(maniplabels);
     title(['Dataset to dataset mean image correlation z-scored (over dataset): ', int2str(i - 1)]);
     colorbar
-    % caxis([-1, 1])
+    caxis([0, 1])
 end
 
 %% Plotting: correlations (dataset to dataset, z-score overall, opt. abs val)

MNIST-image-distance/.DS_Store

@@ -0,0 +1,28 @@
+---- The code in optimizers.py has been modified from the original code provided by Zenke et al, 2017. to include functions for
+---- examining weight space and printing data.
+---- The code in utils.py is the original code.
+
+Installation instructions:
+
+- The following packages and versions are required to run the code: 
+	- matplotlib
+	- pandas
+	- scipy
+	- numpy
+	- seaborn
+	- np_utils
+	- imageio
+	- PIL
+	- gzip
+	- tensorflow 1.2.1
+	- keras 2.0.5
+
+Additionally, Python 3.0 or higher is required.
+
+- We have modified the skdata library from its original format to be compatible with all operating systems (the original format is only
+compatible with MacOS). To install:
+	- Unzip the skdata folder in catastrophic-interference
+	- navigate to the skdata folder
+	- Run "python setup.py develop"
+	- Run "python setup.py install"
+

manip-manip-corrs+zenke0.png MNIST-image-distance/Correlations w: Zenke/manip-manip-corrs+zenke0.png

@@ -0,0 +1,209 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys, os
+import numpy as np
+import scipy 
+import tensorflow as tf
+
+import imageio
+import gzip
+from PIL import Image
+
+import seaborn as sns
+import matplotlib.colors as colors
+import matplotlib.cm as cmx
+import matplotlib.pyplot as plt
+#from tqdm import trange, tqdm
+
+import keras
+from keras import backend as K
+from keras.models import Sequential, load_model
+from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D
+from keras.optimizers import SGD, Adam, RMSprop, Optimizer
+from keras.callbacks import Callback
+from collections import OrderedDict
+
+from helpers import utils
+
+tf.logging.set_verbosity(tf.logging.INFO)
+
+# Extract labels from MNIST labels into vector
+def extract_labels(filename, num_images):
+ with gzip.open(filename) as bytestream:
+   bytestream.read(8)
+   buf = bytestream.read(1 * num_images)
+   labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)
+ return labels
+
+train_labels = extract_labels("MNIST-data/train-labels-idx1-ubyte.gz", 60000)
+eval_labels = extract_labels("MNIST-data/t10k-labels-idx1-ubyte.gz", 10000)
+print(np.shape(train_labels))
+print(np.shape(eval_labels))
+
+# original train
+train_original = np.zeros((60000,28,28), dtype=np.float32)
+images_original = ["MNIST-processed-training/original/original{0}.png".format(k) for k in range(1,60000)]
+for i in range(len(images_original)):
+   img = np.array(Image.open(images_original[i]))
+   train_original[i, :, :] = img
+
+# original test
+eval_original = np.zeros((10000,28,28), dtype=np.float32)
+images2_original = ["MNIST-processed-test/original/test-original{0}.png".format(k) for k in range(1,10001)]
+
+for i in range(len(images2_original)):
+   img = np.array(Image.open(images2_original[i]))
+   eval_original[i, :, :] = img
+
+# ROTATE 90 train
+train_rot90 = np.zeros((60000,28,28), dtype=np.float32)
+images_rot90 = ["MNIST-processed-training/rot90/rot90{0}.png".format(k) for k in range(1,60000)]
+
+for i in range(len(images_rot90)):
+   img = np.array(Image.open(images_rot90[i]))
+   train_rot90[i, :, :] = img
+
+# ROTATE 90 test
+eval_rot90 = np.zeros((10000,28,28), dtype=np.float32)
+images2_rot90 = ["MNIST-processed-test/rot90/test-rot90{0}.png".format(k) for k in range(1,10001)]
+
+for i in range(len(images2_rot90)):
+   img = np.array(Image.open(images2_rot90[i]))
+   eval_rot90[i, :, :] = img
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# # Network params
+# n_hidden_units = 2000
+# activation_fn = tf.nn.relu
+
+# Optimization params
+batch_size = 256
+num_classes = 10
+epochs = 5 # epochs per task
+# learning_rate=1e-3
+# xi = 0.1
+
+# the data, train and test sets
+x_train = train_original
+x_test = eval_original
+y_train = train_labels
+y_test = eval_labels
+
+
+if K.image_data_format() == 'channels_first':
+   x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
+   x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
+   input_shape = (1, img_rows, img_cols)
+else:
+   x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
+   x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
+   input_shape = (img_rows, img_cols, 1)
+   
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+
+# convert class vectors to binary class matrices
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+
+
+model = Sequential()
+model.add(Conv2D(32, kernel_size=(3, 3),
+                activation='relu',
+                input_shape=input_shape))
+model.add(Conv2D(64, (3, 3), activation='relu'))
+model.add(MaxPooling2D(pool_size=(2, 2)))
+model.add(Dropout(0.25))
+model.add(Flatten())
+model.add(Dense(128, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(num_classes, activation='softmax'))
+
+model.compile(loss=keras.losses.categorical_crossentropy,
+             optimizer=keras.optimizers.Adadelta(),
+             metrics=['accuracy'])
+
+
+model.fit(x_train, y_train,
+         batch_size=batch_size,
+         epochs=epochs,
+         verbose=1,
+         validation_data=(x_test, y_test))
+score = model.evaluate(x_test, y_test, verbose=0)
+print('Test loss:', score[0])
+print('Test accuracy:', score[1])
+
+
+
+model2 = Sequential()
+
+# Convolutional layer 1 and input layer
+model2.add(Conv2D(32, kernel_size=(3, 3),
+                activation='relu',
+                input_shape=input_shape))
+
+# Convolutional layer 2
+model2.add(Conv2D(64, (3, 3), activation='relu'))
+
+# Pooling layer 1
+model2.add(MaxPooling2D(pool_size=(2, 2)))
+
+# Dropout layer with flattening
+model2.add(Dropout(0.25))
+model2.add(Flatten())
+
+# Dense layer 1 with dropout 
+model2.add(Dense(128, activation='relu'))
+model2.add(Dropout(0.5))
+
+# Dense layer 2
+model2.add(Dense(num_classes, activation='softmax'))
+
+model2.compile(loss=keras.losses.categorical_crossentropy,
+             optimizer=keras.optimizers.Adadelta(),
+             metrics=['accuracy'])
+
+model.save('trained_model.h5')
+model2 = load_model('trained_model.h5')
+
+x_train2 = train_rot90
+x_test2 = eval_rot90
+
+
+if K.image_data_format() == 'channels_first':
+   x_train2 = x_train2.reshape(x_train2.shape[0], 1, img_rows, img_cols)
+   x_test2 = x_test2.reshape(x_test2.shape[0], 1, img_rows, img_cols)
+   input_shape = (1, img_rows, img_cols)
+else:
+   x_train2 = x_train2.reshape(x_train2.shape[0], img_rows, img_cols, 1)
+   x_test2 = x_test2.reshape(x_test2.shape[0], img_rows, img_cols, 1)
+   input_shape = (img_rows, img_cols, 1)
+   
+x_train2 = x_train2.astype('float32')
+x_test2 = x_test2.astype('float32')
+x_train2 /= 255
+x_test2 /= 255
+
+
+# Continue training
+model2.fit(x_train2, y_train,
+         batch_size=batch_size,
+         epochs=epochs,
+         verbose=1,
+         validation_data=(x_test, y_test))
+score1 = model2.evaluate(x_test, y_test, verbose=0)
+score2 = model2.evaluate(x_test2, y_test, verbose=0)
+
+print('Original data set')
+print('Test loss:', score1[0])
+print('Test accuracy:', score1[1])
+
+print('Second data set')
+print('Test loss:', score2[0])
+print('Test accuracy:', score2[1])

manip-manip-corrs+zenke1.png MNIST-image-distance/Correlations w: Zenke/manip-manip-corrs+zenke1.png

@@ -13,14 +13,17 @@
 import tensorflow as tf
 
 import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib.pylab import figure, axes, pie, title, show
+import seaborn as sb
+
 import keras
 from keras import backend as K
 from keras.optimizers import Optimizer
 from keras.callbacks import Callback
 from utils import extract_weight_changes, compute_updates
 from synapticpenalty import importancePenalty
 from collections import OrderedDict
-from fisher_comp import fishers
 
 class SynapticOptimizer(Optimizer):
     """An optimizer whose loss depends on its own updates."""
@@ -32,7 +35,7 @@ def _allocate_vars(self, names):
         #TODO: add names, better shape/init checking
         self.vars = {name: self._allocate_var(name=name) for name in names}
 
-    def __init__(self, opt, step_updates=[], task_updates=[], init_updates=[], task_metrics = {}, fisher_vars={}, regularizer_fn=importancePenalty,
+    def __init__(self, opt, step_updates=[], task_updates=[], init_updates=[], task_metrics = {}, regularizer_fn=importancePenalty,
                 lam=1.0, model=None, compute_average_loss=False, compute_average_weights=False, **kwargs):
         """Instantiate an optimzier that depends on its own updates.
 
@@ -83,7 +86,7 @@ def update_fn(vars, weight, prev_val):
         self.step_updates = step_updates
         self.task_updates = task_updates
         self.init_updates = init_updates
-        self.fisher_vars = fisher_vars
+        # self.fisher_vars = fisher_vars
         self.compute_average_loss = compute_average_loss
         self.regularizer_fn = regularizer_fn
         # Compute loss and gradients
@@ -108,6 +111,50 @@ def closeFiles(self):
         except Error:
             print("FILE NOT FOUND")
 
+    def outputImageData(self, tasknumber, strength):
+    	fisher_fn = "fisher_task{0}_strength{1}nobounds.png"
+    	weight_fn = "weight_task{0}_strength{1}nobounds.png"
+
+    	reshaped = list()
+
+    	for weight in self.weights:
+    		wt = K.get_value(weight)
+    		
+    		if wt.size >= 2000:
+	    		rows = wt.size/2000
+	    		dat = np.reshape(wt, (int(rows), 2000))
+	    		reshaped.append(dat)
+
+    	weight_dat = np.concatenate(reshaped)
+    	wfn = sb.heatmap(weight_dat, cmap="coolwarm")
+    	wfg = wfn.get_figure()
+
+
+    	name = weight_fn.format(tasknumber, strength)
+    	wfg.savefig(name)
+
+    	wfg.clf()
+
+    	reshaped = list()
+    	# for fish in self._fishers:
+    	# 	dat = K.get_value(fish)
+
+
+    	# 	if dat.size >= 2000:
+    	# 		dat = np.reshape(dat, (int(dat.size/2000), 2000))
+    	# 		reshaped.append(dat)
+
+    	# fish_dat = np.concatenate(reshaped)
+    	# ffn = sb.heatmap(fish_dat, cmap="coolwarm")
+    	# ffg = ffn.get_figure()
+    	# name = fisher_fn.format(tasknumber, strength)
+
+    	# ffg.savefig(name)
+
+    	# ffg.clf()
+
+
+
     def createFiles(self, fishername, weightname):
         self.weight_filename = weightname
         self.fisher_filename = fishername
@@ -130,9 +177,9 @@ def print_weight_state(self):
                 self.weightfile.write(np.array_str(K.get_value(weight)))
 
 
-    def print_fisher_state(self):
-    	for fish in self._fishers:
-    		self.fisherfile.write(np.array_str(K.get_value(fish)))
+    # def print_fisher_state(self):
+    # 	for fish in self._fishers:
+    # 		self.fisherfile.write(np.array_str(K.get_value(fish)))
 
 
     def get_updates(self, weights, constraints, initial_loss, model=None):
@@ -183,8 +230,8 @@ def get_updates(self, weights, constraints, initial_loss, model=None):
                 self.delta_loss = tf.Variable(0.0, trainable=False, name="delta_loss")
             self.ema_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope.name)
             self.init_ema_vars = tf.variables_initializer(self.ema_vars)
-        if self.fisher_vars:
-            self._fishers = fishers(self.model)
+        # if self.fisher_vars:
+        #     self._fishers = fishers(self.model)
             #fish = compute_fisher_information(model)
             #self.vars['fishers'] = dict(zip(weights, self._fishers))
           	#fishers, avg_fishers, update_fishers, zero_fishers = compute_fisher_information(model)