diff --git a/.gitignore b/.gitignore index 1b463cd..d1cd11a 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,6 @@ coverage.xml docs/_build/ # PyBuilder -target/ \ No newline at end of file +target/ + +keras-tests/tfkeras/used_layers.json \ No newline at end of file diff --git a/keras-tests/make_activation_layers.py b/keras-tests/legacy/make_activation_layers.py similarity index 100% rename from keras-tests/make_activation_layers.py rename to keras-tests/legacy/make_activation_layers.py diff --git a/keras-tests/make_causal_conv1d.py b/keras-tests/legacy/make_causal_conv1d.py similarity index 100% rename from keras-tests/make_causal_conv1d.py rename to keras-tests/legacy/make_causal_conv1d.py diff --git a/keras-tests/make_conv1d.py b/keras-tests/legacy/make_conv1d.py similarity index 100% rename from keras-tests/make_conv1d.py rename to keras-tests/legacy/make_conv1d.py diff --git a/keras-tests/make_imdb_fasttext_unit_test.py b/keras-tests/legacy/make_imdb_fasttext_unit_test.py similarity index 100% rename from keras-tests/make_imdb_fasttext_unit_test.py rename to keras-tests/legacy/make_imdb_fasttext_unit_test.py diff --git a/keras-tests/make_imdb_lstm_unit_test.py b/keras-tests/legacy/make_imdb_lstm_unit_test.py similarity index 100% rename from keras-tests/make_imdb_lstm_unit_test.py rename to keras-tests/legacy/make_imdb_lstm_unit_test.py diff --git a/keras-tests/make_mnist_cnn_unit_test.py b/keras-tests/legacy/make_mnist_cnn_unit_test.py similarity index 100% rename from keras-tests/make_mnist_cnn_unit_test.py rename to keras-tests/legacy/make_mnist_cnn_unit_test.py diff --git a/keras-tests/make_mnist_mlp_constraint_unit_test.py b/keras-tests/legacy/make_mnist_mlp_constraint_unit_test.py similarity index 100% rename from keras-tests/make_mnist_mlp_constraint_unit_test.py rename to keras-tests/legacy/make_mnist_mlp_constraint_unit_test.py diff --git a/keras-tests/make_mnist_mlp_unit_test.py b/keras-tests/legacy/make_mnist_mlp_unit_test.py similarity index 100% rename from keras-tests/make_mnist_mlp_unit_test.py rename to keras-tests/legacy/make_mnist_mlp_unit_test.py diff --git a/keras-tests/make_simple_lstm.py b/keras-tests/legacy/make_simple_lstm.py similarity index 100% rename from keras-tests/make_simple_lstm.py rename to keras-tests/legacy/make_simple_lstm.py diff --git a/keras-tests/make_sparsexent_mlp.py b/keras-tests/legacy/make_sparsexent_mlp.py similarity index 100% rename from keras-tests/make_sparsexent_mlp.py rename to keras-tests/legacy/make_sparsexent_mlp.py diff --git a/keras-tests/util.py b/keras-tests/legacy/util.py similarity index 100% rename from keras-tests/util.py rename to keras-tests/legacy/util.py diff --git a/keras-tests/tfkeras/activations.py b/keras-tests/tfkeras/activations.py new file mode 100644 index 0000000..2a3381a --- /dev/null +++ b/keras-tests/tfkeras/activations.py @@ -0,0 +1,34 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm + +np.random.seed(1337) + + +activation = [act for act in dir(tf.keras.activations) + if not act.startswith('_') and act not in ('serialize', 'deserialize', 'get')] + +activation.remove('exponential') #not supported + +use_dense = [True, False] + +@grid(**globals()) +def generate_models(activation, use_dense): + inp = tf.keras.layers.Input((2,)) + if use_dense: + out = tf.keras.layers.Dense(4, activation=activation)(inp) + else: + out = tf.keras.layers.Activation(activation)(inp) + model = tf.keras.models.Model(inp, out) + return model + + +def run(): + gen = generate_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'act_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/advanced_activations.py b/keras-tests/tfkeras/advanced_activations.py new file mode 100644 index 0000000..f4c71b6 --- /dev/null +++ b/keras-tests/tfkeras/advanced_activations.py @@ -0,0 +1,29 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm +import inspect + + +layer = [] +for x in dir(tf.keras.layers): + l = getattr(tf.keras.layers, x) + if inspect.isclass(l) and \ + issubclass(l, tf.keras.layers.Layer) and \ + '.layers.advanced_activations' in l.__module__: + layer.append(l) + +@grid(**globals()) +def generate_models(layer): + inp = tf.keras.layers.Input((2, 3)) + out = layer()(inp) + return tf.keras.models.Model(inp, out) + +def run(): + gen = generate_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'adv_act_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/cnn.py b/keras-tests/tfkeras/cnn.py new file mode 100644 index 0000000..1bf37ff --- /dev/null +++ b/keras-tests/tfkeras/cnn.py @@ -0,0 +1,310 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm +import warnings +from tensorflow.python.framework.ops import disable_eager_execution +import multiprocessing + + +disable_eager_execution() + +np.random.seed(1337) + +max_num_layers = 2 + +#1D +kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) +input_shape = [(20, 32)]#, (None, 16), (3, None), (1, None)] +data_format = ['channels_first', 'channels_last'] +layer_type = [tf.keras.layers.Conv1D]#, tf.keras.layers.LocallyConnected1D]#, tf.keras.layers.SeparableConv1D] +num_layers = [1, 2]#[2]#[i + 1 for i in range(max_num_layers)] +filters = [4] +strides = [1, 2, 3] +dilation_rate = [1, 2] +activation = ['tanh'] +use_bias = [True]#, False] +pooling = [None]#tf.keras.layers.MaxPooling1D, tf.keras.layers.AveragePooling1D] +global_pooling = [None]#tf.keras.layers.GlobalMaxPool1D, tf.keras.layers.GlobalAvgPool1D] +padding = ['causal', 'same', 'valid'] +zero_padding = [None]#, 3] +upsampling = [None] #2] +spatial_dropout = [False]#, True] +cropping = [None]#, (1, 1)] +compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] +@grid(**globals()) +def generate_cnn1ds(input_shape, + data_format, + layer_type, + num_layers, + filters, + strides, + padding, + zero_padding, + dilation_rate, + activation, + use_bias, + pooling, + global_pooling, + spatial_dropout, + upsampling, + cropping, + compile_args): + if strides != 1 and dilation_rate != 1: + return + if data_format == 'channels_last': + input_shape = input_shape[1:] + (input_shape[0],) + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + try: + args = dict(filters=filters, + kernel_size=(kernel_sizes[i],), + strides=strides, + padding=padding, + data_format=data_format, + use_bias=use_bias, + activation=activation) + if layer_type == tf.keras.layers.Conv1D: + args['dilation_rate'] = dilation_rate + elif dilation_rate != 1: + return + if layer_type == tf.keras.layers.LocallyConnected1D and \ + padding != 'valid': + return + x = layer_type(**args)(x) + if spatial_dropout: + x = tf.keras.layers.SpatialDropout1D(0.2)(x) + if zero_padding: + x = tf.keras.layers.ZeroPadding1D(zero_padding)(x) + if upsampling: + x = tf.keras.layers.UpSampling1D(upsampling)(x) + if cropping: + x = tf.keras.layers.Cropping1D(cropping)(x) + except ValueError as e: + print(e) + return None # not all stride / input shape comnbinations are valid + if pooling: + try: + x = pooling()(x) + except: + pass + if global_pooling: + x = global_pooling()(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + + +#2D +kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) +input_shape = [(3, 10, 10)]#, (None, 16), (3, None), (1, None)] +data_format = ['channels_first', 'channels_last'] +layer_type = [tf.keras.layers.Conv2D, + tf.keras.layers.SeparableConv2D, + #tf.keras.layers.LocallyConnected2D, + #tf.keras.layers.DepthwiseConv2D, + tf.keras.layers.Conv2DTranspose] +num_layers = [2]#[i + 1 for i in range(max_num_layers)] +filters = [4] +strides = [1]#, 2] +dilation_rate = [1]#, 2] +activation = ['tanh'] +use_bias = [True]#, False] +pooling = [None]#tf.keras.layers.MaxPooling2D, tf.keras.layers.AveragePooling2D] +global_pooling = [None]#tf.keras.layers.GlobalMaxPool2D, tf.keras.layers.GlobalAvgPool2D] +padding = ['causal', 'same', 'valid'] +zero_padding = [None]#, 3] +upsampling = [None]#, 2] +spatial_dropout = [False]#, True] +cropping = [None]#, (1, 1)] +compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] +@grid(**globals()) +def generate_cnn2ds(input_shape, + data_format, + layer_type, + num_layers, + filters, + strides, + padding, + zero_padding, + dilation_rate, + activation, + use_bias, + pooling, + global_pooling, + spatial_dropout, + upsampling, + cropping, + compile_args): + if strides != 1 and dilation_rate != 1: + return + if data_format == 'channels_last': + input_shape = input_shape[1:] + (input_shape[0],) + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + try: + args = dict( + kernel_size=(kernel_sizes[i], kernel_sizes[i]), + strides=strides, + padding=padding, + data_format=data_format, + use_bias=use_bias, + activation=activation) + if layer_type == tf.keras.layers.Conv2D: + args['dilation_rate'] = dilation_rate + elif dilation_rate != 1: + return + if layer_type != tf.keras.layers.DepthwiseConv2D: + args['filters'] = filters + if layer_type == tf.keras.layers.LocallyConnected2D and \ + padding != 'valid': + return + x = layer_type(**args)(x) + if spatial_dropout: + x = tf.keras.layers.SpatialDropout2D(0.2)(x) + if zero_padding: + x = tf.keras.layers.ZeroPadding2D(zero_padding)(x) + if upsampling: + x = tf.keras.layers.UpSampling2D(upsampling)(x) + if cropping: + x = tf.keras.layers.Cropping2D(cropping)(x) + except ValueError as e: + warnings.warn(str(e)) + return None # not all stride / input shape comnbinations are valid + if pooling: + try: + x = pooling()(x) + except: + pass + if global_pooling: + x = global_pooling()(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + +#3D +kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) +input_shape = [(3, 10, 10, 10)]#, (None, 16), (3, None), (1, None)] +data_format = ['channels_first', 'channels_last'] +layer_type = [tf.keras.layers.Conv3D]#, tf.keras.layers.Conv3DTranspose] +num_layers = [2]#[i + 1 for i in range(max_num_layers)] +filters = [4] +strides = [1]#, 2] +dilation_rate = [1]#, 2] +activation = ['tanh'] +use_bias = [True]#, False] +pooling = [tf.keras.layers.MaxPooling3D, tf.keras.layers.AveragePooling3D] +global_pooling = [None]#tf.keras.layers.GlobalMaxPool3D, tf.keras.layers.GlobalAvgPool3D] +padding = ['causal', 'same', 'valid'] +zero_padding = [None]#, 3] +upsampling = [None]#, 2] +spatial_dropout = [True]#, False] +cropping = [(1, 1, 1), None] +compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] +@grid(**globals()) +def generate_cnn3ds(input_shape, + data_format, + layer_type, + num_layers, + filters, + strides, + padding, + zero_padding, + dilation_rate, + activation, + use_bias, + pooling, + global_pooling, + spatial_dropout, + upsampling, + cropping, + compile_args): + if strides != 1 and dilation_rate != 1: + return + if data_format == 'channels_last': + input_shape = input_shape[1:] + (input_shape[0],) + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + try: + args = dict(filters=filters, + kernel_size=(kernel_sizes[i],) * 3, + strides=strides, + padding=padding, + data_format=data_format, + use_bias=use_bias, + activation=activation) + if layer_type == tf.keras.layers.Conv3D: + args['dilation_rate'] = dilation_rate + elif dilation_rate != 1: + return + x = layer_type(**args)(x) + if spatial_dropout: + x = tf.keras.layers.SpatialDropout3D(0.2)(x) + if zero_padding: + x = tf.keras.layers.ZeroPadding3D(zero_padding)(x) + if upsampling: + x = tf.keras.layers.UpSampling3D(upsampling)(x) + if cropping: + x = tf.keras.layers.Cropping3D(cropping)(x) + except ValueError as e: + warnings.warn(str(e)) + return None # not all stride / input shape comnbinations are valid + if pooling: + try: + x = pooling()(x) + except: + pass + if global_pooling: + x = global_pooling()(x) + x = tf.keras.layers.Flatten()(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + + + +def _start_proc(f): + p = multiprocessing.Process(target=f) + p.start() + p.join() + +def _run_1d(): + gen = generate_cnn1ds() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'cnn1d_' + str(i) + '.h5') + del model + gc.collect() + +def _run_2d(): + gen = generate_cnn2ds() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'cnn2d_' + str(i) + '.h5') + del model + gc.collect() + +def _run_3d(): + gen = generate_cnn3ds() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'cnn3d_' + str(i) + '.h5') + del model + gc.collect() + +def run(): + _start_proc(_run_1d) + _start_proc(_run_2d) + _start_proc(_run_3d) + +if __name__ == '__main__': + run() + + diff --git a/keras-tests/tfkeras/convlstm.py b/keras-tests/tfkeras/convlstm.py new file mode 100644 index 0000000..331d0ad --- /dev/null +++ b/keras-tests/tfkeras/convlstm.py @@ -0,0 +1,69 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm +import warnings + + +np.random.seed(1337) + +max_num_layers = 2 + +kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) +input_shape = [(3, 4, 10, 10)]#... +data_format = ['channels_first', 'channels_last'] +num_layers = [2]#[i + 1 for i in range(max_num_layers)] +filters = [4] +strides = [1]#, 2] +dilation_rate = [1]#, 2] +activation = ['tanh'] +padding = ['same', 'valid'] +use_bias = [True]#, False] +return_sequences = [True, False] +go_backwards = [True, False] +bidirectional = [True, False] +@grid(**globals()) +def generate_convlstm_models(input_shape, + data_format, + num_layers, + filters, + strides, + dilation_rate, + activation, + padding, + use_bias, + return_sequences, + go_backwards, + bidirectional): + if not return_sequences and num_layers > 1: + return + if data_format == 'channels_last': + input_shape = input_shape[1:] + (input_shape[0],) + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + layer = tf.keras.layers.ConvLSTM2D(filters=filters, + kernel_size=(kernel_sizes[i],) * 2, + strides=strides, + dilation_rate=dilation_rate, + padding=padding, + data_format=data_format, + return_sequences=return_sequences, + go_backwards=go_backwards + ) + if bidirectional: + layer = tf.keras.layers.Bidirectional(layer) + x = layer(x) + return tf.keras.models.Model(inp, x) + +def run(): + gen = generate_convlstm_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'convlstm2d_' + str(i) + '.h5') + +if __name__ == '__main__': + run() + + diff --git a/keras-tests/tfkeras/coverage.py b/keras-tests/tfkeras/coverage.py new file mode 100644 index 0000000..82298b3 --- /dev/null +++ b/keras-tests/tfkeras/coverage.py @@ -0,0 +1,11 @@ +from utils import get_coverage + +coverage = get_coverage() + +print('Layers not covered: ') +for l in coverage['uncovered_layers']: + print(l) + +print('Layer coverage: ' + + str(int(100 * len(coverage['covered_layers']) / len(coverage['all_layers']))) + '% (' + + str(len(coverage['covered_layers'])) + '/' + str(len(coverage['all_layers'])) + ')') diff --git a/keras-tests/tfkeras/merge.py b/keras-tests/tfkeras/merge.py new file mode 100644 index 0000000..7d6c93e --- /dev/null +++ b/keras-tests/tfkeras/merge.py @@ -0,0 +1,66 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm + + +input_shapes = [[(2,), (2,)], [(None, 2), (None, 2)], [(None, 2, 3), (None, 2, 3)]] + +merger = [ + tf.keras.layers.Add, + tf.keras.layers.Multiply, + tf.keras.layers.Average, + tf.keras.layers.Subtract, + tf.keras.layers.Maximum, + tf.keras.layers.Minimum +] + + +@grid(**globals()) +def generate_merge_models(input_shapes, merger): + inputs = list(map(tf.keras.layers.Input, input_shapes)) + merged = merger()(inputs) + model = tf.keras.models.Model(inputs, merged) + +def dot_models(): + models = [] + args = [ + [(2, 3), (2, 2), (1, 1)], + [(2, 3), (3, 3), (2, 1)] + ] + for arg in args: + shapes = arg[:2] + inputs = list(map(tf.keras.layers.Input, shapes)) + out = tf.keras.layers.Dot(axes=arg[-1])(inputs) + models.append(tf.keras.models.Model(inputs, out)) + return models + +def concat_models(): + models = [] + args = [ + [(2, 3), (4, 3), 1], + [(2, 3), (2, 4), 2] + ] + for arg in args: + shapes = arg[:2] + inputs = list(map(tf.keras.layers.Input, shapes)) + out = tf.keras.layers.Concatenate(axis=arg[-1])(inputs) + models.append(tf.keras.models.Model(inputs, out)) + return models + + +def run(): + gen = generate_merge_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'merge_' + str(i) + '.h5') + for i, model in tqdm(enumerate(dot_models())): + if model: + save_model(model, 'merge_dot_' + str(i) + '.h5') + for i, model in tqdm(enumerate(concat_models())): + if model: + save_model(model, 'merge_concat_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/misc.py b/keras-tests/tfkeras/misc.py new file mode 100644 index 0000000..bfc5a90 --- /dev/null +++ b/keras-tests/tfkeras/misc.py @@ -0,0 +1,55 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +from utils import tqdm + + +def get_misc_models(): + models = [] + ## TODO + # inp = tf.keras.layers.Input((2, 3)) + # x = tf.keras.layers.Dense(4)(inp) + # x = tf.keras.layers.Flatten()(x) + # x = tf.keras.layers.Dense(1)(x) + # models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((5,)) + x = tf.keras.layers.Embedding(10, 8)(inp) + x = tf.keras.layers.Dense(4)(x) + models.append(tf.keras.models.Model(inp, x)) + + # TODO + # inp = tf.keras.layers.Input((5,)) + # x = tf.keras.layers.Embedding(10, 8, mask_zero=True)(inp) + # x = tf.keras.layers.Masking()(x) + # x = tf.keras.layers.Dense(4)(x) + # models.append(tf.keras.models.Model(inp, x)) + + # TODO + # inp = tf.keras.layers.Input((2, 3)) + # x = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(4))(inp) + # models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((2, 3)) + x = tf.keras.layers.Reshape((3, 2))(inp) + models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((2, 3)) + x = tf.keras.layers.Permute((2, 1))(inp) + models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((2,)) + x = tf.keras.layers.Dense(5)(inp) + x = tf.keras.layers.AlphaDropout(0.5)(x) + models.append(tf.keras.models.Model(inp, x)) + + + return models + +def run(): + gen = get_misc_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'misc_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/mlp.py b/keras-tests/tfkeras/mlp.py new file mode 100644 index 0000000..5fe29ad --- /dev/null +++ b/keras-tests/tfkeras/mlp.py @@ -0,0 +1,41 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +from utils import tqdm + +np.random.seed(1337) + +max_num_layers = 5 +layer_output_sizes = list(np.random.randint(3, 7, (max_num_layers,))) + +# grid +input_shape = [(3,), (4,)]#, (2, 3), (4, 5)] +num_layers = [i + 1 for i in range(max_num_layers)] +activation_type = ['layer', 'arg'] +activation = ['sigmoid', 'relu', 'tanh'] +compile_args = [None, {'loss': 'mse', 'optimizer': 'sgd'}] + +@grid(**globals()) +def generate_mlps(input_shape, num_layers, activation_type, activation, compile_args): + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + if activation_type == 'arg': + x = tf.keras.layers.Dense(layer_output_sizes[0], activation=activation)(x) + else: + x = tf.keras.layers.Dense(layer_output_sizes[0])(x) + x = tf.keras.layers.Activation(activation)(x) + x = tf.keras.layers.Dropout(0.2)(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + + +def run(): + gen = generate_mlps() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'mlp_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/rnn.py b/keras-tests/tfkeras/rnn.py new file mode 100644 index 0000000..901d4a3 --- /dev/null +++ b/keras-tests/tfkeras/rnn.py @@ -0,0 +1,65 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +from utils import tqdm + +np.random.seed(1337) + +max_num_layers = 2 +layer_output_sizes = list(np.random.randint(3, 7, (max_num_layers,))) + + +#grid +input_shape = [(3, 4), (1, 5), (4, 1), (None, 3), (None, 1)] +num_layers = [i + 1 for i in range(max_num_layers)] +activation_type = ['layer', 'arg', 'recurrent'] +activation = ['relu']#, 'tanh'] +rnn_type = [tf.keras.layers.SimpleRNN, tf.keras.layers.LSTM]#, tf.keras.layers.GRU] +return_sequences = [True, False] +dense_after = [False]#[True, False] +compile_args = [None]#, {'loss': 'mse', 'optimizer': 'sgd'}] +bidirectional = [None, 'concat']#, 'sum', 'mul'] + +@grid(**globals()) +def generate_rnns(input_shape, + num_layers, + activation_type, + activation, + rnn_type, + return_sequences, + dense_after, + bidirectional, + compile_args): + if rnn_type == tf.keras.layers.SimpleRNN and activation_type == 'recurrent': + return + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + if bidirectional: + f = lambda *args, **kwargs: tf.keras.layers.Bidirectional(rnn_type(*args, **kwargs), merge_mode=bidirectional) + else: + f = rnn_type + if activation_type == 'arg': + x = f(layer_output_sizes[i], return_sequences=return_sequences, activation=activation)(x) + elif activation_type == 'recurrent': + x = f(layer_output_sizes[i], return_sequences=return_sequences, recurrent_activation=activation)(x) + else: + x = f(layer_output_sizes[i], return_sequences=return_sequences)(x) + x = tf.keras.layers.Activation(activation)(x) + if dense_after: + x = tf.keras.layers.Dense(int(layer_output_sizes[i] / 2) + 1)(x) + if not return_sequences and i != num_layers - 1: + x = tf.keras.layers.RepeatVector(5)(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + +def run(): + gen = generate_rnns() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'rnn_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/run.py b/keras-tests/tfkeras/run.py new file mode 100644 index 0000000..94d2ae6 --- /dev/null +++ b/keras-tests/tfkeras/run.py @@ -0,0 +1,33 @@ +import tensorflow as tf +import mlp +import rnn +import cnn +import merge +import activations +import advanced_activations +import misc +import convlstm +import tfop +import utils +import os +import gc +import warnings +import json +from utils import tqdm +import sys +import subprocess + +jobs = [mlp, rnn, cnn, merge, activations, advanced_activations, misc, convlstm, tfop] + +def run_sequential(): + for job in jobs: + job.run() + +def run_sequential_subprocess(): + for job in jobs: + subprocess.Popen(f'"{sys.executable}" "{job.__file__}"', stdout=subprocess.PIPE).stdout.readlines() + +#run_sequential() +run_sequential_subprocess() + +import coverage diff --git a/keras-tests/tfkeras/tfop.py b/keras-tests/tfkeras/tfop.py new file mode 100644 index 0000000..df05f2a --- /dev/null +++ b/keras-tests/tfkeras/tfop.py @@ -0,0 +1,28 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +from utils import tqdm + + +def get_tfop_models(): + models = [] + + inp = tf.keras.layers.Input((2, 3)) + reshaped = tf.reshape(inp, (-1, 6)) + out = tf.keras.layers.Dense(5)(reshaped) + models.append(tf.keras.models.Model(inp, out)) + + inp = tf.keras.layers.Input((3, 2)) + perm = tf.transpose(inp, (0, 2, 1)) + out = tf.keras.layers.Dense(5)(perm) + models.append(tf.keras.models.Model(inp, out)) + + return models + +def run(): + gen = get_tfop_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'tfop_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py new file mode 100644 index 0000000..c340a04 --- /dev/null +++ b/keras-tests/tfkeras/utils.py @@ -0,0 +1,175 @@ +import os +import itertools +import inspect +import sys +import tensorflow as tf +import json +import os +import h5py +import numpy as np +import click +try: + from tqdm import tqdm +except ImportError: + import warnings + tqdm = lambda _: [_,warnings.warn("Install tqdm for fancy progress bars!")][0] + + +dl4j_test_resources = os.environ.get('DL4J_TEST_RESOURCES') + + +if dl4j_test_resources is None: + # Try and auto detect dl4j_test_resources. + # We are assuming that dl4j-test-resources and dl4j-dev-tools + # are cloned in parallel (i.e they are in the same parent directory) + # NOTE: code should be adapted in case this file is refactored + dirname = os.path.dirname + dl4j_test_resources = os.path.join( + dirname(dirname(dirname(dirname(os.path.abspath(__file__))))), + 'dl4j-test-resources') + if os.path.isdir(dl4j_test_resources): + print('Aoto detected dl4j-test-resources: ' + dl4j_test_resources) + else: + raise Exception('Environment variable not set: DL4J_TEST_RESOURCES') + +tfkeras_dir = os.path.join(dl4j_test_resources, + 'src', 'main', 'resources', + 'modelimport', 'keras', 'tfkeras') + + + +def _get_layers(model): + if hasattr(model, 'layers'): + layers = [] + for layer in model.layers: + layers += _get_layers(layer) + return layers + else: + return [model.__class__.__name__] + +used_layers_file = 'used_layers.json' +if os.path.isfile(used_layers_file): + with open(used_layers_file, 'r') as f: + _used_layers = set(json.load(f)) +else: + _used_layers = set() + +def _update_used_layers(model): + try: + n = len(_used_layers) + layers = _get_layers(model) + _used_layers.update(layers) + if len(_used_layers) > n: + with open(used_layers_file, 'w') as f: + json.dump(list(_used_layers), f) + except Exception as e: + print('Error recording used layers: ' + str(e)) + + +def rand(shape): + if isinstance(shape, list): + return list(map(rand, shape)) + shape = [d if d is not None else 1 for d in shape] + return np.random.random(shape) + +def put_data(model, input_arrays, output_arrays, h5file): + input_names = [i.encode('utf8') for i in model.input_names] + output_names = [o.encode('utf8') for o in model.output_names] + f = h5py.File(h5file) + data = f.create_group('data') + data.attrs['input_names'] = input_names + data.attrs['output_names'] = output_names + for name, arr in zip(input_names, input_arrays): + data.create_dataset(name, shape=arr.shape, dtype='f', data=arr) + for name, arr in zip(output_names, output_arrays): + data.create_dataset(name, shape=arr.shape, dtype='f', data=arr) + f.close() + +def put_rand_data(model, h5file): + input_shapes = [tuple(i.shape) for i in model.inputs] + input_arrays = list(map(rand, input_shapes)) + output_arrays = model.predict(input_arrays) + if not isinstance(output_arrays, list): + output_arrays = [output_arrays] + return put_data(model, input_arrays, output_arrays, h5file) + +def _to_sequential(model): + seq = tf.keras.models.Sequential() + for layer in model.layers[1:]: + seq.add(layer) + return seq + + +def save_model(model, file_name, data='rand', as_sequential=False): + if as_sequential: + model = _to_sequential(model) + print(model.layers[0].input_shape) + model.predict(rand(model.layers[0].input_shape)) + path = os.path.join(tfkeras_dir, file_name) + os.makedirs(os.path.dirname(path), exist_ok=True) + model.predict(rand(model.input_shape)) + model.save(path) + put_rand_data(model, path) + _update_used_layers(model) + + +def grid(*args, **kwargs): + if args and callable(args[0]): + return grid()(args[0]) + def inner(f): + def grid_call(): + class ArgIterator(): + def __init__(self, args, kwargs): + args = list(args) + fargspec = inspect.getfullargspec(f) + if fargspec.varargs or fargspec.kwonlyargs: + raise Exception("*args and and **kwargs not supported for grid call.") + fargs = fargspec.args + fdefs = fargspec.defaults + if fdefs is None: + fdefs = () + for i, arg in enumerate(fargs[len(args):]): + if arg in kwargs: + args.append(kwargs[arg]) + elif i >= len(fargs) -len(args) - len(fdefs): + args.append([fdefs[i - (len(fargs) -len(args) - len(fdefs))]]) + else: + raise Exception('Unable to resolve value for argument ' + arg) + self.args = itertools.product(*args) + n = 1 + for a in args: + n *= len(a) + self.n = n + def __len__(self): return self.n + def __iter__(self): return self + def __next__(self): return self.next() + def __next__(self): return f(*next(self.args)) + return ArgIterator(args, kwargs) + return grid_call + return inner + + +def get_coverage(): + ret = {} + # calculate layer coverage TODO: op coverage + all_layers = [] + all_layers_set = set() + for l in dir(tf.keras.layers): + layer = getattr(tf.keras.layers, l) + if inspect.isclass(layer): + l = layer.__name__ + if issubclass(layer, tf.keras.layers.Layer) \ + and l[0].isupper() and not l.startswith('Abstract') and \ + not l.endswith('Cell') and l not in ('RNN', 'Layer', 'Wrapper'): + if layer not in all_layers_set: + all_layers_set.add(layer) + all_layers.append(l) + ret['all_layers'] = all_layers + with open(used_layers_file, 'r') as f: + used_layers = json.load(f) + unused_layers = [l for l in all_layers if l not in used_layers] + coverage = len(used_layers) / len(all_layers) + ret['covered_layers'] = used_layers + ret['uncovered_layers'] = unused_layers + ret['coverage'] = coverage + return ret \ No newline at end of file