From f8b30c1448412abbf15f63a76b63a5df07d4b0cc Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Fri, 27 Mar 2020 00:17:08 +0400 Subject: [PATCH 01/11] tfkeras test gen --- codegen/adr/0009-aliasing.md | 2 +- .../{ => legacy}/make_activation_layers.py | 0 .../{ => legacy}/make_causal_conv1d.py | 0 keras-tests/{ => legacy}/make_conv1d.py | 0 .../make_imdb_fasttext_unit_test.py | 0 .../{ => legacy}/make_imdb_lstm_unit_test.py | 0 .../{ => legacy}/make_mnist_cnn_unit_test.py | 0 .../make_mnist_mlp_constraint_unit_test.py | 0 .../{ => legacy}/make_mnist_mlp_unit_test.py | 0 keras-tests/{ => legacy}/make_simple_lstm.py | 0 .../{ => legacy}/make_sparsexent_mlp.py | 0 keras-tests/{ => legacy}/util.py | 0 keras-tests/tfkeras/mlp.py | 37 +++++++++++++ keras-tests/tfkeras/rnn.py | 55 +++++++++++++++++++ keras-tests/tfkeras/run.py | 5 ++ keras-tests/tfkeras/utils.py | 49 +++++++++++++++++ 16 files changed, 147 insertions(+), 1 deletion(-) rename keras-tests/{ => legacy}/make_activation_layers.py (100%) rename keras-tests/{ => legacy}/make_causal_conv1d.py (100%) rename keras-tests/{ => legacy}/make_conv1d.py (100%) rename keras-tests/{ => legacy}/make_imdb_fasttext_unit_test.py (100%) rename keras-tests/{ => legacy}/make_imdb_lstm_unit_test.py (100%) rename keras-tests/{ => legacy}/make_mnist_cnn_unit_test.py (100%) rename keras-tests/{ => legacy}/make_mnist_mlp_constraint_unit_test.py (100%) rename keras-tests/{ => legacy}/make_mnist_mlp_unit_test.py (100%) rename keras-tests/{ => legacy}/make_simple_lstm.py (100%) rename keras-tests/{ => legacy}/make_sparsexent_mlp.py (100%) rename keras-tests/{ => legacy}/util.py (100%) create mode 100644 keras-tests/tfkeras/mlp.py create mode 100644 keras-tests/tfkeras/rnn.py create mode 100644 keras-tests/tfkeras/run.py create mode 100644 keras-tests/tfkeras/utils.py diff --git a/codegen/adr/0009-aliasing.md b/codegen/adr/0009-aliasing.md index e57a542..5fc6777 100644 --- a/codegen/adr/0009-aliasing.md +++ b/codegen/adr/0009-aliasing.md @@ -40,7 +40,7 @@ fun BaseOps() = Namespace("BaseOps"){ } fun Linalg() = Namespace("Linalg"){ - Alias(BaseOps().op("mmul")) + Alias(BaseOps(), "mmul") } ``` diff --git a/keras-tests/make_activation_layers.py b/keras-tests/legacy/make_activation_layers.py similarity index 100% rename from keras-tests/make_activation_layers.py rename to keras-tests/legacy/make_activation_layers.py diff --git a/keras-tests/make_causal_conv1d.py b/keras-tests/legacy/make_causal_conv1d.py similarity index 100% rename from keras-tests/make_causal_conv1d.py rename to keras-tests/legacy/make_causal_conv1d.py diff --git a/keras-tests/make_conv1d.py b/keras-tests/legacy/make_conv1d.py similarity index 100% rename from keras-tests/make_conv1d.py rename to keras-tests/legacy/make_conv1d.py diff --git a/keras-tests/make_imdb_fasttext_unit_test.py b/keras-tests/legacy/make_imdb_fasttext_unit_test.py similarity index 100% rename from keras-tests/make_imdb_fasttext_unit_test.py rename to keras-tests/legacy/make_imdb_fasttext_unit_test.py diff --git a/keras-tests/make_imdb_lstm_unit_test.py b/keras-tests/legacy/make_imdb_lstm_unit_test.py similarity index 100% rename from keras-tests/make_imdb_lstm_unit_test.py rename to keras-tests/legacy/make_imdb_lstm_unit_test.py diff --git a/keras-tests/make_mnist_cnn_unit_test.py b/keras-tests/legacy/make_mnist_cnn_unit_test.py similarity index 100% rename from keras-tests/make_mnist_cnn_unit_test.py rename to keras-tests/legacy/make_mnist_cnn_unit_test.py diff --git a/keras-tests/make_mnist_mlp_constraint_unit_test.py b/keras-tests/legacy/make_mnist_mlp_constraint_unit_test.py similarity index 100% rename from keras-tests/make_mnist_mlp_constraint_unit_test.py rename to keras-tests/legacy/make_mnist_mlp_constraint_unit_test.py diff --git a/keras-tests/make_mnist_mlp_unit_test.py b/keras-tests/legacy/make_mnist_mlp_unit_test.py similarity index 100% rename from keras-tests/make_mnist_mlp_unit_test.py rename to keras-tests/legacy/make_mnist_mlp_unit_test.py diff --git a/keras-tests/make_simple_lstm.py b/keras-tests/legacy/make_simple_lstm.py similarity index 100% rename from keras-tests/make_simple_lstm.py rename to keras-tests/legacy/make_simple_lstm.py diff --git a/keras-tests/make_sparsexent_mlp.py b/keras-tests/legacy/make_sparsexent_mlp.py similarity index 100% rename from keras-tests/make_sparsexent_mlp.py rename to keras-tests/legacy/make_sparsexent_mlp.py diff --git a/keras-tests/util.py b/keras-tests/legacy/util.py similarity index 100% rename from keras-tests/util.py rename to keras-tests/legacy/util.py diff --git a/keras-tests/tfkeras/mlp.py b/keras-tests/tfkeras/mlp.py new file mode 100644 index 0000000..f5e0c8b --- /dev/null +++ b/keras-tests/tfkeras/mlp.py @@ -0,0 +1,37 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model + +np.random.seed(1337) + +max_num_layers = 5 +layer_output_sizes = list(np.random.randint(3, 7, (max_num_layers,))) + +# grid +input_shape = [(3,), (4,), (2, 3), (4, 5)] +num_layers = list(range(max_num_layers)) +activation_type = ['layer', 'arg'] +activation = ['sigmoid', 'relu', 'tanh'] +compile_args = [None, {'loss': 'mse', 'optimizer': 'sgd'}] + +@grid(**globals()) +def generate_mlps(input_shape, num_layers, activation_type, activation, compile_args): + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + if activation_type == 'arg': + x = tf.keras.layers.Dense(layer_output_sizes[0], activation=activation)(x) + else: + x = tf.keras.layers.Dense(layer_output_sizes[0])(x) + x = tf.keras.layers.Activation(activation)(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + +def run(): + for i, model in enumerate(generate_mlps()): + save_model(model, 'mlp_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/rnn.py b/keras-tests/tfkeras/rnn.py new file mode 100644 index 0000000..33bf16a --- /dev/null +++ b/keras-tests/tfkeras/rnn.py @@ -0,0 +1,55 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model + +np.random.seed(1337) + +max_num_layers = 2 +layer_output_sizes = list(np.random.randint(3, 7, (max_num_layers,))) + + +#grid +input_shape = [(3, 4), (1, 5), (4, 1), (None, 3), (None, 1)] +num_layers = list(range(max_num_layers)) +activation_type = ['layer', 'arg', 'recurrent'] +activation = ['relu', 'tanh'] +rnn_type = [tf.keras.layers.LSTM, tf.keras.layers.GRU] +return_sequences = [True, False] +dense_after = [False]#[True, False] +compile_args = [None]#, {'loss': 'mse', 'optimizer': 'sgd'}] + + +@grid(**globals()) +def generate_rnns(input_shape, + num_layers, + activation_type, + activation, + rnn_type, + return_sequences, + dense_after, + compile_args): + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + if activation_type == 'arg': + x = rnn_type(layer_output_sizes[i], return_sequences=return_sequences, activation=activation)(x) + elif activation_type == 'recurrent': + x = rnn_type(layer_output_sizes[i], return_sequences=return_sequences, recurrent_activation=activation)(x) + else: + x = rnn_type(layer_output_sizes[i], return_sequences=return_sequences)(x) + x = tf.keras.layers.Activation(activation)(x) + if dense_after: + x = tf.keras.layers.Dense(int(layer_output_sizes[i] / 2) + 1)(x) + if not return_sequences and i != num_layers - 1: + x = tf.keras.layers.RepeatVector(5)(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + +def run(): + for i, model in enumerate(generate_rnns()): + save_model(model, 'rnn_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/run.py b/keras-tests/tfkeras/run.py new file mode 100644 index 0000000..f0911cd --- /dev/null +++ b/keras-tests/tfkeras/run.py @@ -0,0 +1,5 @@ +import mlp +import rnn + +mlp.run() +rnn.run() diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py new file mode 100644 index 0000000..c3f1f05 --- /dev/null +++ b/keras-tests/tfkeras/utils.py @@ -0,0 +1,49 @@ +import os +import itertools +import inspect + +dl4j_test_resources = os.environ.get('DL4J_TEST_RESOURCES') + +if dl4j_test_resources is None: + raise Exception('Environment variable not set: DL4J_TEST_RESOURCES') + +tfkeras_dir = os.path.join(dl4j_test_resources, + 'src', 'main', 'resources', + 'modelimport', 'keras', 'tfkeras') + + +def save_model(model, file_name): + path = os.path.join(tfkeras_dir, file_name) + os.makedirs(os.path.dirname(path), exist_ok=True) + model.save(path) + + +def grid(*args, **kwargs): + if args and callable(args[0]): + return grid()(args[0]) + def inner(f): + def grid_call(): + class ArgIterator(): + def __init__(self, args, kwargs): + args = list(args) + fargspec = inspect.getfullargspec(f) + if fargspec.varargs or fargspec.kwonlyargs: + raise Exception("*args and and **kwargs not supported for grid call.") + fargs = fargspec.args + fdefs = fargspec.defaults + if fdefs is None: + fdefs = () + for i, arg in enumerate(fargs[len(args):]): + if arg in kwargs: + args.append(kwargs[arg]) + elif i >= len(fargs) -len(args) - len(fdefs): + args.append([fdefs[i - (len(fargs) -len(args) - len(fdefs))]]) + else: + raise Exception('Unable to resolve value for argument ' + arg) + self.args = itertools.product(*args) + def __iter__(self): return self + def __next__(self): return self.next() + def __next__(self): return f(*next(self.args)) + return ArgIterator(args, kwargs) + return grid_call + return inner From f6d6e368afedb90f13ac56f2c3abe808018f8082 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Fri, 27 Mar 2020 00:40:13 +0400 Subject: [PATCH 02/11] rem bad file --- codegen/adr/0009-aliasing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/adr/0009-aliasing.md b/codegen/adr/0009-aliasing.md index 5fc6777..e57a542 100644 --- a/codegen/adr/0009-aliasing.md +++ b/codegen/adr/0009-aliasing.md @@ -40,7 +40,7 @@ fun BaseOps() = Namespace("BaseOps"){ } fun Linalg() = Namespace("Linalg"){ - Alias(BaseOps(), "mmul") + Alias(BaseOps().op("mmul")) } ``` From b6485611b92670bc4f60da271696291190873fc7 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Fri, 27 Mar 2020 06:07:51 +0400 Subject: [PATCH 03/11] autodetect dl4j-test-res repo --- keras-tests/tfkeras/utils.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py index c3f1f05..09eafb1 100644 --- a/keras-tests/tfkeras/utils.py +++ b/keras-tests/tfkeras/utils.py @@ -1,11 +1,24 @@ import os import itertools import inspect +import sys + dl4j_test_resources = os.environ.get('DL4J_TEST_RESOURCES') if dl4j_test_resources is None: - raise Exception('Environment variable not set: DL4J_TEST_RESOURCES') + # Try and auto detect dl4j_test_resources. + # We are assuming that dl4j-test-resources and dl4j-dev-tools + # are cloned in parallel (i.e they are in the same parent directory) + # NOTE: code should be adapted in case this file is refactored + dirname = os.path.dirname + dl4j_test_resources = os.path.join( + dirname(dirname(dirname(dirname(os.path.abspath(__file__))))), + 'dl4j-test-resources') + if os.path.isdir(dl4j_test_resources): + print('Aoto detected dl4j-test-resources: ' + dl4j_test_resources) + else: + raise Exception('Environment variable not set: DL4J_TEST_RESOURCES') tfkeras_dir = os.path.join(dl4j_test_resources, 'src', 'main', 'resources', From 8495c64e37fd4186e58282c0ecb15b6fc4953280 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Tue, 31 Mar 2020 09:58:54 +0400 Subject: [PATCH 04/11] layer coverage --- keras-tests/tfkeras/run.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/keras-tests/tfkeras/run.py b/keras-tests/tfkeras/run.py index f0911cd..709d74f 100644 --- a/keras-tests/tfkeras/run.py +++ b/keras-tests/tfkeras/run.py @@ -1,5 +1,41 @@ +import tensorflow as tf import mlp import rnn +import utils +import os + mlp.run() rnn.run() + + + +# calculate layer coverage TODO: op coverage + +all_layers = [l for l in dir(tf.keras.layers) if l[0].isupper()] + +def get_layers(model): + if hasattr(model, 'layers'): + layers = [] + for layer in model.layers: + layers += get_layers(layer) + return layers + else: + return [model.__class__.__name__] + +used_layers = set() +for r, _, fs in os.walk(utils.tfkeras_dir): + for f in fs: + if f.lower().endswith('.h5'): + model = tf.keras.models.load_model(os.path.join(r, f)) + used_layers.update(get_layers(model)) + + +unused_layers = [l for l in all_layers if l not in used_layers] +coverage = len(used_layers) / len(all_layers) + +print('Layers not covered:') +for l in unused_layers: + print(l) + +print('Layer coverage: ' + str(int(coverage * 100)) + '%') From f00a4329c2d227dc8f47ee16e8e1586c3a135bf3 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Mon, 6 Apr 2020 03:33:23 +0400 Subject: [PATCH 05/11] test coverage 41% --- .gitignore | 4 +- keras-tests/tfkeras/activations.py | 32 +++++++++ keras-tests/tfkeras/advanced_activations.py | 29 ++++++++ keras-tests/tfkeras/cnn.py | 75 +++++++++++++++++++++ keras-tests/tfkeras/coverage.py | 11 +++ keras-tests/tfkeras/merge.py | 48 +++++++++++++ keras-tests/tfkeras/misc.py | 46 +++++++++++++ keras-tests/tfkeras/mlp.py | 8 ++- keras-tests/tfkeras/rnn.py | 15 +++-- keras-tests/tfkeras/run.py | 62 +++++++---------- keras-tests/tfkeras/utils.py | 70 +++++++++++++++++++ 11 files changed, 356 insertions(+), 44 deletions(-) create mode 100644 keras-tests/tfkeras/activations.py create mode 100644 keras-tests/tfkeras/advanced_activations.py create mode 100644 keras-tests/tfkeras/cnn.py create mode 100644 keras-tests/tfkeras/coverage.py create mode 100644 keras-tests/tfkeras/merge.py create mode 100644 keras-tests/tfkeras/misc.py diff --git a/.gitignore b/.gitignore index 1b463cd..d1cd11a 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,6 @@ coverage.xml docs/_build/ # PyBuilder -target/ \ No newline at end of file +target/ + +keras-tests/tfkeras/used_layers.json \ No newline at end of file diff --git a/keras-tests/tfkeras/activations.py b/keras-tests/tfkeras/activations.py new file mode 100644 index 0000000..a5a9c8f --- /dev/null +++ b/keras-tests/tfkeras/activations.py @@ -0,0 +1,32 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm + +np.random.seed(1337) + + +activation = [act for act in dir(tf.keras.activations) + if not act.startswith('_') and act not in ('serialize', 'deserialize', 'get')] + +use_dense = [True, False] + +@grid(**globals()) +def generate_models(activation, use_dense): + inp = tf.keras.layers.Input((2, 3)) + if use_dense: + out = tf.keras.layers.Dense(4, activation=activation)(inp) + else: + out = tf.keras.layers.Activation(activation)(inp) + model = tf.keras.models.Model(inp, out) + return model + + +def run(): + gen = generate_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'act_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/advanced_activations.py b/keras-tests/tfkeras/advanced_activations.py new file mode 100644 index 0000000..f4c71b6 --- /dev/null +++ b/keras-tests/tfkeras/advanced_activations.py @@ -0,0 +1,29 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm +import inspect + + +layer = [] +for x in dir(tf.keras.layers): + l = getattr(tf.keras.layers, x) + if inspect.isclass(l) and \ + issubclass(l, tf.keras.layers.Layer) and \ + '.layers.advanced_activations' in l.__module__: + layer.append(l) + +@grid(**globals()) +def generate_models(layer): + inp = tf.keras.layers.Input((2, 3)) + out = layer()(inp) + return tf.keras.models.Model(inp, out) + +def run(): + gen = generate_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'adv_act_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/cnn.py b/keras-tests/tfkeras/cnn.py new file mode 100644 index 0000000..f9e0267 --- /dev/null +++ b/keras-tests/tfkeras/cnn.py @@ -0,0 +1,75 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm + +np.random.seed(1337) + +max_num_layers = 3 +kernel_sizes = list(np.random.randint(5, 10, (max_num_layers,))) + +input_shape = [(3, 16), (1, 16)]#, (None, 16), (3, None), (1, None)] +data_format = ['channels_first', 'channels_last'] +num_layers = [i + 1 for i in range(max_num_layers)] +filters = [4] +strides = [1, 2, 3] +dilation_rate = [1, 2] +activation = ['tanh'] +use_bias = [True, False] +pooling = [tf.keras.layers.MaxPooling1D, tf.keras.layers.AveragePooling1D] +padding = ['causal', 'same', 'valid'] +compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] +@grid(**globals()) +def generate_cnn1ds(input_shape, + data_format, + num_layers, + filters, + strides, + padding, + dilation_rate, + activation, + use_bias, + pooling, + compile_args): + if strides != 1 and dilation_rate != 1: + return + if data_format == 'channels_last': + input_shape = (input_shape[1], input_shape[0]) + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + try: + x = tf.keras.layers.Conv1D(filters=filters, + kernel_size=(kernel_sizes[i],), + strides=strides, + dilation_rate=dilation_rate, + padding=padding, + data_format=data_format, + use_bias=use_bias, + activation=activation)(x) + except ValueError: + return None # not all stride / input shape comnbinations are valid + if pooling: + try: + x = pooling()(x) + except: + pass + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + + +def run(): + gen = generate_cnn1ds() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'cnn1d_' + str(i) + '.h5') + del model + gc.collect() + +if __name__ == '__main__': + run() + + diff --git a/keras-tests/tfkeras/coverage.py b/keras-tests/tfkeras/coverage.py new file mode 100644 index 0000000..82298b3 --- /dev/null +++ b/keras-tests/tfkeras/coverage.py @@ -0,0 +1,11 @@ +from utils import get_coverage + +coverage = get_coverage() + +print('Layers not covered: ') +for l in coverage['uncovered_layers']: + print(l) + +print('Layer coverage: ' + + str(int(100 * len(coverage['covered_layers']) / len(coverage['all_layers']))) + '% (' + + str(len(coverage['covered_layers'])) + '/' + str(len(coverage['all_layers'])) + ')') diff --git a/keras-tests/tfkeras/merge.py b/keras-tests/tfkeras/merge.py new file mode 100644 index 0000000..b1e0ec5 --- /dev/null +++ b/keras-tests/tfkeras/merge.py @@ -0,0 +1,48 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm + + +input_shapes_and_axis = [[(2,), (2,), -1], [(None, 2), (None, 3), -1], +[(None, 2), (3, 4), -1], [(3, 2), (4, 2), -2], [(3, 2), (4, 2), -1], +[(2, 3), (3, 4), (1, 0)]] +merger = [ + tf.keras.layers.Dot, + tf.keras.layers.Concatenate, + tf.keras.layers.Add, + tf.keras.layers.Multiply, + tf.keras.layers.Average, + tf.keras.layers.Subtract, + tf.keras.layers.Maximum, + tf.keras.layers.Minimum +] + +@grid(**globals()) +def generate_merge_models(input_shapes_and_axis, merger): + input_shapes = input_shapes_and_axis[:-1] + axis = input_shapes_and_axis[-1] + inputs = list(map(tf.keras.layers.Input, input_shapes)) + if merger == tf.keras.layers.Concatenate: + merge_args = {'axis' : axis} + elif merger == tf.keras.layers.Dot: + merge_args = {'axes': axis} + else: + merge_args = {} + try: + merged = merger(**merge_args)(inputs) + except: + return + model = tf.keras.models.Model(inputs, merged) + return model + + +def run(): + gen = generate_merge_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'merge_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/misc.py b/keras-tests/tfkeras/misc.py new file mode 100644 index 0000000..91a12c7 --- /dev/null +++ b/keras-tests/tfkeras/misc.py @@ -0,0 +1,46 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +from utils import tqdm + + +def get_misc_models(): + models = [] + inp = tf.keras.layers.Input((2, 3)) + x = tf.keras.layers.Dense(4)(inp) + x = tf.keras.layers.Flatten()(x) + x = tf.keras.layers.Dense(1)(x) + models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((5,)) + x = tf.keras.layers.Embedding(10, 8)(inp) + x = tf.keras.layers.Dense(4)(x) + models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((5,)) + x = tf.keras.layers.Embedding(10, 8, mask_zero=True)(inp) + x = tf.keras.layers.Masking()(x) + x = tf.keras.layers.Dense(4)(x) + models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((2, 3)) + x = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(4))(inp) + models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((2, 3)) + x = tf.keras.layers.Reshape((3, 2))(inp) + models.append(tf.keras.models.Model(inp, x)) + + inp = tf.keras.layers.Input((2, 3)) + x = tf.keras.layers.Permute((2, 1))(inp) + models.append(tf.keras.models.Model(inp, x)) + + return models + +def run(): + gen = get_misc_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'misc_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/mlp.py b/keras-tests/tfkeras/mlp.py index f5e0c8b..671a861 100644 --- a/keras-tests/tfkeras/mlp.py +++ b/keras-tests/tfkeras/mlp.py @@ -1,6 +1,7 @@ import tensorflow as tf import numpy as np from utils import grid, save_model +from utils import tqdm np.random.seed(1337) @@ -9,7 +10,7 @@ # grid input_shape = [(3,), (4,), (2, 3), (4, 5)] -num_layers = list(range(max_num_layers)) +num_layers = [i + 1 for i in range(max_num_layers)] activation_type = ['layer', 'arg'] activation = ['sigmoid', 'relu', 'tanh'] compile_args = [None, {'loss': 'mse', 'optimizer': 'sgd'}] @@ -24,13 +25,16 @@ def generate_mlps(input_shape, num_layers, activation_type, activation, compile_ else: x = tf.keras.layers.Dense(layer_output_sizes[0])(x) x = tf.keras.layers.Activation(activation)(x) + x = tf.keras.layers.Dropout(0.2)(x) model = tf.keras.models.Model(inp, x) if compile_args: model.compile(**compile_args) return model + def run(): - for i, model in enumerate(generate_mlps()): + gen = generate_mlps() + for i, model in tqdm(enumerate(gen), total=len(gen)): save_model(model, 'mlp_' + str(i) + '.h5') if __name__ == '__main__': diff --git a/keras-tests/tfkeras/rnn.py b/keras-tests/tfkeras/rnn.py index 33bf16a..393cc49 100644 --- a/keras-tests/tfkeras/rnn.py +++ b/keras-tests/tfkeras/rnn.py @@ -1,6 +1,7 @@ import tensorflow as tf import numpy as np from utils import grid, save_model +from utils import tqdm np.random.seed(1337) @@ -10,10 +11,10 @@ #grid input_shape = [(3, 4), (1, 5), (4, 1), (None, 3), (None, 1)] -num_layers = list(range(max_num_layers)) +num_layers = [i + 1 for i in range(max_num_layers)] activation_type = ['layer', 'arg', 'recurrent'] -activation = ['relu', 'tanh'] -rnn_type = [tf.keras.layers.LSTM, tf.keras.layers.GRU] +activation = ['relu']#, 'tanh'] +rnn_type = [tf.keras.layers.LSTM, tf.keras.layers.GRU, tf.keras.layers.SimpleRNN] return_sequences = [True, False] dense_after = [False]#[True, False] compile_args = [None]#, {'loss': 'mse', 'optimizer': 'sgd'}] @@ -28,6 +29,8 @@ def generate_rnns(input_shape, return_sequences, dense_after, compile_args): + if rnn_type == tf.keras.layers.SimpleRNN and activation_type == 'recurrent': + return inp = tf.keras.layers.Input(input_shape) x = inp for i in range(num_layers): @@ -48,8 +51,10 @@ def generate_rnns(input_shape, return model def run(): - for i, model in enumerate(generate_rnns()): - save_model(model, 'rnn_' + str(i) + '.h5') + gen = generate_rnns() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'rnn_' + str(i) + '.h5') if __name__ == '__main__': run() diff --git a/keras-tests/tfkeras/run.py b/keras-tests/tfkeras/run.py index 709d74f..512951f 100644 --- a/keras-tests/tfkeras/run.py +++ b/keras-tests/tfkeras/run.py @@ -1,41 +1,31 @@ import tensorflow as tf import mlp import rnn +import cnn +import merge +import activations +import advanced_activations import utils import os - - -mlp.run() -rnn.run() - - - -# calculate layer coverage TODO: op coverage - -all_layers = [l for l in dir(tf.keras.layers) if l[0].isupper()] - -def get_layers(model): - if hasattr(model, 'layers'): - layers = [] - for layer in model.layers: - layers += get_layers(layer) - return layers - else: - return [model.__class__.__name__] - -used_layers = set() -for r, _, fs in os.walk(utils.tfkeras_dir): - for f in fs: - if f.lower().endswith('.h5'): - model = tf.keras.models.load_model(os.path.join(r, f)) - used_layers.update(get_layers(model)) - - -unused_layers = [l for l in all_layers if l not in used_layers] -coverage = len(used_layers) / len(all_layers) - -print('Layers not covered:') -for l in unused_layers: - print(l) - -print('Layer coverage: ' + str(int(coverage * 100)) + '%') +import gc +import warnings +import json +from utils import tqdm +import sys +import subprocess + +jobs = [mlp, rnn, cnn, merge, activations, advanced_activations] +jobs.remove(cnn) +jobs.remove(rnn) +def run_sequential(): + for job in jobs: + job.run() + +def run_sequential_subprocess(): + for job in jobs: + subprocess.Popen(f'"{sys.executable}" "{job.__file__}"', stdout=subprocess.PIPE).stdout.readlines() + +#run_sequential() +run_sequential_subprocess() + +import coverage diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py index 09eafb1..2b60ef0 100644 --- a/keras-tests/tfkeras/utils.py +++ b/keras-tests/tfkeras/utils.py @@ -2,10 +2,19 @@ import itertools import inspect import sys +import tensorflow as tf +import json +import os +try: + from tqdm import tqdm +except ImportError: + import warnings + tqdm = lambda _: [_,warnings.warn("Install tqdm for fancy progress bars!")][0] dl4j_test_resources = os.environ.get('DL4J_TEST_RESOURCES') + if dl4j_test_resources is None: # Try and auto detect dl4j_test_resources. # We are assuming that dl4j-test-resources and dl4j-dev-tools @@ -25,10 +34,35 @@ 'modelimport', 'keras', 'tfkeras') + +def _get_layers(model): + if hasattr(model, 'layers'): + layers = [] + for layer in model.layers: + layers += _get_layers(layer) + return layers + else: + return [model.__class__.__name__] + +used_layers_file = 'used_layers.json' +if os.path.isfile(used_layers_file): + with open(used_layers_file, 'r') as f: + _used_layers = set(json.load(f)) +else: + _used_layers = set() + +def _update_used_layers(model): + layers = _get_layers(model) + _used_layers.update(layers) + with open(used_layers_file, 'w') as f: + json.dump(list(_used_layers), f) + def save_model(model, file_name): path = os.path.join(tfkeras_dir, file_name) os.makedirs(os.path.dirname(path), exist_ok=True) model.save(path) + _update_used_layers(model) + def grid(*args, **kwargs): @@ -54,9 +88,45 @@ def __init__(self, args, kwargs): else: raise Exception('Unable to resolve value for argument ' + arg) self.args = itertools.product(*args) + n = 1 + for a in args: + n *= len(a) + self.n = n + def __len__(self): return self.n def __iter__(self): return self def __next__(self): return self.next() def __next__(self): return f(*next(self.args)) return ArgIterator(args, kwargs) return grid_call return inner + + +def get_coverage(): + ret = {} + # calculate layer coverage TODO: op coverage + all_layers = [l for l in dir(tf.keras.layers) + if inspect.isclass(getattr(tf.keras.layers, l)) and + issubclass(getattr(tf.keras.layers, l), tf.keras.layers.Layer) and + l[0].isupper() and + not l.startswith('Abstract') and + not l.endswith('Cell') + ] + all_layers = [] + all_layers_set = set() + for l in dir(tf.keras.layers): + layer = getattr(tf.keras.layers, l) + if inspect.isclass(layer) and issubclass(layer, tf.keras.layers.Layer) \ + and l[0].isupper() and not l.startswith('Abstract') and \ + not l.endswith('Cell') and l not in ('RNN', 'Layer', 'Wrapper'): + if layer not in all_layers_set: + all_layers_set.add(layer) + all_layers.append(l) + ret['all_layers'] = all_layers + with open(used_layers_file, 'r') as f: + used_layers = json.load(f) + unused_layers = [l for l in all_layers if l not in used_layers] + coverage = len(used_layers) / len(all_layers) + ret['covered_layers'] = used_layers + ret['uncovered_layers'] = unused_layers + ret['coverage'] = coverage + return ret \ No newline at end of file From 8fe35b160c8433d91c5fa2daffadbcc9b2fd8497 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Mon, 6 Apr 2020 06:10:37 +0400 Subject: [PATCH 06/11] 86% --- keras-tests/tfkeras/cnn.py | 267 ++++++++++++++++++++++++++++++-- keras-tests/tfkeras/convlstm.py | 69 +++++++++ keras-tests/tfkeras/misc.py | 6 + keras-tests/tfkeras/rnn.py | 13 +- keras-tests/tfkeras/utils.py | 27 ++-- 5 files changed, 347 insertions(+), 35 deletions(-) create mode 100644 keras-tests/tfkeras/convlstm.py diff --git a/keras-tests/tfkeras/cnn.py b/keras-tests/tfkeras/cnn.py index f9e0267..99da578 100644 --- a/keras-tests/tfkeras/cnn.py +++ b/keras-tests/tfkeras/cnn.py @@ -3,65 +3,279 @@ from utils import grid, save_model import gc from utils import tqdm +import warnings +from tensorflow.python.framework.ops import disable_eager_execution +import multiprocessing + + +disable_eager_execution() np.random.seed(1337) -max_num_layers = 3 -kernel_sizes = list(np.random.randint(5, 10, (max_num_layers,))) +max_num_layers = 2 -input_shape = [(3, 16), (1, 16)]#, (None, 16), (3, None), (1, None)] +#1D +kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) +input_shape = [(20, 32)]#, (None, 16), (3, None), (1, None)] data_format = ['channels_first', 'channels_last'] -num_layers = [i + 1 for i in range(max_num_layers)] +layer_type = [tf.keras.layers.Conv1D, tf.keras.layers.SeparableConv1D, tf.keras.layers.LocallyConnected1D] +num_layers = [2]#[i + 1 for i in range(max_num_layers)] filters = [4] strides = [1, 2, 3] dilation_rate = [1, 2] activation = ['tanh'] -use_bias = [True, False] +use_bias = [True]#, False] pooling = [tf.keras.layers.MaxPooling1D, tf.keras.layers.AveragePooling1D] +global_pooling = [tf.keras.layers.GlobalMaxPool1D, tf.keras.layers.GlobalAvgPool1D] padding = ['causal', 'same', 'valid'] +zero_padding = [3]#, None] +upsampling = [2]#, None] +spatial_dropout = [True]#, False] +cropping = [(1, 1)]#, None] compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] @grid(**globals()) def generate_cnn1ds(input_shape, data_format, + layer_type, num_layers, filters, strides, padding, + zero_padding, dilation_rate, activation, use_bias, pooling, + global_pooling, + spatial_dropout, + upsampling, + cropping, compile_args): if strides != 1 and dilation_rate != 1: return if data_format == 'channels_last': - input_shape = (input_shape[1], input_shape[0]) + input_shape = input_shape[1:] + (input_shape[0],) inp = tf.keras.layers.Input(input_shape) x = inp for i in range(num_layers): try: - x = tf.keras.layers.Conv1D(filters=filters, - kernel_size=(kernel_sizes[i],), - strides=strides, - dilation_rate=dilation_rate, - padding=padding, - data_format=data_format, - use_bias=use_bias, - activation=activation)(x) - except ValueError: + args = dict(filters=filters, + kernel_size=(kernel_sizes[i],), + strides=strides, + padding=padding, + data_format=data_format, + use_bias=use_bias, + activation=activation) + if layer_type == tf.keras.layers.Conv1D: + args['dilation_rate'] = dilation_rate + elif dilation_rate != 1: + return + if layer_type == tf.keras.layers.LocallyConnected1D and \ + padding != 'valid': + return + x = layer_type(**args)(x) + if spatial_dropout: + x = tf.keras.layers.SpatialDropout1D(0.2)(x) + if zero_padding: + x = tf.keras.layers.ZeroPadding1D(zero_padding)(x) + if upsampling: + x = tf.keras.layers.UpSampling1D(upsampling)(x) + if cropping: + x = tf.keras.layers.Cropping1D(cropping)(x) + except ValueError as e: + warnings.warn(str(e)) return None # not all stride / input shape comnbinations are valid if pooling: try: x = pooling()(x) except: pass + if global_pooling: + x = global_pooling()(x) model = tf.keras.models.Model(inp, x) if compile_args: model.compile(**compile_args) return model -def run(): +#2D +kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) +input_shape = [(3, 10, 10)]#, (None, 16), (3, None), (1, None)] +data_format = ['channels_first', 'channels_last'] +layer_type = [tf.keras.layers.Conv2D, + tf.keras.layers.SeparableConv2D, + tf.keras.layers.LocallyConnected2D, + tf.keras.layers.DepthwiseConv2D, + tf.keras.layers.Conv2DTranspose] +num_layers = [2]#[i + 1 for i in range(max_num_layers)] +filters = [4] +strides = [1]#, 2] +dilation_rate = [1]#, 2] +activation = ['tanh'] +use_bias = [True]#, False] +pooling = [tf.keras.layers.MaxPooling2D, tf.keras.layers.AveragePooling2D] +global_pooling = [tf.keras.layers.GlobalMaxPool2D, tf.keras.layers.GlobalAvgPool2D] +padding = ['causal', 'same', 'valid'] +zero_padding = [3]#, None] +upsampling = [2]#, None] +spatial_dropout = [True]#, False] +cropping = [(1, 1)]#, None] +compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] +@grid(**globals()) +def generate_cnn2ds(input_shape, + data_format, + layer_type, + num_layers, + filters, + strides, + padding, + zero_padding, + dilation_rate, + activation, + use_bias, + pooling, + global_pooling, + spatial_dropout, + upsampling, + cropping, + compile_args): + if strides != 1 and dilation_rate != 1: + return + if data_format == 'channels_last': + input_shape = input_shape[1:] + (input_shape[0],) + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + try: + args = dict( + kernel_size=(kernel_sizes[i], kernel_sizes[i]), + strides=strides, + padding=padding, + data_format=data_format, + use_bias=use_bias, + activation=activation) + if layer_type == tf.keras.layers.Conv2D: + args['dilation_rate'] = dilation_rate + elif dilation_rate != 1: + return + if layer_type != tf.keras.layers.DepthwiseConv2D: + args['filters'] = filters + if layer_type == tf.keras.layers.LocallyConnected2D and \ + padding != 'valid': + return + x = layer_type(**args)(x) + if spatial_dropout: + x = tf.keras.layers.SpatialDropout2D(0.2)(x) + if zero_padding: + x = tf.keras.layers.ZeroPadding2D(zero_padding)(x) + if upsampling: + x = tf.keras.layers.UpSampling2D(upsampling)(x) + if cropping: + x = tf.keras.layers.Cropping2D(cropping)(x) + except ValueError as e: + warnings.warn(str(e)) + return None # not all stride / input shape comnbinations are valid + if pooling: + try: + x = pooling()(x) + except: + pass + if global_pooling: + x = global_pooling()(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + +#3D +kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) +input_shape = [(3, 10, 10, 10)]#, (None, 16), (3, None), (1, None)] +data_format = ['channels_first', 'channels_last'] +layer_type = [tf.keras.layers.Conv3D, tf.keras.layers.Conv3DTranspose] +num_layers = [2]#[i + 1 for i in range(max_num_layers)] +filters = [4] +strides = [1]#, 2] +dilation_rate = [1]#, 2] +activation = ['tanh'] +use_bias = [True]#, False] +pooling = [tf.keras.layers.MaxPooling3D, tf.keras.layers.AveragePooling3D] +global_pooling = [tf.keras.layers.GlobalMaxPool3D, tf.keras.layers.GlobalAvgPool3D] +padding = ['causal', 'same', 'valid'] +zero_padding = [3]#, None] +upsampling = [2]#, None] +spatial_dropout = [True]#, False] +cropping = [(1, 1, 1)]#, None] +compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] +@grid(**globals()) +def generate_cnn3ds(input_shape, + data_format, + layer_type, + num_layers, + filters, + strides, + padding, + zero_padding, + dilation_rate, + activation, + use_bias, + pooling, + global_pooling, + spatial_dropout, + upsampling, + cropping, + compile_args): + if strides != 1 and dilation_rate != 1: + return + if data_format == 'channels_last': + input_shape = input_shape[1:] + (input_shape[0],) + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + try: + args = dict(filters=filters, + kernel_size=(kernel_sizes[i],) * 3, + strides=strides, + padding=padding, + data_format=data_format, + use_bias=use_bias, + activation=activation) + if layer_type == tf.keras.layers.Conv3D: + args['dilation_rate'] = dilation_rate + elif dilation_rate != 1: + return + x = layer_type(**args)(x) + if spatial_dropout: + x = tf.keras.layers.SpatialDropout3D(0.2)(x) + if zero_padding: + x = tf.keras.layers.ZeroPadding3D(zero_padding)(x) + if upsampling: + x = tf.keras.layers.UpSampling3D(upsampling)(x) + if cropping: + x = tf.keras.layers.Cropping3D(cropping)(x) + except ValueError as e: + warnings.warn(str(e)) + return None # not all stride / input shape comnbinations are valid + if pooling: + try: + x = pooling()(x) + except: + pass + if global_pooling: + x = global_pooling()(x) + model = tf.keras.models.Model(inp, x) + if compile_args: + model.compile(**compile_args) + return model + + + +def _start_proc(f): + p = multiprocessing.Process(target=f) + p.start() + p.join() + +def _run_1d(): + return gen = generate_cnn1ds() for i, model in tqdm(enumerate(gen), total=len(gen)): if model: @@ -69,6 +283,27 @@ def run(): del model gc.collect() +def _run_2d(): + gen = generate_cnn2ds() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'cnn2d_' + str(i) + '.h5') + del model + gc.collect() + +def _run_3d(): + gen = generate_cnn3ds() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'cnn3d_' + str(i) + '.h5') + del model + gc.collect() + +def run(): + #_start_proc(_run_1d) + _start_proc(_run_2d) + #_start_proc(_run_3d) + if __name__ == '__main__': run() diff --git a/keras-tests/tfkeras/convlstm.py b/keras-tests/tfkeras/convlstm.py new file mode 100644 index 0000000..331d0ad --- /dev/null +++ b/keras-tests/tfkeras/convlstm.py @@ -0,0 +1,69 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +import gc +from utils import tqdm +import warnings + + +np.random.seed(1337) + +max_num_layers = 2 + +kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) +input_shape = [(3, 4, 10, 10)]#... +data_format = ['channels_first', 'channels_last'] +num_layers = [2]#[i + 1 for i in range(max_num_layers)] +filters = [4] +strides = [1]#, 2] +dilation_rate = [1]#, 2] +activation = ['tanh'] +padding = ['same', 'valid'] +use_bias = [True]#, False] +return_sequences = [True, False] +go_backwards = [True, False] +bidirectional = [True, False] +@grid(**globals()) +def generate_convlstm_models(input_shape, + data_format, + num_layers, + filters, + strides, + dilation_rate, + activation, + padding, + use_bias, + return_sequences, + go_backwards, + bidirectional): + if not return_sequences and num_layers > 1: + return + if data_format == 'channels_last': + input_shape = input_shape[1:] + (input_shape[0],) + inp = tf.keras.layers.Input(input_shape) + x = inp + for i in range(num_layers): + layer = tf.keras.layers.ConvLSTM2D(filters=filters, + kernel_size=(kernel_sizes[i],) * 2, + strides=strides, + dilation_rate=dilation_rate, + padding=padding, + data_format=data_format, + return_sequences=return_sequences, + go_backwards=go_backwards + ) + if bidirectional: + layer = tf.keras.layers.Bidirectional(layer) + x = layer(x) + return tf.keras.models.Model(inp, x) + +def run(): + gen = generate_convlstm_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + if model: + save_model(model, 'convlstm2d_' + str(i) + '.h5') + +if __name__ == '__main__': + run() + + diff --git a/keras-tests/tfkeras/misc.py b/keras-tests/tfkeras/misc.py index 91a12c7..bd404ad 100644 --- a/keras-tests/tfkeras/misc.py +++ b/keras-tests/tfkeras/misc.py @@ -35,6 +35,12 @@ def get_misc_models(): x = tf.keras.layers.Permute((2, 1))(inp) models.append(tf.keras.models.Model(inp, x)) + inp = tf.keras.layers.Input((2, 3)) + x = tf.keras.layers.Dense(5)(inp) + x = tf.keras.layers.AlphaDropout(0.5)(x) + models.append(tf.keras.models.Model(inp, x)) + + return models def run(): diff --git a/keras-tests/tfkeras/rnn.py b/keras-tests/tfkeras/rnn.py index 393cc49..d9d948e 100644 --- a/keras-tests/tfkeras/rnn.py +++ b/keras-tests/tfkeras/rnn.py @@ -18,7 +18,7 @@ return_sequences = [True, False] dense_after = [False]#[True, False] compile_args = [None]#, {'loss': 'mse', 'optimizer': 'sgd'}] - +bidirectional = [None, 'concat']#, 'sum', 'mul'] @grid(**globals()) def generate_rnns(input_shape, @@ -28,18 +28,23 @@ def generate_rnns(input_shape, rnn_type, return_sequences, dense_after, + bidirectional, compile_args): if rnn_type == tf.keras.layers.SimpleRNN and activation_type == 'recurrent': return inp = tf.keras.layers.Input(input_shape) x = inp for i in range(num_layers): + if bidirectional: + f = lambda *args, **kwargs: tf.keras.layers.Bidirectional(rnn_type(*args, **kwargs), merge_mode=bidirectional) + else: + f = rnn_type if activation_type == 'arg': - x = rnn_type(layer_output_sizes[i], return_sequences=return_sequences, activation=activation)(x) + x = f(layer_output_sizes[i], return_sequences=return_sequences, activation=activation)(x) elif activation_type == 'recurrent': - x = rnn_type(layer_output_sizes[i], return_sequences=return_sequences, recurrent_activation=activation)(x) + x = f(layer_output_sizes[i], return_sequences=return_sequences, recurrent_activation=activation)(x) else: - x = rnn_type(layer_output_sizes[i], return_sequences=return_sequences)(x) + x = f(layer_output_sizes[i], return_sequences=return_sequences)(x) x = tf.keras.layers.Activation(activation)(x) if dense_after: x = tf.keras.layers.Dense(int(layer_output_sizes[i] / 2) + 1)(x) diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py index 2b60ef0..3c4f274 100644 --- a/keras-tests/tfkeras/utils.py +++ b/keras-tests/tfkeras/utils.py @@ -52,10 +52,12 @@ def _get_layers(model): _used_layers = set() def _update_used_layers(model): + n = len(_used_layers) layers = _get_layers(model) _used_layers.update(layers) - with open(used_layers_file, 'w') as f: - json.dump(list(_used_layers), f) + if len(_used_layers) > n: + with open(used_layers_file, 'w') as f: + json.dump(list(_used_layers), f) def save_model(model, file_name): path = os.path.join(tfkeras_dir, file_name) @@ -104,23 +106,18 @@ def __next__(self): return f(*next(self.args)) def get_coverage(): ret = {} # calculate layer coverage TODO: op coverage - all_layers = [l for l in dir(tf.keras.layers) - if inspect.isclass(getattr(tf.keras.layers, l)) and - issubclass(getattr(tf.keras.layers, l), tf.keras.layers.Layer) and - l[0].isupper() and - not l.startswith('Abstract') and - not l.endswith('Cell') - ] all_layers = [] all_layers_set = set() for l in dir(tf.keras.layers): layer = getattr(tf.keras.layers, l) - if inspect.isclass(layer) and issubclass(layer, tf.keras.layers.Layer) \ - and l[0].isupper() and not l.startswith('Abstract') and \ - not l.endswith('Cell') and l not in ('RNN', 'Layer', 'Wrapper'): - if layer not in all_layers_set: - all_layers_set.add(layer) - all_layers.append(l) + if inspect.isclass(layer): + l = layer.__name__ + if issubclass(layer, tf.keras.layers.Layer) \ + and l[0].isupper() and not l.startswith('Abstract') and \ + not l.endswith('Cell') and l not in ('RNN', 'Layer', 'Wrapper'): + if layer not in all_layers_set: + all_layers_set.add(layer) + all_layers.append(l) ret['all_layers'] = all_layers with open(used_layers_file, 'r') as f: used_layers = json.load(f) From 9e25f9759d75d5d708958cb41d02003938a99d26 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Sun, 12 Apr 2020 21:50:10 +0400 Subject: [PATCH 07/11] include arrays --- keras-tests/tfkeras/merge.py | 2 +- keras-tests/tfkeras/run.py | 3 +-- keras-tests/tfkeras/utils.py | 42 ++++++++++++++++++++++++++++++------ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/keras-tests/tfkeras/merge.py b/keras-tests/tfkeras/merge.py index b1e0ec5..56439d4 100644 --- a/keras-tests/tfkeras/merge.py +++ b/keras-tests/tfkeras/merge.py @@ -32,9 +32,9 @@ def generate_merge_models(input_shapes_and_axis, merger): merge_args = {} try: merged = merger(**merge_args)(inputs) + model = tf.keras.models.Model(inputs, merged) except: return - model = tf.keras.models.Model(inputs, merged) return model diff --git a/keras-tests/tfkeras/run.py b/keras-tests/tfkeras/run.py index 512951f..afa697c 100644 --- a/keras-tests/tfkeras/run.py +++ b/keras-tests/tfkeras/run.py @@ -15,8 +15,7 @@ import subprocess jobs = [mlp, rnn, cnn, merge, activations, advanced_activations] -jobs.remove(cnn) -jobs.remove(rnn) + def run_sequential(): for job in jobs: job.run() diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py index 3c4f274..45c1f46 100644 --- a/keras-tests/tfkeras/utils.py +++ b/keras-tests/tfkeras/utils.py @@ -5,6 +5,8 @@ import tensorflow as tf import json import os +import h5py +import numpy as np try: from tqdm import tqdm except ImportError: @@ -59,13 +61,39 @@ def _update_used_layers(model): with open(used_layers_file, 'w') as f: json.dump(list(_used_layers), f) -def save_model(model, file_name): - path = os.path.join(tfkeras_dir, file_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - model.save(path) - _update_used_layers(model) - - +def rand(shape): + shape = [d if d is not None else 1 for d in shape] + return np.random.random(shape) + +def put_data(model, input_arrays, output_arrays, h5file): + input_names = [i.encode('utf8') for i in model.input_names] + output_names = [o.encode('utf8') for o in model.output_names] + f = h5py.File(h5file, mode='w') + data = f.create_group('data') + data.attrs['input_names'] = input_names + data.attrs['output_names'] = output_names + for name, arr in zip(input_names, input_arrays): + data.create_dataset(name, shape=arr.shape, dtype='f', data=arr) + for name, arr in zip(output_names, output_arrays): + data.create_dataset(name, shape=arr.shape, dtype='f', data=arr) + +def put_rand_data(model, h5file): + input_shapes = [tuple(i.shape) for i in model.inputs] + input_arrays = list(map(rand, input_shapes)) + output_arrays = model.predict(input_arrays) + if not isinstance(output_arrays, list): + output_arrays = [output_arrays] + return put_data(model, input_arrays, output_arrays, h5file) + +def save_model(model, file_name, data='rand'): + try: + path = os.path.join(tfkeras_dir, file_name) + os.makedirs(os.path.dirname(path), exist_ok=True) + model.save(path) + put_rand_data(model, path) + _update_used_layers(model) + except: + pass def grid(*args, **kwargs): if args and callable(args[0]): From f5594fa446d0d0412692f7772c80c368a21e366f Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Mon, 13 Apr 2020 02:04:24 +0400 Subject: [PATCH 08/11] updates --- keras-tests/tfkeras/cnn.py | 5 ++--- keras-tests/tfkeras/run.py | 4 +++- keras-tests/tfkeras/utils.py | 6 ++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/keras-tests/tfkeras/cnn.py b/keras-tests/tfkeras/cnn.py index 99da578..a1b0015 100644 --- a/keras-tests/tfkeras/cnn.py +++ b/keras-tests/tfkeras/cnn.py @@ -275,7 +275,6 @@ def _start_proc(f): p.join() def _run_1d(): - return gen = generate_cnn1ds() for i, model in tqdm(enumerate(gen), total=len(gen)): if model: @@ -300,9 +299,9 @@ def _run_3d(): gc.collect() def run(): - #_start_proc(_run_1d) + _start_proc(_run_1d) _start_proc(_run_2d) - #_start_proc(_run_3d) + _start_proc(_run_3d) if __name__ == '__main__': run() diff --git a/keras-tests/tfkeras/run.py b/keras-tests/tfkeras/run.py index afa697c..ac6d3f0 100644 --- a/keras-tests/tfkeras/run.py +++ b/keras-tests/tfkeras/run.py @@ -5,6 +5,8 @@ import merge import activations import advanced_activations +import misc +import convlstm import utils import os import gc @@ -14,7 +16,7 @@ import sys import subprocess -jobs = [mlp, rnn, cnn, merge, activations, advanced_activations] +jobs = [mlp, rnn, cnn, merge, activations, advanced_activations, misc, convlstm] def run_sequential(): for job in jobs: diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py index 45c1f46..7f4d43a 100644 --- a/keras-tests/tfkeras/utils.py +++ b/keras-tests/tfkeras/utils.py @@ -7,6 +7,7 @@ import os import h5py import numpy as np +import click try: from tqdm import tqdm except ImportError: @@ -68,7 +69,7 @@ def rand(shape): def put_data(model, input_arrays, output_arrays, h5file): input_names = [i.encode('utf8') for i in model.input_names] output_names = [o.encode('utf8') for o in model.output_names] - f = h5py.File(h5file, mode='w') + f = h5py.File(h5file) data = f.create_group('data') data.attrs['input_names'] = input_names data.attrs['output_names'] = output_names @@ -76,6 +77,7 @@ def put_data(model, input_arrays, output_arrays, h5file): data.create_dataset(name, shape=arr.shape, dtype='f', data=arr) for name, arr in zip(output_names, output_arrays): data.create_dataset(name, shape=arr.shape, dtype='f', data=arr) + f.close() def put_rand_data(model, h5file): input_shapes = [tuple(i.shape) for i in model.inputs] @@ -92,7 +94,7 @@ def save_model(model, file_name, data='rand'): model.save(path) put_rand_data(model, path) _update_used_layers(model) - except: + except Exception as e: pass def grid(*args, **kwargs): From 425ccb65daf88a4ceb9d9c7c0ba2df161b712d3a Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Sun, 26 Apr 2020 14:33:09 +0530 Subject: [PATCH 09/11] updates --- keras-tests/tfkeras/cnn.py | 14 +++++++------- keras-tests/tfkeras/mlp.py | 2 +- keras-tests/tfkeras/rnn.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/keras-tests/tfkeras/cnn.py b/keras-tests/tfkeras/cnn.py index a1b0015..be5fdeb 100644 --- a/keras-tests/tfkeras/cnn.py +++ b/keras-tests/tfkeras/cnn.py @@ -18,20 +18,20 @@ kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) input_shape = [(20, 32)]#, (None, 16), (3, None), (1, None)] data_format = ['channels_first', 'channels_last'] -layer_type = [tf.keras.layers.Conv1D, tf.keras.layers.SeparableConv1D, tf.keras.layers.LocallyConnected1D] +layer_type = [tf.keras.layers.Conv1D]#, tf.keras.layers.LocallyConnected1D]#, tf.keras.layers.SeparableConv1D] num_layers = [2]#[i + 1 for i in range(max_num_layers)] filters = [4] strides = [1, 2, 3] dilation_rate = [1, 2] activation = ['tanh'] use_bias = [True]#, False] -pooling = [tf.keras.layers.MaxPooling1D, tf.keras.layers.AveragePooling1D] -global_pooling = [tf.keras.layers.GlobalMaxPool1D, tf.keras.layers.GlobalAvgPool1D] +pooling = [None]#tf.keras.layers.MaxPooling1D, tf.keras.layers.AveragePooling1D] +global_pooling = [None]#tf.keras.layers.GlobalMaxPool1D, tf.keras.layers.GlobalAvgPool1D] padding = ['causal', 'same', 'valid'] -zero_padding = [3]#, None] -upsampling = [2]#, None] -spatial_dropout = [True]#, False] -cropping = [(1, 1)]#, None] +zero_padding = [None]#, 3] +upsampling = [None] #2] +spatial_dropout = [False]#, True] +cropping = [None]#, (1, 1)] compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] @grid(**globals()) def generate_cnn1ds(input_shape, diff --git a/keras-tests/tfkeras/mlp.py b/keras-tests/tfkeras/mlp.py index 671a861..5fe29ad 100644 --- a/keras-tests/tfkeras/mlp.py +++ b/keras-tests/tfkeras/mlp.py @@ -9,7 +9,7 @@ layer_output_sizes = list(np.random.randint(3, 7, (max_num_layers,))) # grid -input_shape = [(3,), (4,), (2, 3), (4, 5)] +input_shape = [(3,), (4,)]#, (2, 3), (4, 5)] num_layers = [i + 1 for i in range(max_num_layers)] activation_type = ['layer', 'arg'] activation = ['sigmoid', 'relu', 'tanh'] diff --git a/keras-tests/tfkeras/rnn.py b/keras-tests/tfkeras/rnn.py index d9d948e..901d4a3 100644 --- a/keras-tests/tfkeras/rnn.py +++ b/keras-tests/tfkeras/rnn.py @@ -14,7 +14,7 @@ num_layers = [i + 1 for i in range(max_num_layers)] activation_type = ['layer', 'arg', 'recurrent'] activation = ['relu']#, 'tanh'] -rnn_type = [tf.keras.layers.LSTM, tf.keras.layers.GRU, tf.keras.layers.SimpleRNN] +rnn_type = [tf.keras.layers.SimpleRNN, tf.keras.layers.LSTM]#, tf.keras.layers.GRU] return_sequences = [True, False] dense_after = [False]#[True, False] compile_args = [None]#, {'loss': 'mse', 'optimizer': 'sgd'}] From 67656350a936b7d0a088dd758944037a177b9701 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Mon, 27 Apr 2020 02:47:57 +0530 Subject: [PATCH 10/11] updates --- keras-tests/tfkeras/activations.py | 4 +++- keras-tests/tfkeras/cnn.py | 15 ++++++++------- keras-tests/tfkeras/merge.py | 1 + keras-tests/tfkeras/misc.py | 31 ++++++++++++++++-------------- keras-tests/tfkeras/run.py | 3 ++- keras-tests/tfkeras/tfop.py | 28 +++++++++++++++++++++++++++ keras-tests/tfkeras/utils.py | 26 ++++++++++++++++++++----- 7 files changed, 80 insertions(+), 28 deletions(-) create mode 100644 keras-tests/tfkeras/tfop.py diff --git a/keras-tests/tfkeras/activations.py b/keras-tests/tfkeras/activations.py index a5a9c8f..2a3381a 100644 --- a/keras-tests/tfkeras/activations.py +++ b/keras-tests/tfkeras/activations.py @@ -10,11 +10,13 @@ activation = [act for act in dir(tf.keras.activations) if not act.startswith('_') and act not in ('serialize', 'deserialize', 'get')] +activation.remove('exponential') #not supported + use_dense = [True, False] @grid(**globals()) def generate_models(activation, use_dense): - inp = tf.keras.layers.Input((2, 3)) + inp = tf.keras.layers.Input((2,)) if use_dense: out = tf.keras.layers.Dense(4, activation=activation)(inp) else: diff --git a/keras-tests/tfkeras/cnn.py b/keras-tests/tfkeras/cnn.py index be5fdeb..61aa6ba 100644 --- a/keras-tests/tfkeras/cnn.py +++ b/keras-tests/tfkeras/cnn.py @@ -19,7 +19,7 @@ input_shape = [(20, 32)]#, (None, 16), (3, None), (1, None)] data_format = ['channels_first', 'channels_last'] layer_type = [tf.keras.layers.Conv1D]#, tf.keras.layers.LocallyConnected1D]#, tf.keras.layers.SeparableConv1D] -num_layers = [2]#[i + 1 for i in range(max_num_layers)] +num_layers = [1]#[2]#[i + 1 for i in range(max_num_layers)] filters = [4] strides = [1, 2, 3] dilation_rate = [1, 2] @@ -83,7 +83,7 @@ def generate_cnn1ds(input_shape, if cropping: x = tf.keras.layers.Cropping1D(cropping)(x) except ValueError as e: - warnings.warn(str(e)) + print(e) return None # not all stride / input shape comnbinations are valid if pooling: try: @@ -191,7 +191,7 @@ def generate_cnn2ds(input_shape, kernel_sizes = list(np.random.randint(2, 4, (max_num_layers,))) input_shape = [(3, 10, 10, 10)]#, (None, 16), (3, None), (1, None)] data_format = ['channels_first', 'channels_last'] -layer_type = [tf.keras.layers.Conv3D, tf.keras.layers.Conv3DTranspose] +layer_type = [tf.keras.layers.Conv3D]#, tf.keras.layers.Conv3DTranspose] num_layers = [2]#[i + 1 for i in range(max_num_layers)] filters = [4] strides = [1]#, 2] @@ -199,12 +199,12 @@ def generate_cnn2ds(input_shape, activation = ['tanh'] use_bias = [True]#, False] pooling = [tf.keras.layers.MaxPooling3D, tf.keras.layers.AveragePooling3D] -global_pooling = [tf.keras.layers.GlobalMaxPool3D, tf.keras.layers.GlobalAvgPool3D] +global_pooling = [None]#tf.keras.layers.GlobalMaxPool3D, tf.keras.layers.GlobalAvgPool3D] padding = ['causal', 'same', 'valid'] -zero_padding = [3]#, None] -upsampling = [2]#, None] +zero_padding = [None]#, 3] +upsampling = [None]#, 2] spatial_dropout = [True]#, False] -cropping = [(1, 1, 1)]#, None] +cropping = [(1, 1, 1), None] compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] @grid(**globals()) def generate_cnn3ds(input_shape, @@ -262,6 +262,7 @@ def generate_cnn3ds(input_shape, pass if global_pooling: x = global_pooling()(x) + x = tf.keras.layers.Flatten()(x) model = tf.keras.models.Model(inp, x) if compile_args: model.compile(**compile_args) diff --git a/keras-tests/tfkeras/merge.py b/keras-tests/tfkeras/merge.py index 56439d4..9404325 100644 --- a/keras-tests/tfkeras/merge.py +++ b/keras-tests/tfkeras/merge.py @@ -44,5 +44,6 @@ def run(): if model: save_model(model, 'merge_' + str(i) + '.h5') + if __name__ == '__main__': run() diff --git a/keras-tests/tfkeras/misc.py b/keras-tests/tfkeras/misc.py index bd404ad..bfc5a90 100644 --- a/keras-tests/tfkeras/misc.py +++ b/keras-tests/tfkeras/misc.py @@ -6,26 +6,29 @@ def get_misc_models(): models = [] - inp = tf.keras.layers.Input((2, 3)) - x = tf.keras.layers.Dense(4)(inp) - x = tf.keras.layers.Flatten()(x) - x = tf.keras.layers.Dense(1)(x) - models.append(tf.keras.models.Model(inp, x)) + ## TODO + # inp = tf.keras.layers.Input((2, 3)) + # x = tf.keras.layers.Dense(4)(inp) + # x = tf.keras.layers.Flatten()(x) + # x = tf.keras.layers.Dense(1)(x) + # models.append(tf.keras.models.Model(inp, x)) inp = tf.keras.layers.Input((5,)) x = tf.keras.layers.Embedding(10, 8)(inp) x = tf.keras.layers.Dense(4)(x) models.append(tf.keras.models.Model(inp, x)) - inp = tf.keras.layers.Input((5,)) - x = tf.keras.layers.Embedding(10, 8, mask_zero=True)(inp) - x = tf.keras.layers.Masking()(x) - x = tf.keras.layers.Dense(4)(x) - models.append(tf.keras.models.Model(inp, x)) + # TODO + # inp = tf.keras.layers.Input((5,)) + # x = tf.keras.layers.Embedding(10, 8, mask_zero=True)(inp) + # x = tf.keras.layers.Masking()(x) + # x = tf.keras.layers.Dense(4)(x) + # models.append(tf.keras.models.Model(inp, x)) - inp = tf.keras.layers.Input((2, 3)) - x = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(4))(inp) - models.append(tf.keras.models.Model(inp, x)) + # TODO + # inp = tf.keras.layers.Input((2, 3)) + # x = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(4))(inp) + # models.append(tf.keras.models.Model(inp, x)) inp = tf.keras.layers.Input((2, 3)) x = tf.keras.layers.Reshape((3, 2))(inp) @@ -35,7 +38,7 @@ def get_misc_models(): x = tf.keras.layers.Permute((2, 1))(inp) models.append(tf.keras.models.Model(inp, x)) - inp = tf.keras.layers.Input((2, 3)) + inp = tf.keras.layers.Input((2,)) x = tf.keras.layers.Dense(5)(inp) x = tf.keras.layers.AlphaDropout(0.5)(x) models.append(tf.keras.models.Model(inp, x)) diff --git a/keras-tests/tfkeras/run.py b/keras-tests/tfkeras/run.py index ac6d3f0..94d2ae6 100644 --- a/keras-tests/tfkeras/run.py +++ b/keras-tests/tfkeras/run.py @@ -7,6 +7,7 @@ import advanced_activations import misc import convlstm +import tfop import utils import os import gc @@ -16,7 +17,7 @@ import sys import subprocess -jobs = [mlp, rnn, cnn, merge, activations, advanced_activations, misc, convlstm] +jobs = [mlp, rnn, cnn, merge, activations, advanced_activations, misc, convlstm, tfop] def run_sequential(): for job in jobs: diff --git a/keras-tests/tfkeras/tfop.py b/keras-tests/tfkeras/tfop.py new file mode 100644 index 0000000..df05f2a --- /dev/null +++ b/keras-tests/tfkeras/tfop.py @@ -0,0 +1,28 @@ +import tensorflow as tf +import numpy as np +from utils import grid, save_model +from utils import tqdm + + +def get_tfop_models(): + models = [] + + inp = tf.keras.layers.Input((2, 3)) + reshaped = tf.reshape(inp, (-1, 6)) + out = tf.keras.layers.Dense(5)(reshaped) + models.append(tf.keras.models.Model(inp, out)) + + inp = tf.keras.layers.Input((3, 2)) + perm = tf.transpose(inp, (0, 2, 1)) + out = tf.keras.layers.Dense(5)(perm) + models.append(tf.keras.models.Model(inp, out)) + + return models + +def run(): + gen = get_tfop_models() + for i, model in tqdm(enumerate(gen), total=len(gen)): + save_model(model, 'tfop_' + str(i) + '.h5') + +if __name__ == '__main__': + run() diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py index 7f4d43a..14029b0 100644 --- a/keras-tests/tfkeras/utils.py +++ b/keras-tests/tfkeras/utils.py @@ -87,15 +87,31 @@ def put_rand_data(model, h5file): output_arrays = [output_arrays] return put_data(model, input_arrays, output_arrays, h5file) -def save_model(model, file_name, data='rand'): +def _to_sequential(model): + seq = tf.keras.models.Sequential() + for layer in model.layers[1:]: + seq.add(layer) + return seq + + +def save_model(model, file_name, data='rand', as_sequential=False): try: + if as_sequential: + model = _to_sequential(model) + print(model.layers[0].input_shape) + model.predict(rand(model.layers[0].input_shape)) path = os.path.join(tfkeras_dir, file_name) os.makedirs(os.path.dirname(path), exist_ok=True) - model.save(path) - put_rand_data(model, path) - _update_used_layers(model) + try: + model.predict(rand(model.input_shape)) + model.save(path) + put_rand_data(model, path) + _update_used_layers(model) + except: + pass + except Exception as e: - pass + print(e) def grid(*args, **kwargs): if args and callable(args[0]): From cda25deda0c9700921be4b0bc7091bd9c2e57d39 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Mon, 27 Apr 2020 06:58:57 +0530 Subject: [PATCH 11/11] fix merge --- keras-tests/tfkeras/cnn.py | 18 +++++------ keras-tests/tfkeras/merge.py | 59 +++++++++++++++++++++++------------- keras-tests/tfkeras/utils.py | 44 +++++++++++++-------------- 3 files changed, 69 insertions(+), 52 deletions(-) diff --git a/keras-tests/tfkeras/cnn.py b/keras-tests/tfkeras/cnn.py index 61aa6ba..1bf37ff 100644 --- a/keras-tests/tfkeras/cnn.py +++ b/keras-tests/tfkeras/cnn.py @@ -19,7 +19,7 @@ input_shape = [(20, 32)]#, (None, 16), (3, None), (1, None)] data_format = ['channels_first', 'channels_last'] layer_type = [tf.keras.layers.Conv1D]#, tf.keras.layers.LocallyConnected1D]#, tf.keras.layers.SeparableConv1D] -num_layers = [1]#[2]#[i + 1 for i in range(max_num_layers)] +num_layers = [1, 2]#[2]#[i + 1 for i in range(max_num_layers)] filters = [4] strides = [1, 2, 3] dilation_rate = [1, 2] @@ -104,8 +104,8 @@ def generate_cnn1ds(input_shape, data_format = ['channels_first', 'channels_last'] layer_type = [tf.keras.layers.Conv2D, tf.keras.layers.SeparableConv2D, - tf.keras.layers.LocallyConnected2D, - tf.keras.layers.DepthwiseConv2D, + #tf.keras.layers.LocallyConnected2D, + #tf.keras.layers.DepthwiseConv2D, tf.keras.layers.Conv2DTranspose] num_layers = [2]#[i + 1 for i in range(max_num_layers)] filters = [4] @@ -113,13 +113,13 @@ def generate_cnn1ds(input_shape, dilation_rate = [1]#, 2] activation = ['tanh'] use_bias = [True]#, False] -pooling = [tf.keras.layers.MaxPooling2D, tf.keras.layers.AveragePooling2D] -global_pooling = [tf.keras.layers.GlobalMaxPool2D, tf.keras.layers.GlobalAvgPool2D] +pooling = [None]#tf.keras.layers.MaxPooling2D, tf.keras.layers.AveragePooling2D] +global_pooling = [None]#tf.keras.layers.GlobalMaxPool2D, tf.keras.layers.GlobalAvgPool2D] padding = ['causal', 'same', 'valid'] -zero_padding = [3]#, None] -upsampling = [2]#, None] -spatial_dropout = [True]#, False] -cropping = [(1, 1)]#, None] +zero_padding = [None]#, 3] +upsampling = [None]#, 2] +spatial_dropout = [False]#, True] +cropping = [None]#, (1, 1)] compile_args = [{'optimizer': 'rmsprop', 'loss': 'categorical_crossentropy', 'metrics': ['accuracy']}] @grid(**globals()) def generate_cnn2ds(input_shape, diff --git a/keras-tests/tfkeras/merge.py b/keras-tests/tfkeras/merge.py index 9404325..7d6c93e 100644 --- a/keras-tests/tfkeras/merge.py +++ b/keras-tests/tfkeras/merge.py @@ -5,12 +5,9 @@ from utils import tqdm -input_shapes_and_axis = [[(2,), (2,), -1], [(None, 2), (None, 3), -1], -[(None, 2), (3, 4), -1], [(3, 2), (4, 2), -2], [(3, 2), (4, 2), -1], -[(2, 3), (3, 4), (1, 0)]] +input_shapes = [[(2,), (2,)], [(None, 2), (None, 2)], [(None, 2, 3), (None, 2, 3)]] + merger = [ - tf.keras.layers.Dot, - tf.keras.layers.Concatenate, tf.keras.layers.Add, tf.keras.layers.Multiply, tf.keras.layers.Average, @@ -19,31 +16,51 @@ tf.keras.layers.Minimum ] + @grid(**globals()) -def generate_merge_models(input_shapes_and_axis, merger): - input_shapes = input_shapes_and_axis[:-1] - axis = input_shapes_and_axis[-1] +def generate_merge_models(input_shapes, merger): inputs = list(map(tf.keras.layers.Input, input_shapes)) - if merger == tf.keras.layers.Concatenate: - merge_args = {'axis' : axis} - elif merger == tf.keras.layers.Dot: - merge_args = {'axes': axis} - else: - merge_args = {} - try: - merged = merger(**merge_args)(inputs) - model = tf.keras.models.Model(inputs, merged) - except: - return - return model + merged = merger()(inputs) + model = tf.keras.models.Model(inputs, merged) + +def dot_models(): + models = [] + args = [ + [(2, 3), (2, 2), (1, 1)], + [(2, 3), (3, 3), (2, 1)] + ] + for arg in args: + shapes = arg[:2] + inputs = list(map(tf.keras.layers.Input, shapes)) + out = tf.keras.layers.Dot(axes=arg[-1])(inputs) + models.append(tf.keras.models.Model(inputs, out)) + return models +def concat_models(): + models = [] + args = [ + [(2, 3), (4, 3), 1], + [(2, 3), (2, 4), 2] + ] + for arg in args: + shapes = arg[:2] + inputs = list(map(tf.keras.layers.Input, shapes)) + out = tf.keras.layers.Concatenate(axis=arg[-1])(inputs) + models.append(tf.keras.models.Model(inputs, out)) + return models + def run(): gen = generate_merge_models() for i, model in tqdm(enumerate(gen), total=len(gen)): if model: save_model(model, 'merge_' + str(i) + '.h5') - + for i, model in tqdm(enumerate(dot_models())): + if model: + save_model(model, 'merge_dot_' + str(i) + '.h5') + for i, model in tqdm(enumerate(concat_models())): + if model: + save_model(model, 'merge_concat_' + str(i) + '.h5') if __name__ == '__main__': run() diff --git a/keras-tests/tfkeras/utils.py b/keras-tests/tfkeras/utils.py index 14029b0..c340a04 100644 --- a/keras-tests/tfkeras/utils.py +++ b/keras-tests/tfkeras/utils.py @@ -55,14 +55,20 @@ def _get_layers(model): _used_layers = set() def _update_used_layers(model): - n = len(_used_layers) - layers = _get_layers(model) - _used_layers.update(layers) - if len(_used_layers) > n: - with open(used_layers_file, 'w') as f: - json.dump(list(_used_layers), f) + try: + n = len(_used_layers) + layers = _get_layers(model) + _used_layers.update(layers) + if len(_used_layers) > n: + with open(used_layers_file, 'w') as f: + json.dump(list(_used_layers), f) + except Exception as e: + print('Error recording used layers: ' + str(e)) + def rand(shape): + if isinstance(shape, list): + return list(map(rand, shape)) shape = [d if d is not None else 1 for d in shape] return np.random.random(shape) @@ -95,23 +101,17 @@ def _to_sequential(model): def save_model(model, file_name, data='rand', as_sequential=False): - try: - if as_sequential: - model = _to_sequential(model) - print(model.layers[0].input_shape) - model.predict(rand(model.layers[0].input_shape)) - path = os.path.join(tfkeras_dir, file_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - try: - model.predict(rand(model.input_shape)) - model.save(path) - put_rand_data(model, path) - _update_used_layers(model) - except: - pass + if as_sequential: + model = _to_sequential(model) + print(model.layers[0].input_shape) + model.predict(rand(model.layers[0].input_shape)) + path = os.path.join(tfkeras_dir, file_name) + os.makedirs(os.path.dirname(path), exist_ok=True) + model.predict(rand(model.input_shape)) + model.save(path) + put_rand_data(model, path) + _update_used_layers(model) - except Exception as e: - print(e) def grid(*args, **kwargs): if args and callable(args[0]):