diff --git a/docs/models.rst b/docs/models.rst index 8dd54ea..671e71e 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -12,19 +12,18 @@ We include three `examples`_ for you to try: a model trained on the `MNIST`_ dat If you built your model with Keras using a `Sequential`_ model, you should be more or less good to go. If you used Tensorflow, you'll need to manually specify the entry and exit points [#]_. -You can specify the backend (Tensorflow or Keras) using the ``PICASSO_BACKEND_ML`` setting. The allowed values are ``tensorflow`` or ``keras`` (see :doc:`settings`). - Your model data =============== -You can specify the data directory with the ``PICASSO_DATA_DIR`` setting. This directory should contain the Keras or Tensorflow checkpoint files. If multiple checkpoints are found, the latest one will be used (see example `Keras model code`_). +You can specify the data directory with the ``MODEL_LOAD_ARGS.data_dir`` setting (see :doc:`settings`). This directory should contain the Keras or Tensorflow checkpoint files. If multiple checkpoints are found, the latest one will be used (see example `Keras model code`_). Utility functions ================= In addition to the graph and weight information of the model itself, you'll need to define a few functions to help the visualization interact with user input, and interpret raw output from your computational graph. These are arbitrary python functions, and their locations can be specified in the :doc:`settings`. -We'll draw from the `Keras MNIST example`_ for this guide. +We'll draw from the `Keras MNIST example`_ for this guide. All custom models +from the relevant model: either ``KerasModel`` or ``TensorflowModel``. Preprocessor ------------ @@ -33,39 +32,29 @@ The preprocessor takes images uploaded to the webapp and converts them into arra .. code-block:: python3 - MNIST_DIM = (28, 28) - - def preprocess(targets): - image_arrays = [] - for target in targets: - im = target.convert('L') - im = im.resize(MNIST_DIM, Image.ANTIALIAS) - arr = np.array(im) - image_arrays.append(arr) - - all_targets = np.array(image_arrays) - return all_targets.reshape(len(all_targets), - MNIST_DIM[0], - MNIST_DIM[1], 1).astype('float32') / 255 - -Specifically, we have to convert an arbitrary input color image to a float array of the input size specified with ``MNIST_DIM``. + import numpy as np + from PIL import Image + + from picasso.models.keras import KerasModel -Postprocessor -------------- + MNIST_DIM = (28, 28) -For some visualizations, it's useful to convert a flat representation back into an array with the same shape as the original image. + class KerasMNISTModel(KerasModel): -.. code-block:: python3 + def preprocess(self, raw_inputs): + image_arrays = [] + for target in targets: + im = target.convert('L') + im = im.resize(MNIST_DIM, Image.ANTIALIAS) + arr = np.array(im) + image_arrays.append(arr) - def postprocess(output_arr): - images = [] - for row in output_arr: - im_array = row.reshape(MNIST_DIM) - images.append(im_array) + all_targets = np.array(image_arrays) + return all_targets.reshape(len(all_targets), + MNIST_DIM[0], + MNIST_DIM[1], 1).astype('float32') / 255 - return images - -This therefore takes an arbitrary array (with the same number of total entries as the image array) and reshapes it back. +Specifically, we have to convert an arbitrary input color image to a float array of the input size specified with ``MNIST_DIM``. Class Decoder ------------- @@ -74,24 +63,27 @@ Class probabilities are usually returned in an array. For any visualization whe .. code-block:: python3 - def prob_decode(probability_array, top=5): - results = [] - for row in probability_array: - entries = [] - for i, prob in enumerate(row): - entries.append({'index': i, - 'name': str(i), - 'prob': prob}) + class KerasMNISTModel(KerasModel): - entries = sorted(entries, - key=itemgetter('prob'), - reverse=True)[:top] + ... + + def decode_prob(self, class_probabilities): + results = [] + for row in class_probabilities: + entries = [] + for i, prob in enumerate(row): + entries.append({'index': i, + 'name': str(i), + 'prob': prob}) - for entry in entries: - entry['prob'] = '{:.3f}'.format(entry['prob']) - results.append(entries) + entries = sorted(entries, + key=itemgetter('prob'), + reverse=True)[:self.top_probs] - return results + for entry in entries: + entry['prob'] = '{:.3f}'.format(entry['prob']) + results.append(entries) + return results ``results`` is then a list of dicts in the format ``[{'index': class_index, 'name': class_name, 'prob': class_probability}, ...]``. In the case of the MNIST dataset, the index is the same as the class name (digits 0-9). @@ -103,9 +95,9 @@ Class probabilities are usually returned in an array. For any visualization whe .. _Sequential: https://keras.io/models/sequential/ -.. _Keras model code: https://github.com/merantix/picasso/blob/master/picasso/ml_frameworks/keras/model.py +.. _Keras model code: https://github.com/merantix/picasso/blob/master/picasso/keras/keras.py -.. _Keras MNIST example: https://github.com/merantix/picasso/blob/master/picasso/examples/keras/util.py +.. _Keras MNIST example: https://github.com/merantix/picasso/blob/master/picasso/examples/keras/model.py .. _PIL Image: http://pillow.readthedocs.io/en/latest/reference/Image.html diff --git a/docs/settings.rst b/docs/settings.rst index e465a4c..a2d189a 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -21,29 +21,18 @@ Tells the app to use this configuration instead of the default one. Inside base_dir = os.path.split(os.path.abspath(__file__))[0] - BACKEND_ML = 'tensorflow' - BACKEND_PREPROCESSOR_NAME = 'util' - BACKEND_PREPROCESSOR_PATH = os.path.join(base_dir, 'util.py') - BACKEND_POSTPROCESSOR_NAME = 'postprocess' - BACKEND_POSTPROCESSOR_PATH = os.path.join(base_dir, 'util.py') - BACKEND_PROB_DECODER_NAME = 'prob_decode' - BACKEND_PROB_DECODER_PATH = os.path.join(base_dir, 'util.py') - DATA_DIR = os.path.join(base_dir, 'data-volume') - -Any lowercase line is ignored for the purposes of determining a setting. These -can also be set via environment variables, but you must append the app name. -For instance ``BACKEND_ML = 'tensorflow'`` would become ``export -PICASSO_BACKEND_ML=tensorflow``. - -For explanations of each setting, see :mod:`picasso.settings`. Any -additional settings starting with `BACKEND_` will be sent to the model backend -as a keyword argument. The input and output tensor names can be passed to the -Tensorflow backend in this way: - -.. code-block:: python3 - - ... - BACKEND_TF_PREDICT_VAR='Softmax:0' - BACKEND_TF_INPUT_VAR='convolution2d_input_1:0' + MODEL_CLS_PATH = os.path.join(base_dir, 'model.py') + MODEL_CLS_NAME = 'TensorflowMNISTModel' + MODEL_LOAD_ARGS = { + 'data_dir': os.path.join(base_dir, 'data-volume'), + 'tf_input_var': 'convolution2d_input_1:0', + 'tf_predict_var': 'Softmax:0', + } + +Any lowercase line is ignored for the purposes of determining a setting. +``MODEL_LOAD_ARGS`` will pass the arguments along to the model's ``load`` +function. + +For explanations of each setting, see :mod:`picasso.config`. .. _managed by Flask: http://flask.pocoo.org/docs/latest/config/ diff --git a/docs/visualizations.rst b/docs/visualizations.rst index 86d65c1..51cd834 100644 --- a/docs/visualizations.rst +++ b/docs/visualizations.rst @@ -13,14 +13,12 @@ For our example, ``FunViz``, we'll need ``picasso/visualizations/fun_viz.py``: .. code-block:: python3 - from picasso.visualizations import BaseVisualization + from picasso.visualizations.base import BaseVisualization class FunViz(BaseVisualization): - def __init__(self, model): - self.description = 'A fun visualization!' - self.model = model + DESCRIPTION = 'A fun visualization!' def make_visualization(self, inputs, output_dir, settings=None): pass @@ -34,7 +32,7 @@ and ``picasso/templates/FunViz.html``: your visualization html goes here {% endblock %} -Some explanation for the ``FunViz`` class in ``fun_viz.py``: All visualizations should inherit from :class:`~picasso.visualizations.__init__.BaseVisualization` (see `code `_). You must implement the ``__init__`` method, and it should accept one argument, ``model``. ``model`` will be an instance of a child class of `Model`_, which provides an interface to the machine learning backend. You can also add a description which will display on the landing page. +Some explanation for the ``FunViz`` class in ``fun_viz.py``: All visualizations should inherit from :class:`~picasso.visualizations.base.__init__.BaseVisualization`. You can also add a description which will display on the landing page. Some explanation for ``FunViz.html``: The web app is uses `Flask`_, which uses `Jinja2`_ templating. This explains the funny ``{% %}`` delimiters. The ``{% extends "result.html" %}`` just tells the your page to inherit from a boilerplate. All your html should sit within the ``vis`` block. @@ -53,16 +51,14 @@ Add visualization logic Our visualization should actually do something. It's just going to compute the class probabilities and pass them back along to the web app. So we'll add: .. code-block:: python3 - :emphasize-lines: 11-21 + :emphasize-lines: 9-21 - from picasso.visualizations import BaseVisualization + from picasso.visualizations.base import BaseVisualization class FunViz(BaseVisualization): - def __init__(self, model): - self.description = 'A fun visualization!' - self.model = model + DESCRIPTION = 'A fun visualization!' def make_visualization(self, inputs, output_dir, settings=None): pre_processed_arrays = self.model.preprocess([example['data'] @@ -311,20 +307,19 @@ Similarly, there is an ``outputs/`` folder (not shown in this example). Its pat Add some settings ================= -Maybe we'd like the user to be able to limit the number of classes shown. We can easily do this by adding a ``settings`` property to the ``FunViz`` class. +Maybe we'd like the user to be able to limit the number of classes shown. We can easily do this by adding an ``ALLOWED_SETTINGS`` property to the ``FunViz`` class. .. code-block:: python3 - :emphasize-lines: 5, 21 + :emphasize-lines: 6, 20 from picasso.visualizations import BaseVisualization class FunViz(BaseVisualization): - settings = {'Display': ['1', '2', '3']} - def __init__(self, model): - self.description = 'A fun visualization!' - self.model = model + ALLOWED_SETTINGS = {'Display': ['1', '2', '3']} + + DESCRIPTION = 'A fun visualization!' def make_visualization(self, inputs, output_dir, settings=None): pre_processed_arrays = self.model.preprocess([example['data'] @@ -391,10 +386,6 @@ For more complex visualizations, see the examples in `the visualizations module` .. _template: https://github.com/merantix/picasso/blob/master/picasso/templates/ClassProbabilities.html -.. _BaseVisualization: https://github.com/merantix/picasso/blob/master/picasso/visualizations/__init__.py - -.. _Model: https://github.com/merantix/picasso/blob/master/picasso/ml_frameworks/model.py - .. _Flask: http://flask.pocoo.org/ .. _Jinja2: http://jinja.pocoo.org/docs/ diff --git a/picasso/__init__.py b/picasso/__init__.py index d931652..9d21010 100644 --- a/picasso/__init__.py +++ b/picasso/__init__.py @@ -13,9 +13,28 @@ raise SystemError('Python 3.5+ required, found {}'.format(sys.version)) app = Flask(__name__) -app.config.from_object('picasso.settings.Default') +app.config.from_object('picasso.config.Default') if os.getenv('PICASSO_SETTINGS'): app.config.from_envvar('PICASSO_SETTINGS') +deprecated_settings = ['BACKEND_PREPROCESSOR_NAME', + 'BACKEND_PREPROCESSOR_PATH', + 'BACKEND_POSTPROCESSOR_NAME', + 'BACKEND_POSTPROCESSOR_PATH', + 'BACKEND_PROB_DECODER_NAME', + 'BACKEND_PROB_DECODER_PATH', + 'DATA_DIR'] + +if any([x in app.config.keys() for x in deprecated_settings]): + raise ValueError('It looks like you\'re using a deprecated' + ' setting. The settings and utility functions' + ' have been changed as of version v0.2.0 (and ' + 'you\'re using {}). Changing to the updated ' + ' settings is trivial: see ' + 'https://picasso.readthedocs.io/en/latest/models.html' + ' and ' + 'https://picasso.readthedocs.io/en/latest/settings.html' + .format(__version__)) + import picasso.picasso diff --git a/picasso/config.py b/picasso/config.py new file mode 100644 index 0000000..9e8dd9d --- /dev/null +++ b/picasso/config.py @@ -0,0 +1,30 @@ +import os + +base_dir = os.path.dirname(__file__) # only for default config + + +class Default: + """Default settings for the Flask app. + + The Flask app uses these settings if no custom settings are defined. You + can define custom settings by creating a Python module, defining global + variables in that module, and setting the environment variable + `PICASSO_SETTINGS` to the path to that module. + + If `PICASSO_SETTINGS` is not set, or if any particular setting is not + defined in the indicated module, then the Flask app uses these default + settings. + + """ + # :obj:`str`: filepath of the module containing the model to run + MODEL_CLS_PATH = os.path.join( + base_dir, 'examples', 'keras', 'model.py') + + # :obj:`str`: name of model class + MODEL_CLS_NAME = 'KerasMNISTModel' + + # :obj:`dict`: dictionary of args to pass to the `load` method of the + # model instance. + MODEL_LOAD_ARGS = { + 'data_dir': os.path.join(base_dir, 'examples', 'keras', 'data-volume'), + } diff --git a/picasso/examples/keras-vgg16/config.py b/picasso/examples/keras-vgg16/config.py index 7ab3fa5..4cbde9a 100644 --- a/picasso/examples/keras-vgg16/config.py +++ b/picasso/examples/keras-vgg16/config.py @@ -1,12 +1,15 @@ +# Note: By default, Flask doesn't know that this file exists. If you want +# Flask to load the settings you specify here, you must set the environment +# variable `PICASSO_SETTINGS` to point to this file. E.g.: +# +# export PICASSO_SETTINGS=/path/to/examples/keras-vgg16/config.py +# import os base_dir = os.path.dirname(os.path.abspath(__file__)) -BACKEND_ML = 'keras' -BACKEND_PREPROCESSOR_NAME = 'preprocess' -BACKEND_PREPROCESSOR_PATH = os.path.join(base_dir, 'util.py') -BACKEND_POSTPROCESSOR_NAME = 'postprocess' -BACKEND_POSTPROCESSOR_PATH = os.path.join(base_dir, 'util.py') -BACKEND_PROB_DECODER_NAME = 'prob_decode' -BACKEND_PROB_DECODER_PATH = os.path.join(base_dir, 'util.py') -DATA_DIR = os.path.join(base_dir, 'data-volume') +MODEL_CLS_PATH = os.path.join(base_dir, 'model.py') +MODEL_CLS_NAME = 'KerasVGG16Model' +MODEL_LOAD_ARGS = { + 'data_dir': os.path.join(base_dir, 'data-volume'), +} diff --git a/picasso/examples/keras-vgg16/model.py b/picasso/examples/keras-vgg16/model.py new file mode 100644 index 0000000..963e512 --- /dev/null +++ b/picasso/examples/keras-vgg16/model.py @@ -0,0 +1,48 @@ +from keras.applications import imagenet_utils +import numpy as np +from PIL import Image + +from picasso.models.keras import KerasModel + +VGG16_DIM = (224, 224, 3) + + +class KerasVGG16Model(KerasModel): + + def preprocess(self, raw_inputs): + """ + Args: + raw_inputs (list of Images): a list of PIL Image objects + Returns: + array (float32): num images * height * width * num channels + """ + image_arrays = [] + for raw_im in raw_inputs: + im = raw_im.resize(VGG16_DIM[:2], Image.ANTIALIAS) + im = im.convert('RGB') + arr = np.array(im).astype('float32') + image_arrays.append(arr) + + all_raw_inputs = np.array(image_arrays) + return imagenet_utils.preprocess_input(all_raw_inputs) + + def decode_prob(self, class_probabilities): + r = imagenet_utils.decode_predictions(class_probabilities, + top=self.top_probs) + results = [ + [{'code': entry[0], + 'name': entry[1], + 'prob': '{:.3f}'.format(entry[2])} + for entry in row] + for row in r + ] + classes = imagenet_utils.CLASS_INDEX + class_keys = list(classes.keys()) + class_values = list(classes.values()) + + for result in results: + for entry in result: + entry['index'] = int( + class_keys[class_values.index([entry['code'], + entry['name']])]) + return results diff --git a/picasso/examples/keras-vgg16/util.py b/picasso/examples/keras-vgg16/util.py deleted file mode 100644 index e56eaa0..0000000 --- a/picasso/examples/keras-vgg16/util.py +++ /dev/null @@ -1,53 +0,0 @@ -from keras.applications.imagenet_utils import (decode_predictions, - preprocess_input) -import keras.applications.imagenet_utils -from PIL import Image -import numpy as np - -VGG16_DIM = (224, 224, 3) - - -def preprocess(targets): - image_arrays = [] - for target in targets: - im = target.resize(VGG16_DIM[:2], Image.ANTIALIAS) - im = im.convert('RGB') - arr = np.array(im).astype('float32') - image_arrays.append(arr) - - all_targets = np.array(image_arrays) - return preprocess_input(all_targets) - - -def postprocess(output_arr): - images = [] - for row in output_arr: - im_array = row.reshape(VGG16_DIM[:2]) - images.append(im_array) - - return images - - -def prob_decode(probability_array, top=5): - r = decode_predictions(probability_array, top=top) - results = [ - [{'code': entry[0], - 'name': entry[1], - 'prob': '{:.3f}'.format(entry[2])} - for entry in row] - for row in r - ] - classes = keras.applications.imagenet_utils.CLASS_INDEX - class_keys = list(classes.keys()) - class_values = list(classes.values()) - - for result in results: - for entry in result: - entry.update( - {'index': - int( - class_keys[class_values.index([entry['code'], - entry['name']])] - )} - ) - return results diff --git a/picasso/examples/keras/config.py b/picasso/examples/keras/config.py index 0ecdf76..994ce70 100644 --- a/picasso/examples/keras/config.py +++ b/picasso/examples/keras/config.py @@ -1,19 +1,15 @@ -# Note: this settings file duplicates the default settings in the top-level -# file `settings.py`. If you want to modify settings here, you must export the -# path to this file: +# Note: By default, Flask doesn't know that this file exists. If you want +# Flask to load the settings you specify here, you must set the environment +# variable `PICASSO_SETTINGS` to point to this file. E.g.: # -# export PICASSO_SETTINGS=/path/to/picasso/picasso/examples/keras/config.py +# export PICASSO_SETTINGS=/path/to/examples/keras/config.py # -# otherwise, these settings will not be loaded. import os base_dir = os.path.dirname(os.path.abspath(__file__)) -BACKEND_ML = 'keras' -BACKEND_PREPROCESSOR_NAME = 'preprocess' -BACKEND_PREPROCESSOR_PATH = os.path.join(base_dir, 'util.py') -BACKEND_POSTPROCESSOR_NAME = 'postprocess' -BACKEND_POSTPROCESSOR_PATH = os.path.join(base_dir, 'util.py') -BACKEND_PROB_DECODER_NAME = 'prob_decode' -BACKEND_PROB_DECODER_PATH = os.path.join(base_dir, 'util.py') -DATA_DIR = os.path.join(base_dir, 'data-volume') +MODEL_CLS_PATH = os.path.join(base_dir, 'model.py') +MODEL_CLS_NAME = 'KerasMNISTModel' +MODEL_LOAD_ARGS = { + 'data_dir': os.path.join(base_dir, 'data-volume'), +} diff --git a/picasso/examples/keras/model.py b/picasso/examples/keras/model.py new file mode 100644 index 0000000..b6534ad --- /dev/null +++ b/picasso/examples/keras/model.py @@ -0,0 +1,35 @@ +import numpy as np +from PIL import Image + +from picasso.models.keras import KerasModel + +MNIST_DIM = (28, 28) + + +class KerasMNISTModel(KerasModel): + + def preprocess(self, raw_inputs): + """Convert images into the format required by our model. + + Our model requires that inputs be grayscale (mode 'L'), be resized to + `MNIST_DIM`, and be represented as float32 numpy arrays in range + [0, 1]. + + Args: + raw_inputs (list of Images): a list of PIL Image objects + + Returns: + array (float32): num images * height * width * num channels + + """ + image_arrays = [] + for raw_im in raw_inputs: + im = raw_im.convert('L') + im = im.resize(MNIST_DIM, Image.ANTIALIAS) + arr = np.array(im) + image_arrays.append(arr) + + inputs = np.array(image_arrays) + return inputs.reshape(len(inputs), + MNIST_DIM[0], + MNIST_DIM[1], 1).astype('float32') / 255 diff --git a/picasso/examples/keras/util.py b/picasso/examples/keras/util.py deleted file mode 100644 index 6f6202e..0000000 --- a/picasso/examples/keras/util.py +++ /dev/null @@ -1,90 +0,0 @@ -from PIL import Image -from operator import itemgetter -import numpy as np - -MNIST_DIM = (28, 28) - - -def preprocess(targets): - """Turn images into computation inputs - - Converts an iterable of PIL Images into a suitably-sized numpy array which - can be used as an input to the evaluation portion of the Keras/tensorflow - graph. - - Args: - targets (list of Images): a list of PIL Image objects - - Returns: - array (float32) - - """ - image_arrays = [] - for target in targets: - im = target.convert('L') - im = im.resize(MNIST_DIM, Image.ANTIALIAS) - arr = np.array(im) - image_arrays.append(arr) - - all_targets = np.array(image_arrays) - return all_targets.reshape(len(all_targets), - MNIST_DIM[0], - MNIST_DIM[1], 1).astype('float32') / 255 - - -def postprocess(output_arr): - """Reshape arrays to original image dimensions - - Typically used for outputs or computations on intermediate layers which - make sense to represent as an image in the original dimension of the input - images (see ``SaliencyMaps``). - - Args: - output_arr (array of float32): Array of leading dimension n containing - n arrays to be reshaped - - Returns: - reshaped array - - """ - images = [] - for row in output_arr: - im_array = row.reshape(MNIST_DIM) - images.append(im_array) - - return images - - -def prob_decode(probability_array, top=5): - """Provide class information from output probabilities - - Gives the visualization additional context for the computed class - probabilities. - - Args: - probability_array (array): class probabilities - top (int): number of class entries to return. Useful for limiting - output in models with many classes. Defaults to 5. - - Returns: - result list of dict in the format [{'index': class_index, 'name': - class_name, 'prob': class_probability}, ...] - - """ - results = [] - for row in probability_array: - entries = [] - for i, prob in enumerate(row): - entries.append({'index': i, - 'name': str(i), - 'prob': prob}) - - entries = sorted(entries, - key=itemgetter('prob'), - reverse=True)[:top] - - for entry in entries: - entry['prob'] = '{:.3f}'.format(entry['prob']) - results.append(entries) - - return results diff --git a/picasso/examples/tensorflow/config.py b/picasso/examples/tensorflow/config.py index 60de7b0..08e8e11 100644 --- a/picasso/examples/tensorflow/config.py +++ b/picasso/examples/tensorflow/config.py @@ -1,14 +1,17 @@ +# Note: By default, Flask doesn't know that this file exists. If you want +# Flask to load the settings you specify here, you must set the environment +# variable `PICASSO_SETTINGS` to point to this file. E.g.: +# +# export PICASSO_SETTINGS=/path/to/examples/tensorflow/config.py +# import os base_dir = os.path.dirname(os.path.abspath(__file__)) -BACKEND_ML = 'tensorflow' -BACKEND_PREPROCESSOR_NAME = 'preprocess' -BACKEND_PREPROCESSOR_PATH = os.path.join(base_dir, 'util.py') -BACKEND_POSTPROCESSOR_NAME = 'postprocess' -BACKEND_POSTPROCESSOR_PATH = os.path.join(base_dir, 'util.py') -BACKEND_PROB_DECODER_NAME = 'prob_decode' -BACKEND_PROB_DECODER_PATH = os.path.join(base_dir, 'util.py') -BACKEND_TF_PREDICT_VAR = 'Softmax:0' -BACKEND_TF_INPUT_VAR = 'convolution2d_input_1:0' -DATA_DIR = os.path.join(base_dir, 'data-volume') +MODEL_CLS_PATH = os.path.join(base_dir, 'model.py') +MODEL_CLS_NAME = 'TensorflowMNISTModel' +MODEL_LOAD_ARGS = { + 'data_dir': os.path.join(base_dir, 'data-volume'), + 'tf_input_var': 'convolution2d_input_1:0', + 'tf_predict_var': 'Softmax:0', +} diff --git a/picasso/examples/tensorflow/model.py b/picasso/examples/tensorflow/model.py new file mode 100644 index 0000000..3c40b0a --- /dev/null +++ b/picasso/examples/tensorflow/model.py @@ -0,0 +1,35 @@ +import numpy as np +from PIL import Image + +from picasso.models.tensorflow import TFModel + +MNIST_DIM = (28, 28) + + +class TensorflowMNISTModel(TFModel): + + def preprocess(self, raw_inputs): + """Convert images into the format required by our model. + + Our model requires that inputs be grayscale (mode 'L'), be resized to + `MNIST_DIM`, and be represented as float32 numpy arrays in range + [0, 1]. + + Args: + raw_inputs (list of Images): a list of PIL Image objects + + Returns: + array (float32): num images * height * width * num channels + + """ + image_arrays = [] + for raw_im in raw_inputs: + im = raw_im.convert('L') + im = im.resize(MNIST_DIM, Image.ANTIALIAS) + arr = np.array(im) + image_arrays.append(arr) + + inputs = np.array(image_arrays) + return inputs.reshape(len(inputs), + MNIST_DIM[0], + MNIST_DIM[1], 1).astype('float32') / 255 diff --git a/picasso/examples/tensorflow/util.py b/picasso/examples/tensorflow/util.py deleted file mode 100644 index 1820184..0000000 --- a/picasso/examples/tensorflow/util.py +++ /dev/null @@ -1,48 +0,0 @@ -from PIL import Image -from operator import itemgetter -import numpy as np - -MNIST_DIM = (28, 28) - - -def preprocess(targets): - image_arrays = [] - for target in targets: - im = target.convert('L') - im = im.resize(MNIST_DIM, Image.ANTIALIAS) - arr = np.array(im) - image_arrays.append(arr) - - all_targets = np.array(image_arrays) - return all_targets.reshape(len(all_targets), - MNIST_DIM[0], - MNIST_DIM[1], 1).astype('float32') / 255 - - -def postprocess(output_arr): - images = [] - for row in output_arr: - im_array = row.reshape(MNIST_DIM) - images.append(im_array) - - return images - - -def prob_decode(probability_array, top=5): - results = [] - for row in probability_array: - entries = [] - for i, prob in enumerate(row): - entries.append({'index': i, - 'name': str(i), - 'prob': prob}) - - entries = sorted(entries, - key=itemgetter('prob'), - reverse=True)[:top] - - for entry in entries: - entry['prob'] = '{:.3f}'.format(entry['prob']) - results.append(entries) - - return results diff --git a/picasso/ml_frameworks/keras/__init__.py b/picasso/ml_frameworks/keras/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/picasso/ml_frameworks/model.py b/picasso/ml_frameworks/model.py deleted file mode 100644 index 4d687b1..0000000 --- a/picasso/ml_frameworks/model.py +++ /dev/null @@ -1,237 +0,0 @@ -import importlib.util -import warnings -from importlib import import_module -from operator import itemgetter - -ML_LIBRARIES = { - 'tensorflow': - 'picasso.ml_frameworks.tensorflow.model.TFModel', - 'keras': - 'picasso.ml_frameworks.keras.model.KerasModel' -} - - -class Model: - """Model class interface. - - All ML frameworks should derive from this class for the purposes of - the visualization. This class loads saved files generated by various - ML frameworks and allows us to extract the graph topology, weights, etc. - - """ - - def __init__(self, - preprocessor_name='preprocess', - preprocessor_path=None, - postprocessor_name='postprocess', - postprocessor_path=None, - prob_decoder_name='prob_decode', - prob_decoder_path=None, - top_probs=5, - **kwargs): - """Attempt to load utilities - - The class constructor attempts to import a preprocessor, postprocessor, - and probability decoder if a path is supplied. - - Args: - preprocessor_name (str, optional): the name of the preprocessing - function. Defaults to 'preprocess'. - preprocessor_path (str, optional): the absolute path to the file - containing the function named above. If `None`, then do not - try to load a preprocessor. Defaults to `None`. - postprocessor_name (str, optional): the name of the postprocessing - function. Defaults to 'postprocess'. - postprocessor_path (str, optional): the absolute path to the file - containing the function named above. If `None`, then do not - try to load a postprocessor. Defaults to `None`. - prob_decoder_name (str, optional): the name of the postprocessing - function. Defaults to 'prob_decode'. - prob_decoder_path (str, optional): the absolute path to the file - containing the function named above. If `None`, then do not - try to load a prob_decoder. Defaults to `None`. - top_probs (int): Number of classes to display per result. For - instance, VGG16 has 1000 classes, we don't want to display a - visualization for every single possibility. Defaults to 5. - **kwargs: Arbitrary keyword arguments, useful for passing specific - settings to derived classes. - - Example: - If you define a function called "preprocess" at "/path/to/util.py", - then try:: - - preprocessor_name='preprocess', - preprocessor_path='/path/to/util.py' - - """ - self.latest_ckpt_name = None - self.latest_ckpt_time = None - self.top_probs = top_probs - - self.preprocessor_name = preprocessor_name - self.preprocessor_path = preprocessor_path - self.postprocessor_name = postprocessor_name - self.postprocessor_path = postprocessor_path - self.prob_decoder_name = prob_decoder_name - self.prob_decoder_path = prob_decoder_path - - for util in ('preprocessor', 'postprocessor', 'prob_decoder'): - if getattr(self, '{}_path'.format(util)): - spec = importlib.util.\ - spec_from_file_location( - getattr(self, '{}_name'.format(util)), - getattr(self, '{}_path'.format(util))) - setattr(self, util, importlib.util.module_from_spec(spec)) - spec.loader.exec_module(getattr(self, util)) - - if kwargs: - for key, value in kwargs.items(): - setattr(self, key, value) - - def load(self, data_dir, **kwargs): - """Load the model in the desired framework - - Given a directory where model data (weights and graph - structure), should be able to restore the model locally to the point - where it can be evaluated. - - Args: - data_dir (:obj:`str`): full path to directory containing - weight and graph data - **kwargs: Arbitrary keyword arguments, useful for passing specific - settings to derived classes. - - """ - raise NotImplementedError - - def _predict(self, targets): - """Evaluate new examples and return class probablilites - - Given an iterable of examples or numpy array where the first - dimension is the number of example, return a n_examples x - n_classes array of class predictions - - Args: - targets: iterable of arrays suitable for input into graph - - Returns: - array of class probabilities - - """ - raise NotImplementedError - - def predict(self, raw_targets): - """Predict from raw data - - Takes an iterable of data in its raw format. Passes to the - preprocessor and then the child class _predict. - - Args: - raw_targets (:obj:`list` of :obj:`PIL.Image`): the images - to be processed - - Returns: - array of class probabilities - - """ - return self._predict(self.preprocess(raw_targets)) - - def preprocess(self, raw_targets): - """Preprocess raw input for evaluation by model - - Usually, input will need some preprocessing before submission - to a computation graph. For instance, the raw image may need - to converted to a numpy array of appropriate dimension - - Args: - raw_targets (:obj:`list` of :obj:`PIL.Image`): the images - to be processed - - Returns: - iterable of arrays of the correct shape for input into graph - - """ - try: - return getattr(self.preprocessor, - self.preprocessor_name)(raw_targets) - except AttributeError: - warnings.warn('Evaluating without preprocessor') - return raw_targets - - def postprocess(self, output_arr): - """Postprocess prediction results back into images - - Sometimes it's useful to display an intermediate computation - as image. This is model-dependent. - - Args: - output_arr (iterable of arrays): any array with the - same total number of entries an input array - - Returns: - iterable of arrays in original image shape - - """ - - try: - return getattr(self.postprocessor, - self.postprocessor_name)(output_arr) - except AttributeError: - warnings.warn('Evaluating without postprocessor') - return output_arr - - def decode_prob(self, output_arr): - """Label class probabilites with class names - - Args: - output_arr (array): class probabilities - - Returns: - result list of dict in the format [{'index': class_index, 'name': - class_name, 'prob': class_probability}, ...] - - """ - - try: - return getattr(self.prob_decoder, - self.prob_decoder_name)(output_arr, - top=self.top_probs) - except AttributeError: - warnings.warn('Evaluating without class decoder') - results = [] - for row in output_arr: - entries = [] - for i, prob in enumerate(row): - entries.append({'index': i, - 'name': str(i), - 'prob': prob}) - - entries = sorted(entries, - key=itemgetter('prob'), - reverse=True)[:self.top_probs] - - for entry in entries: - entry['prob'] = '{:.3f}'.format(entry['prob']) - results.append(entries) - return results - - -def generate_model(backend_ml, **kwargs): - """Create a new instance of ML backend - - Args: - backend_ml (:obj:`str`): name of the backend to use - **kwargs: Arbitrary keyword arguments - - Returns: - An instance of :class:`.ml_frameworks.model.Model` - - """ - module_name, _, class_name = \ - ML_LIBRARIES[backend_ml].rpartition('.') - - cls = getattr(import_module(module_name), class_name) - - kwargs = {k.partition('_')[-1]: - v for (k, v) in kwargs.items()} - return cls(**kwargs) diff --git a/picasso/ml_frameworks/tensorflow/__init__.py b/picasso/ml_frameworks/tensorflow/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/picasso/ml_frameworks/tensorflow/model.py b/picasso/ml_frameworks/tensorflow/model.py deleted file mode 100644 index 97a1c3d..0000000 --- a/picasso/ml_frameworks/tensorflow/model.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import glob -from datetime import datetime - -import tensorflow as tf - -from picasso.ml_frameworks.model import Model - - -class TFModel(Model): - """Implements model loading functions for tensorflow""" - - def load(self, data_dir='./'): - """Load graph and weight data - - Args: - data_dir (:obj:`str`): location of tensorflow checkpoint - data. We'll need the .meta file to reconstruct - the graph and the data (checkpoint) files to - fill in the weights of the model. The default - behavior is take the latest files, by OS timestamp. - - """ - - self.sess = tf.Session() - self.sess.as_default() - # find newest ckpt and meta files - try: - latest_ckpt_fn = max( - filter( - # exclude index and meta files which may have earlier - # timestamps - lambda x: os.path.splitext(x)[-1].startswith('.meta') or - os.path.splitext(x)[-1].startswith('.index'), - glob.glob(os.path.join(data_dir, '*.ckpt*')) - ), - key=os.path.getctime) - self.latest_ckpt_time = str(datetime.fromtimestamp( - os.path.getmtime(latest_ckpt_fn) - )) - # remove any step info that's been appended to the - # extenstion - fileext_div = latest_ckpt_fn.rfind('.ckpt') - additional_ext = latest_ckpt_fn.rfind('.', fileext_div + 1) - if additional_ext < 0: - latest_ckpt = latest_ckpt_fn - else: - latest_ckpt = latest_ckpt_fn[:additional_ext] - except ValueError: - raise FileNotFoundError('No checkpoint (.ckpt) files ' - 'available at {}'.format(data_dir)) - try: - latest_meta = max(glob.iglob(os.path.join(data_dir, '*.meta')), - key=os.path.getctime) - except ValueError: - raise FileNotFoundError('No graph (.meta) files ' - 'available at {}'.format(data_dir)) - - with self.sess.as_default() as sess: - self.saver = tf.train.import_meta_graph(latest_meta) - self.saver.restore(sess, latest_ckpt) - - self.tf_predict_var = \ - self.sess.graph.get_tensor_by_name(self.tf_predict_var) - self.tf_input_var = \ - self.sess.graph.get_tensor_by_name(self.tf_input_var) - - def _predict(self, input_array): - return self.sess.run(self.tf_predict_var, - {self.tf_input_var: input_array}) diff --git a/picasso/ml_frameworks/__init__.py b/picasso/models/__init__.py similarity index 100% rename from picasso/ml_frameworks/__init__.py rename to picasso/models/__init__.py diff --git a/picasso/models/base.py b/picasso/models/base.py new file mode 100644 index 0000000..c4b32d2 --- /dev/null +++ b/picasso/models/base.py @@ -0,0 +1,198 @@ +import importlib +from operator import itemgetter +import warnings + + +def load_model(model_cls_path, model_cls_name, model_load_args): + """Get an instance of the described model. + + Args: + model_cls_path: Path to the module in which the model class + is defined. + model_cls_name: Name of the model class. + model_load_args: Dictionary of args to pass to the `load` method + of the model instance. + + Returns: + An instance of :class:`.models.model.BaseModel` or subclass + + """ + spec = importlib.util.spec_from_file_location('active_model', + model_cls_path) + model_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(model_module) + model_cls = getattr(model_module, model_cls_name) + model = model_cls() + if not isinstance(model, BaseModel): + warnings.warn("Loaded model '%s' at '%s' is not an instance of %r" + % (model_cls_name, model_cls_path, BaseModel)) + model.load(**model_load_args) + return model + + +class BaseModel: + """Interface encapsulating a trained NN model usable for prediction. + + This interface defines: + + - How to load the model's topology and parameters from disk + - How to preprocess a batch of examples for the model + - How to perform prediction using the model + - Etc + + """ + + def __init__(self, + top_probs=5): + """Create a new instance of this model. + + `BaseModel` is an interface and should only be instantiated via a + subclass. + + Args: + top_probs (int): Number of classes to display per result. For + instance, VGG16 has 1000 classes, we don't want to display a + visualization for every single possibility. Defaults to 5. + + """ + self.top_probs = top_probs + + self._sess = None + self._tf_input_var = None + self._tf_predict_var = None + self._model_name = None + self._latest_ckpt_name = None + self._latest_ckpt_time = None + + def load(self, *args, **kwargs): + """Load the model's graph and parameters from disk, restoring the model + into `self._sess` so that it can be run for inference. + + Subclasses should set the instance variables [self._sess, + self._tf_input_var, self._tf_predict_var, self._description] in their + implementation. + + """ + raise NotImplementedError + + @property + def sess(self): + """Tensorflow session that can be used to evaluate tensors in the + model. + + :type: :obj:`tf.Session` + + """ + return self._sess + + @property + def tf_input_var(self): + """Tensorflow tensor that represents the model's inputs. + + :type: :obj:`tf.Tensor` + + """ + return self._tf_input_var + + @property + def tf_predict_var(self): + """Tensorflow tensor that represents the model's predicted class + probabilities. + + :type: :obj:`tf.Tensor` + + """ + return self._tf_predict_var + + @property + def latest_ckpt_time(self): + """Timestamp of the latest checkpoint + + :type: str + + """ + return self._latest_ckpt_time + + @property + def latest_ckpt_name(self): + """Filename of the checkpoint + + :type: str + + """ + return self._latest_ckpt_name + + def preprocess(self, raw_inputs): + """Preprocess raw inputs into the format required by the model. + + E.g, the raw image may need to converted to a numpy array of the + appropriate dimension. + + By default, we perform no preprocessing. + + Args: + raw_inputs (:obj:`list` of :obj:`PIL.Image`): List of raw + input images of any mode and shape. + + Returns: + array (float32): Images ready to be fed into the model. + + """ + return raw_inputs + + def predict(self, inputs): + """Given preprocessed inputs, generate class probabilities by using the + model to perform inference. + + Given an iterable of examples or numpy array where the first + dimension is the number of example, return a n_examples x + n_classes array of class predictions + + Args: + inputs: Iterable of examples (e.g., a numpy array whose first + dimension is the batch size). + + Returns: + Class probabilities for each input example, as a numpy array of + shape (num_examples, num_classes). + + """ + raise NotImplementedError + + def decode_prob(self, class_probabilities): + """Given predicted class probabilites for a set of examples, annotate + each logit with a class name. + + By default, we name each class using its index in the logits array. + + Args: + class_probabilities (array): Class probabilities as output by + `self.predict`, i.e., a numpy array of shape (num_examples, + num_classes). + + Returns: + Annotated class probabilities for each input example, as a list of + dicts where each dict is formatted as: + { + 'index': class_index, + 'name': class_name, + 'prob': class_probability + } + + """ + results = [] + for row in class_probabilities: + entries = [] + for i, prob in enumerate(row): + entries.append({'index': i, + 'name': str(i), + 'prob': prob}) + + entries = sorted(entries, + key=itemgetter('prob'), + reverse=True)[:self.top_probs] + + for entry in entries: + entry['prob'] = '{:.3f}'.format(entry['prob']) + results.append(entries) + return results diff --git a/picasso/ml_frameworks/keras/model.py b/picasso/models/keras.py similarity index 50% rename from picasso/ml_frameworks/keras/model.py rename to picasso/models/keras.py index c454264..b62a5a1 100644 --- a/picasso/ml_frameworks/keras/model.py +++ b/picasso/models/keras.py @@ -1,29 +1,25 @@ -import os +from datetime import datetime import glob import json -from datetime import datetime +import os import keras.backend as K from keras.models import model_from_json, load_model -from picasso.ml_frameworks.tensorflow.model import TFModel +from picasso.models.base import BaseModel + + +class KerasModel(BaseModel): + """Implements model loading functions for Keras. + Using this Keras module will require the h5py library, which is not + included with Keras. -class KerasModel(TFModel): - """Implements model loading functions for Keras - Using this Keras module will require the h5py library, - which is not included with Keras - Attributes: - sess (Tensorflow :obj:`Session`): underlying Tensorflow session of - the Keras model. - tf_predict_var (:obj:`Tensor`): tensorflow tensor which represents - the class probabilities - tf_input_var (:obj:`Tensor`): tensorflow tensor which represents - the inputs """ - def load(self, data_dir='./'): - """Load graph and weight data + def load(self, data_dir): + """Load graph and weight data. + Args: data_dir (:obj:`str`): location of Keras checkpoint (`.hdf5`) files and model (in `.json`) structure. The default behavior @@ -35,13 +31,10 @@ def load(self, data_dir='./'): # find newest ckpt and graph files try: latest_ckpt = max(glob.iglob( - os.path.join(data_dir, '*.h*5')), - key=os.path.getctime) - self.latest_ckpt_name = os.path.basename(latest_ckpt) - self.latest_ckpt_time = str(datetime.fromtimestamp( - os.path.getmtime(latest_ckpt)) - ) - + os.path.join(data_dir, '*.h*5')), key=os.path.getctime) + latest_ckpt_name = os.path.basename(latest_ckpt) + latest_ckpt_time = str( + datetime.fromtimestamp(os.path.getmtime(latest_ckpt))) except ValueError: raise FileNotFoundError('No checkpoint (.hdf5 or .h5) files ' 'available at {}'.format(data_dir)) @@ -50,23 +43,24 @@ def load(self, data_dir='./'): key=os.path.getctime) with open(latest_json, 'r') as f: model_json = json.loads(f.read()) - self.model = model_from_json(model_json) + self._model = model_from_json(model_json) - self.model.load_weights(latest_ckpt) + self._model.load_weights(latest_ckpt) except ValueError: try: - self.model = load_model(latest_ckpt) - + self._model = load_model(latest_ckpt) except ValueError: raise FileNotFoundError('The (.hdf5 or .h5) files available at' '{} don\'t have the model' ' architecture.' .format(latest_ckpt)) - self.sess = K.get_session() - - self.tf_predict_var = self.model.outputs[0] - self.tf_input_var = self.model.inputs[0] + self._sess = K.get_session() + self._tf_predict_var = self._model.outputs[0] + self._tf_input_var = self._model.inputs[0] + self._model_name = type(self).__name__ + self._latest_ckpt_name = latest_ckpt_name + self._latest_ckpt_time = latest_ckpt_time - def _predict(self, input_array): - return self.model.predict(input_array) + def predict(self, input_array): + return self._model.predict(input_array) diff --git a/picasso/models/tensorflow.py b/picasso/models/tensorflow.py new file mode 100644 index 0000000..4be7c71 --- /dev/null +++ b/picasso/models/tensorflow.py @@ -0,0 +1,76 @@ +from datetime import datetime +import glob +import os + +import tensorflow as tf + +from picasso.models.base import BaseModel + + +class TFModel(BaseModel): + """Implements model loading functions for Tensorflow. + + """ + + def load(self, data_dir, tf_input_var=None, tf_predict_var=None): + """Load graph and weight data. + + Args: + data_dir (:obj:`str`): location of tensorflow checkpoint data. + We'll need the .meta file to reconstruct the graph and the data + (checkpoint) files to fill in the weights of the model. The + default behavior is take the latest files, by OS timestamp. + tf_input_var (:obj:`str`): Name of the tensor corresponding to the + model's inputs. You must define this if you are loading the + model from a checkpoint. + tf_predict_var (:obj:`str`): Name of the tensor corresponding to + the model's predictions. You must define this if you are + loading the model from a checkpoint. + + """ + # find newest ckpt and meta files + try: + latest_ckpt_fn = max( + filter( + # exclude index and meta files which may have earlier + # timestamps + lambda x: os.path.splitext(x)[-1].startswith('.meta') or + os.path.splitext(x)[-1].startswith('.index'), + glob.glob(os.path.join(data_dir, '*.ckpt*'))), + key=os.path.getctime) + latest_ckpt_time = str( + datetime.fromtimestamp(os.path.getmtime(latest_ckpt_fn))) + # remove any step info that's been appended to the extension + fileext_div = latest_ckpt_fn.rfind('.ckpt') + additional_ext = latest_ckpt_fn.rfind('.', fileext_div + 1) + if additional_ext < 0: + latest_ckpt = latest_ckpt_fn + else: + latest_ckpt = latest_ckpt_fn[:additional_ext] + except ValueError: + raise FileNotFoundError('No checkpoint (.ckpt) files ' + 'available at {}'.format(data_dir)) + + try: + latest_meta = max(glob.iglob(os.path.join(data_dir, '*.meta')), + key=os.path.getctime) + except ValueError: + raise FileNotFoundError('No graph (.meta) files ' + 'available at {}'.format(data_dir)) + + self._sess = tf.Session() + self._sess.as_default() + + self._saver = tf.train.import_meta_graph(latest_meta) + self._saver.restore(self._sess, latest_ckpt) + + self._tf_input_var = self._sess.graph.get_tensor_by_name(tf_input_var) + self._tf_predict_var = self._sess.graph.get_tensor_by_name( + tf_predict_var) + self._model_name = type(self).__name__ + self._latest_ckpt_name = latest_ckpt_fn + self._latest_ckpt_time = latest_ckpt_time + + def predict(self, input_array): + return self.sess.run(self.tf_predict_var, + {self.tf_input_var: input_array}) diff --git a/picasso/picasso.py b/picasso/picasso.py index bb62518..85f4521 100644 --- a/picasso/picasso.py +++ b/picasso/picasso.py @@ -21,18 +21,18 @@ Attributes: APP_TITLE (:obj:`str`): Name of the application to display in the title bar - VISUALIZATON_CLASSES(:obj:`tuple` of :class:`.BaseVisualization`): + VISUALIZATION_CLASSES (:obj:`tuple` of :class:`.BaseVisualization`): Visualization classes available for rendering. """ -import os -import io -import time +from importlib import import_module import inspect -import shutil +import io +import os from operator import itemgetter +import shutil from tempfile import mkdtemp -from importlib import import_module +import time from types import ModuleType from PIL import Image @@ -46,47 +46,53 @@ ) from werkzeug.utils import secure_filename -from picasso import app from picasso import __version__ -from picasso.ml_frameworks.model import generate_model -from picasso.visualizations import BaseVisualization +from picasso import app +from picasso.models.base import load_model from picasso.visualizations import * +from picasso.visualizations.base import BaseVisualization APP_TITLE = 'Picasso Visualizer' -# import visualizations classes dynamically -visualization_attr = vars( - import_module('picasso.visualizations')) -visualization_submodules = [visualization_attr[x] for x in visualization_attr - if isinstance(visualization_attr[x], ModuleType)] -VISUALIZATON_CLASSES = [] -for submodule in visualization_submodules: - members = vars(submodule) - classes = [members[x] for x in members if inspect.isclass(members[x]) and - issubclass(members[x], BaseVisualization) and - members[x] is not BaseVisualization] - VISUALIZATON_CLASSES += classes - -# Use a bogus secret key for debugging ease. No -# client information is stored, the secret key is only -# necessary for generating the session cookie. + +def _get_visualization_classes(): + """Import visualizations classes dynamically + + """ + visualization_attr = vars( + import_module('picasso.visualizations')) + visualization_submodules = [ + visualization_attr[x] + for x in visualization_attr + if isinstance(visualization_attr[x], ModuleType)] + + visualization_classes = [] + for submodule in visualization_submodules: + attrs = vars(submodule) + for attr_name in attrs: + attr = attrs[attr_name] + if (inspect.isclass(attr) + and issubclass(attr, BaseVisualization) + and attr is not BaseVisualization): + visualization_classes.append(attr) + return visualization_classes + + +VISUALIZATION_CLASSES = _get_visualization_classes() + +# Use a bogus secret key for debugging ease. No client information is stored; +# the secret key is only necessary for generating the session cookie. if app.debug: app.secret_key = '...' else: app.secret_key = os.urandom(24) -# This pattern is used in other projects with Flask and -# tensorflow, but probably isn't the most stable or -# safest way. Would be much better to connect to a -# persistent tensorflow session running in another process or +# This pattern is used in other projects with Flask and Tensorflow, but +# but probably isn't the most stable or safest way. Would be much better to +# connect to a persistent Tensorflow session running in another process or # machine. -ml_backend = \ - generate_model( - **{k.lower(): v for (k, v) - in app.config.items() - if k.startswith('BACKEND')} - ) -ml_backend.load(app.config['DATA_DIR']) +model = load_model(app.config['MODEL_CLS_PATH'], app.config['MODEL_CLS_NAME'], + app.config['MODEL_LOAD_ARGS']) @app.before_request @@ -110,11 +116,23 @@ def initialize_new_session(): session['img_input_dir'] = mkdtemp() session['img_output_dir'] = mkdtemp() -def get_visualizations(): - """Get visualization classes in context - Puts the available visualizations in the request context - and returns them. +def get_model(): + """Get the NN model that's being analyzed from the request context. Put + the model in the request context if it is not yet there. + + Returns: + instance of :class:`.models.model.Model` or derived + class + """ + if not hasattr(g, 'model'): + g.model = model + return g.model + + +def get_visualizations(): + """Get the available visualizations from the request context. Put the + visualizations in the request context if they are not yet there. Returns: :obj:`list` of instances of :class:`.BaseVisualization` or @@ -123,27 +141,13 @@ def get_visualizations(): """ if not hasattr(g, 'visualizations'): g.visualizations = {} - for VisClass in VISUALIZATON_CLASSES: - vis = VisClass(get_ml_backend()) + for VisClass in VISUALIZATION_CLASSES: + vis = VisClass(get_model()) g.visualizations[vis.__class__.__name__] = vis return g.visualizations -def get_ml_backend(): - """Get machine learning backend in context - - Puts the backend in the request context and returns it. - - Returns: - instance of :class:`.ml_frameworks.model.Model` or derived - class - """ - if not hasattr(g, 'ml_backend'): - g.ml_backend = ml_backend - return g.ml_backend - - def get_app_state(): """Get current status of application in context @@ -152,10 +156,10 @@ def get_app_state(): """ if not hasattr(g, 'app_state'): - model = get_ml_backend() + model = get_model() g.app_state = { 'app_title': APP_TITLE, - 'backend': type(model).__name__, + 'model_name': type(model).__name__, 'latest_ckpt_name': model.latest_ckpt_name, 'latest_ckpt_time': model.latest_ckpt_time } @@ -262,29 +266,28 @@ def landing(): if request.method == 'POST': session['vis_name'] = request.form.get('choice') vis = get_visualizations()[session['vis_name']] - if hasattr(vis, 'settings'): + if vis.ALLOWED_SETTINGS: return visualization_settings() return select_files() # otherwise, on GET request visualizations = get_visualizations() vis_desc = [{'name': vis, - 'description': visualizations[vis].description} + 'description': visualizations[vis].DESCRIPTION} for vis in visualizations] session.clear() return render_template('select_visualization.html', app_state=get_app_state(), visualizations=sorted(vis_desc, - key=itemgetter('name')) - ) + key=itemgetter('name'))) @app.route('/visualization_settings', methods=['POST']) def visualization_settings(): """Visualization settings page - Will only render if the visualization object has a `settings` - attribute. + Will only render if the visualization object has a non-null + `ALLOWED_SETTINGS` attribute. """ if request.method == 'POST': @@ -292,7 +295,7 @@ def visualization_settings(): return render_template('settings.html', app_state=get_app_state(), current_vis=session['vis_name'], - settings=vis.settings) + settings=vis.ALLOWED_SETTINGS) @app.route('/select_files', methods=['GET', 'POST']) @@ -341,8 +344,8 @@ def select_files(): entry['data'].save(path, 'PNG') kwargs = {} - if hasattr(vis, 'reference_link'): - kwargs.update({'reference_link': vis.reference_link}) + if vis.REFERENCE_LINK: + kwargs['reference_link'] = vis.REFERENCE_LINK return render_template('{}.html'.format(session['vis_name']), inputs=inputs, diff --git a/picasso/settings.py b/picasso/settings.py deleted file mode 100644 index d469651..0000000 --- a/picasso/settings.py +++ /dev/null @@ -1,44 +0,0 @@ -import os - -base_dir = os.path.dirname(__file__) # only for default config - - -class Default: - """Default configuration settings - - The app will use these settings if none are specified. That is, - if no configuration file is specified by PICASSO_SETTINGS - or any individual setting is specified by environment variable. - These are, in effect, "settings of last resort." - - The paths will automatically be generated based on the location of - the source. - """ - - #: :obj:`str`: which backend to use - BACKEND_ML = 'keras' - - #: :obj:`str`: name of the preprocess function - BACKEND_PREPROCESSOR_NAME = 'preprocess' - - #: :obj:`str`: filepath of the preprocess function - BACKEND_PREPROCESSOR_PATH = os.path.join( - base_dir, 'examples', 'keras', 'util.py') - - #: :obj:`str`: name of the postprocess function - BACKEND_POSTPROCESSOR_NAME = 'postprocess' - - #: :obj:`str`: filepath of the postprocess function - BACKEND_POSTPROCESSOR_PATH = os.path.join( - base_dir, 'examples', 'keras', 'util.py') - - #: :obj:`str`: name of the probability decoder function - BACKEND_PROB_DECODER_NAME = 'prob_decode' - - #: :obj:`str`: filepath of the probability decoder function - BACKEND_PROB_DECODER_PATH = os.path.join( - base_dir, 'examples', 'keras', 'util.py') - - #: :obj:`str`: path to directory containing weights and graph - DATA_DIR = os.path.join( - base_dir, 'examples', 'keras', 'data-volume') diff --git a/picasso/templates/layout.html b/picasso/templates/layout.html index cd30161..a6444ab 100644 --- a/picasso/templates/layout.html +++ b/picasso/templates/layout.html @@ -17,7 +17,7 @@

{{ app_state.app_title }}

-

Current backend: {{ app_state.backend }}

+

Current backend: {{ app_state.model_name }}

{% if app_state.latest_ckpt_name is defined %}

Current checkpoint: {{ app_state.latest_ckpt_name }}

{% endif %} diff --git a/picasso/visualizations/__init__.py b/picasso/visualizations/__init__.py index 0146dd3..af60df9 100644 --- a/picasso/visualizations/__init__.py +++ b/picasso/visualizations/__init__.py @@ -6,37 +6,6 @@ """ import os + __all__ = [x.rpartition('.')[0] for x in os.listdir(__path__[0]) if not x.startswith('__') and x.endswith('py')] - - -class BaseVisualization: - """Template for visualizations - - Attributes: - description (:obj:`str`): short description of the visualization - model (instance of :class:`.ml_frameworks.model.Model` or derived class): - backend to use - settings (:obj:`dict`): a settings dictionary. Settings defined - here will be rendered in html for the user to select. See - derived classes for examples. - """ - def __init__(self, model): - self.model = model - - def make_visualization(self, inputs, output_dir, settings=None): - """Generate the visualization - - All visualizations must implement this method. - - Args: - inputs (iterable of :class:`PIL.Image`): images uploaded by the - user. Will have already been converted to :obj:`Image` - objects. - output_dir (:obj:`str`): a directory to store outputs (e.g. plots) - - Returns: - data needed to render the visualization. Since there is an - associated HTML template, the return type is arbitrary. - """ - raise NotImplementedError diff --git a/picasso/visualizations/base.py b/picasso/visualizations/base.py new file mode 100644 index 0000000..34d0114 --- /dev/null +++ b/picasso/visualizations/base.py @@ -0,0 +1,62 @@ +class BaseVisualization: + """Interface encapsulating a NN visualization. + + This interface defines how a visualization is computed for a given NN + model. + + """ + # (:obj:`str`): Short description of the visualization. + DESCRIPTION = None + + # (:obj:`str`): Optional link to the paper specifying the visualization. + REFERENCE_LINK = None + + # (:obj:`dict`): Optional visualization settings that the user can select, + # as a dict mapping setting names to lists of their allowed values. + ALLOWED_SETTINGS = None + + def __init__(self, model): + """Create a new instance of this visualization. + + `BaseVisualization` is an interface and should only be instantiated via + a subclass. + + Args: + model (:obj:`.models.model.BaseModel`): NN model to be + visualized. + + """ + self._model = model + + @property + def model(self): + """NN model to be visualized. + + (:obj:`.models.model.BaseModel`) + + """ + return self._model + + def make_visualization(self, inputs, output_dir, settings=None): + """Generate the visualization. + + All visualizations must implement this method. + + Args: + inputs (iterable of :class:`PIL.Image`): Batch of input images to + make visualizations for, as PIL :obj:`Image` objects. + output_dir (:obj:`str`): A directory to write outputs (e.g., + plots) to. + settings (:obj:`str`): Dictionary of settings that the user + selected, as a dict mapping setting names to values. This + should only be provided if this class's `ALLOWED_SETTINGS` + attribute is non-null. + + Returns: + Object used to render the visualization, passed directly to the + visualization class's associated HTML template. Since this HTML + template is custom for each visualization class, the return type + is arbitrary. + + """ + raise NotImplementedError diff --git a/picasso/visualizations/class_probabilities.py b/picasso/visualizations/class_probabilities.py index f87d5f3..b1299ad 100644 --- a/picasso/visualizations/class_probabilities.py +++ b/picasso/visualizations/class_probabilities.py @@ -1,4 +1,4 @@ -from picasso.visualizations import BaseVisualization +from picasso.visualizations.base import BaseVisualization class ClassProbabilities(BaseVisualization): @@ -9,12 +9,11 @@ class probabilities of the input image. """ - description = 'Predict class probabilities from new examples' + DESCRIPTION = 'Predict class probabilities from new examples' - def make_visualization(self, inputs, - output_dir, settings=None): + def make_visualization(self, inputs, output_dir, settings=None): pre_processed_arrays = self.model.preprocess([example['data'] - for example in inputs]) + for example in inputs]) predictions = self.model.sess.run(self.model.tf_predict_var, feed_dict={self.model.tf_input_var: pre_processed_arrays}) diff --git a/picasso/visualizations/partial_occlusion.py b/picasso/visualizations/partial_occlusion.py index 239b4d6..9a79e16 100644 --- a/picasso/visualizations/partial_occlusion.py +++ b/picasso/visualizations/partial_occlusion.py @@ -8,7 +8,7 @@ matplotlib.use('Agg') from matplotlib import pyplot -from picasso.visualizations import BaseVisualization +from picasso.visualizations.base import BaseVisualization class PartialOcclusion(BaseVisualization): @@ -22,16 +22,17 @@ class PartialOcclusion(BaseVisualization): classifying on the image feature we expect. """ - settings = { + DESCRIPTION = ('Partially occlude image to determine regions ' + 'important to classification') + + REFERENCE_LINK = 'https://arxiv.org/abs/1311.2901' + + ALLOWED_SETTINGS = { 'Window': ['0.50', '0.40', '0.30', '0.20', '0.10', '0.05'], 'Strides': ['2', '5', '10', '20', '30'], 'Occlusion': ['grey', 'black', 'white'] } - description = ('Partially occlude image to determine regions ' - 'important to classification') - reference_link = 'https://arxiv.org/abs/1311.2901' - def __init__(self, model): super(PartialOcclusion, self).__init__(model) self.predict_tensor = self.get_predict_tensor() @@ -53,11 +54,10 @@ def make_visualization(self, inputs, output_dir, settings=None): # get class predictions as in ClassProbabilities pre_processed_arrays = self.model.preprocess([example['data'] - for example in inputs]) - class_predictions = \ - self.model.sess.run(self.model.tf_predict_var, - feed_dict={self.model.tf_input_var: - pre_processed_arrays}) + for example in inputs]) + class_predictions = self.model.sess.run( + self.model.tf_predict_var, + feed_dict={self.model.tf_input_var: pre_processed_arrays}) decoded_predictions = self.model.decode_prob(class_predictions) results = [] @@ -86,11 +86,9 @@ def make_visualization(self, inputs, output_dir, settings=None): os.path.join(output_dir, example_filename), format=im_format) - filenames = \ - self.make_heatmaps(predictions, - output_dir, - example['filename'], - decoded_predictions=decoded_predictions[i]) + filenames = self.make_heatmaps( + predictions, output_dir, example['filename'], + decoded_predictions=decoded_predictions[i]) results.append({'input_filename': example['filename'], 'result_filenames': filenames, 'predict_probs': decoded_predictions[i], @@ -100,8 +98,8 @@ def make_visualization(self, inputs, output_dir, settings=None): def get_predict_tensor(self): # Assume that predict is the softmax # tensor in the computation graph - return self.model.sess.graph. \ - get_tensor_by_name(self.model.tf_predict_var.name) + return self.model.sess.graph.get_tensor_by_name( + self.model.tf_predict_var.name) def update_settings(self, settings): def error_string(setting, setting_val): @@ -112,19 +110,19 @@ def error_string(setting, setting_val): vis=self.__class__.__name__) if 'Window' in settings: - if settings['Window'] in self.settings['Window']: + if settings['Window'] in self.ALLOWED_SETTINGS['Window']: self.window = float(settings['Window']) else: raise ValueError(error_string(settings['Window'], 'Window')) if 'Strides' in settings: - if settings['Strides'] in self.settings['Strides']: + if settings['Strides'] in self.ALLOWED_SETTINGS['Strides']: self.num_windows = int(settings['Strides']) else: raise ValueError(error_string(settings['Strides'], 'Strides')) if 'Occlusion' in settings: - if settings['Occlusion'] in self.settings['Occlusion']: + if settings['Occlusion'] in self.ALLOWED_SETTINGS['Occlusion']: self.occlusion_method = settings['Occlusion'] else: raise ValueError(error_string(settings['Occlusion'], @@ -166,12 +164,9 @@ def occluded_images(self, im): win_length = round(self.window * length) pad_horizontal = win_width // 2 pad_vertical = win_length // 2 - centers_horizontal, centers_vertical = \ - self.get_centers(width, length, - win_width, win_length, - pad_horizontal, pad_vertical, - self.num_windows - ) + centers_horizontal, centers_vertical = self.get_centers( + width, length, win_width, win_length, pad_horizontal, pad_vertical, + self.num_windows) upper_left_corners = np.array( [(w - pad_vertical, v - pad_horizontal) for w in centers_vertical diff --git a/picasso/visualizations/saliency_maps.py b/picasso/visualizations/saliency_maps.py index 060df75..87bd078 100644 --- a/picasso/visualizations/saliency_maps.py +++ b/picasso/visualizations/saliency_maps.py @@ -8,7 +8,7 @@ matplotlib.use('Agg') from matplotlib import pyplot -from picasso.visualizations import BaseVisualization +from picasso.visualizations.base import BaseVisualization class SaliencyMaps(BaseVisualization): @@ -21,24 +21,27 @@ class SaliencyMaps(BaseVisualization): classification (as changing them would change the classification). """ - description = ('See maximal derivates against class with respect ' + DESCRIPTION = ('See maximal derivates against class with respect ' 'to input') - reference_link = 'https://arxiv.org/pdf/1312.6034' + + REFERENCE_LINK = 'https://arxiv.org/pdf/1312.6034' def __init__(self, model, logit_tensor_name=None): super(SaliencyMaps, self).__init__(model) if logit_tensor_name: - self.logit_tensor = self.model.sess.graph \ - .get_tensor_by_name(logit_tensor_name) + self.logit_tensor = self.model.sess.graph.get_tensor_by_name( + logit_tensor_name) else: self.logit_tensor = self.get_logit_tensor() + self.input_shape = self.model.tf_input_var.get_shape()[1:].as_list() + def get_gradient_wrt_class(self, class_index): - gradient_name = 'bv_{class_index}_gradient' \ - .format(class_index=class_index) + gradient_name = 'bv_{class_index}_gradient'.format( + class_index=class_index) try: - return self.model.sess.graph. \ - get_tensor_by_name('{}:0'.format(gradient_name)) + return self.model.sess.graph.get_tensor_by_name( + '{}:0'.format(gradient_name)) except KeyError: class_logit = tf.slice(self.logit_tensor, [0, class_index], @@ -61,24 +64,29 @@ def make_visualization(self, inputs, output_dir, settings=None): results = [] for i, inp in enumerate(inputs): class_gradients = [] - output_images = [] relevant_class_indices = [pred['index'] for pred in decoded_predictions[i]] - gradients_wrt_class = [self.get_gradient_wrt_class(index) for index - in relevant_class_indices] + gradients_wrt_class = [self.get_gradient_wrt_class(index) + for index in relevant_class_indices] for gradient_wrt_class in gradients_wrt_class: class_gradients.append([self.model.sess.run( gradient_wrt_class, feed_dict={self.model.tf_input_var: [arr]}) for arr in pre_processed_arrays]) - output_fns = [] - output_arrays = np.array([gradient[i] for - gradient in class_gradients]) + + output_arrays = np.array([gradient[i] + for gradient in class_gradients]) # if images are color, take the maximum channel if output_arrays.shape[-1] == 3: output_arrays = output_arrays.max(-1) + # we care about the size of the derivative, not the sign + output_arrays = np.abs(output_arrays) + + # We want each array to be represented as a 1-channel image of + # the same size as the model's input image. + output_images = output_arrays.reshape([-1] + self.input_shape[0:2]) - output_images = self.model.postprocess(np.abs(output_arrays)) + output_fns = [] for j, image in enumerate(output_images): output_fn = '{fn}-{j}-{ts}.png'.format(ts=str(time.time()), j=j, @@ -105,9 +113,9 @@ def make_visualization(self, inputs, output_dir, settings=None): def get_logit_tensor(self): # Assume that the logits are the tensor input to the last softmax # operation in the computation graph - sm = [node for node in self.model.sess.graph_def.node - if node.name == - self.model.tf_predict_var.name.split(':')[0]][-1] + sm = [node + for node in self.model.sess.graph_def.node + if node.name == self.model.tf_predict_var.name.split(':')[0]][-1] logit_op_name = sm.input[0] - return self.model.sess.graph. \ - get_tensor_by_name('{}:0'.format(logit_op_name)) + return self.model.sess.graph.get_tensor_by_name( + '{}:0'.format(logit_op_name)) diff --git a/tests/conftest.py b/tests/conftest.py index 777ac6f..f35d84d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,5 +27,14 @@ def example_prob_array(): @pytest.fixture def base_model(): - from picasso.ml_frameworks.model import Model - return Model() + from picasso.models.base import BaseModel + class BaseModelForTest(BaseModel): + def load(self, data_dir): + pass + return BaseModelForTest() + + +@pytest.fixture +def tensorflow_model(): + from picasso.models.tensorflow import TFModel + return TFModel() diff --git a/tests/test_picasso.py b/tests/test_picasso.py index 9f668b0..8bcfae3 100644 --- a/tests/test_picasso.py +++ b/tests/test_picasso.py @@ -7,9 +7,9 @@ Tests for `picasso` module. """ -import os import io import json +import os from flask import url_for import pytest @@ -17,18 +17,18 @@ class TestWebApp: - from picasso.picasso import VISUALIZATON_CLASSES + from picasso.picasso import VISUALIZATION_CLASSES def test_landing_page_get(self, client): assert client.get(url_for('landing')).status_code == 200 - @pytest.mark.parametrize("vis", VISUALIZATON_CLASSES) + @pytest.mark.parametrize("vis", VISUALIZATION_CLASSES) def test_landing_page_post(self, client, vis): rv = client.post(url_for('landing'), data=dict(choice=vis.__name__)) assert rv.status_code == 200 - @pytest.mark.parametrize("vis", VISUALIZATON_CLASSES) + @pytest.mark.parametrize("vis", VISUALIZATION_CLASSES) def test_settings_page(self, client, vis): if hasattr(vis, 'settings'): with client.session_transaction() as sess: @@ -36,14 +36,14 @@ def test_settings_page(self, client, vis): rv = client.post(url_for('visualization_settings')) assert rv.status_code == 200 - @pytest.mark.parametrize("vis", VISUALIZATON_CLASSES) + @pytest.mark.parametrize("vis", VISUALIZATION_CLASSES) def test_file_selection_get(self, client, vis): with client.session_transaction() as sess: sess['vis_name'] = vis.__name__ rv = client.get(url_for('select_files')) assert rv.status_code == 200 - @pytest.mark.parametrize("vis", VISUALIZATON_CLASSES) + @pytest.mark.parametrize("vis", VISUALIZATION_CLASSES) def test_file_selection_post(self, client, vis, random_image_files): with client.session_transaction() as sess: sess['vis_name'] = vis.__name__ @@ -66,7 +66,7 @@ def test_file_selection_post(self, client, vis, random_image_files): class TestRestAPI: - from picasso.picasso import VISUALIZATON_CLASSES + from picasso.picasso import VISUALIZATION_CLASSES def test_api_root_get(self, client): assert client.get(url_for('api_root')).status_code == 200 @@ -84,7 +84,7 @@ def test_api_uploading_file(self, client, random_image_files): assert type(data['file']) is str assert type(data['uid']) is int - @pytest.mark.parametrize("vis", VISUALIZATON_CLASSES) + @pytest.mark.parametrize("vis", VISUALIZATION_CLASSES) def test_api_visualizing_input(self, client, random_image_files, vis): upload_file = str(random_image_files.listdir()[0]) with open(upload_file, "rb") as imageFile: @@ -124,7 +124,7 @@ class TestKerasModel: def test_saved_model(self): # tests that KerasModel can load from a saved model import tempfile - from picasso.ml_frameworks.keras.model import KerasModel + from picasso.models.keras import KerasModel data_path = os.path.join('picasso', 'examples', 'keras', 'data-volume') @@ -133,7 +133,7 @@ def test_saved_model(self): km.load(data_path) temp = tempfile.mkdtemp() - km.model.save(os.path.join(temp, 'temp.h5')) + km._model.save(os.path.join(temp, 'temp.h5')) km = KerasModel() km.load(temp) @@ -143,16 +143,14 @@ def test_saved_model(self): class TestTensorflowBackend: - def test_tensorflow_backend(self, client, monkeypatch): + def test_tensorflow_backend(self, tensorflow_model): """Only tests tensorflow backend loads without error """ - - from picasso.ml_frameworks.tensorflow.model import TFModel - data_path = os.path.join('picasso', 'examples', - 'tensorflow', 'data-volume') - tfm = TFModel(tf_predict_var='Softmax:0', - tf_input_var='convolution2d_input_1:0') - tfm.load(data_path) - assert tfm.tf_predict_var is not None - assert tfm.tf_input_var is not None + tensorflow_model.load( + data_dir=os.path.join('picasso', 'examples', 'tensorflow', + 'data-volume'), + tf_predict_var='Softmax:0', + tf_input_var='convolution2d_input_1:0') + assert tensorflow_model.tf_predict_var is not None + assert tensorflow_model.tf_input_var is not None