Commit 7cae7df7 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

First commit of robust models

parent 53e88059

deepchem/models/tensorflow_models/__init__.py

+3 −2
Original line number Diff line number Diff line
@@ -110,8 +110,8 @@ class TensorflowGraphModel(object): 


  def __init__(self, n_tasks, n_features, logdir, layer_sizes=[1000],

               weight_init_stddevs=[.02], bias_init_consts=[1.], penalty=0.0,

               dropouts=[0.5], learning_rate=.001, momentum=".9",

               optimizer="adam", batch_size=50, n_classes=2,

               penalty_type="l2", dropouts=[0.5], learning_rate=.001,

               momentum=".9", optimizer="adam", batch_size=50, n_classes=2,

               train=True, verbosity=None, **kwargs):

    """Constructs the computational graph.
@@ -130,6 +130,7 @@ class TensorflowGraphModel(object): 
    self.weight_init_stddevs = weight_init_stddevs

    self.bias_init_consts = bias_init_consts

    self.penalty = penalty

    self.penalty_type = penalty_type

    self.dropouts = dropouts

    self.learning_rate = learning_rate

    self.momentum = momentum

deepchem/models/tensorflow_models/robust_multitask.py

+44 −32
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ from __future__ import unicode_literals 
import numpy as np

import tensorflow as tf



from deepchem.models.tensorflow_models import TensorflowGraph

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskRegressor

from deepchem.models.tensorflow_models import model_ops
@@ -15,35 +16,44 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
  Key idea is to have bypass layers that feed directly from features to task

  output. Hopefully will allow tasks to route around bad multitasking.

  """



  def build(self):

  def __init__(self, n_tasks, n_features, logdir,

               bypass_layer_sizes=[100],

               bypass_weight_init_stddevs=[.02],

               bypass_bias_init_consts=[1.],

               bypass_dropouts=[.5], **kwargs):

    self.bypass_layer_sizes = bypass_layer_sizes

    self.bypass_weight_init_stddevs = bypass_weight_init_stddevs

    self.bypass_bias_init_consts = bypass_bias_init_consts

    self.bypass_dropouts = bypass_dropouts

    super(RobustMultitaskRegressor, self).__init__(

        n_tasks, n_features, logdir, **kwargs)



  def build(self, graph, name_scopes, training):

    """Constructs the graph architecture as specified in its config.



    This method creates the following Placeholders:

      mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape

        batch_size x num_features.

    """

    ############################################################### DEBUG

    print("ENTERING BUILD!")

    ############################################################### DEBUG

    assert len(self.model_params["data_shape"]) == 1

    num_features = self.model_params["data_shape"][0]

    with self.graph.as_default():

      with tf.name_scope(self.placeholder_scope):

    num_features = self.n_features 

    placeholder_scope = TensorflowGraph.get_placeholder_scope(

        graph, name_scopes)

    with graph.as_default():

      with placeholder_scope:

        self.mol_features = tf.placeholder(

            tf.float32,

            shape=[None, num_features],

            name='mol_features')



      layer_sizes = self.model_params["layer_sizes"]

      weight_init_stddevs = self.model_params["weight_init_stddevs"]

      bias_init_consts = self.model_params["bias_init_consts"]

      dropouts = self.model_params["dropouts"]

      layer_sizes = self.layer_sizes

      weight_init_stddevs = self.weight_init_stddevs

      bias_init_consts = self.bias_init_consts

      dropouts = self.dropouts



      bypass_layer_sizes = self.model_params["bypass_layer_sizes"]

      bypass_weight_init_stddevs = self.model_params["bypass_weight_init_stddevs"]

      bypass_bias_init_consts = self.model_params["bypass_bias_init_consts"]

      bypass_dropouts = self.model_params["bypass_dropouts"]

      bypass_layer_sizes = self.bypass_layer_sizes

      bypass_weight_init_stddevs = self.bypass_weight_init_stddevs

      bypass_bias_init_consts = self.bypass_bias_init_consts

      bypass_dropouts = self.bypass_dropouts



      lengths_set = {

          len(layer_sizes),
@@ -51,9 +61,9 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
          len(bias_init_consts),

          len(dropouts),

          }

      assert len(lengths_set) == 1, 'All layer params must have same length.'

      assert len(lengths_set) == 1, "All layer params must have same length."

      num_layers = lengths_set.pop()

      assert num_layers > 0, 'Must have some layers defined.'

      assert num_layers > 0, "Must have some layers defined."



      bypass_lengths_set = {

          len(bypass_layer_sizes),
@@ -61,7 +71,8 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
          len(bypass_bias_init_consts),

          len(bypass_dropouts),

          }

      assert len(bypass_lengths_set) == 1, 'All bypass_layer params must have same length.'

      assert (len(bypass_lengths_set) == 1,

              "All bypass_layer params must have same length.")

      num_bypass_layers = bypass_lengths_set.pop()



      prev_layer = self.mol_features
@@ -71,7 +82,7 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
        ########################################################## DEBUG

        print("Adding weights of shape %s" % str([prev_layer_size, layer_sizes[i]]))

        ########################################################## DEBUG

        layer = tf.nn.relu(model_ops.FullyConnectedLayer(

        layer = tf.nn.relu(model_ops.fully_connected_layer(

            tensor=prev_layer,

            size=layer_sizes[i],

            weight_init=tf.truncated_normal(
@@ -79,14 +90,14 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
                stddev=weight_init_stddevs[i]),

            bias_init=tf.constant(value=bias_init_consts[i],

                                  shape=[layer_sizes[i]])))

        layer = model_ops.Dropout(layer, dropouts[i])

        layer = model_ops.dropout(layer, dropouts[i], training)

        prev_layer = layer

        prev_layer_size = layer_sizes[i]



      self.output = []

      output = []

      # top_multitask_layer has shape [None, layer_sizes[-1]]

      top_multitask_layer = prev_layer

      for task in range(self.num_tasks):

      for task in range(self.n_tasks):

        # TODO(rbharath): Might want to make it feasible to have multiple

        # bypass layers.

        # Construct task bypass layer
@@ -98,7 +109,7 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
          print("Adding bypass weights of shape %s"

                % str([prev_bypass_layer_size, bypass_layer_sizes[i]]))

          ########################################################## DEBUG

          bypass_layer = tf.nn.relu(model_ops.FullyConnectedLayer(

          bypass_layer = tf.nn.relu(model_ops.fully_connected_layer(

            tensor = prev_bypass_layer,

            size = bypass_layer_sizes[i],

            weight_init=tf.truncated_normal(
@@ -107,7 +118,7 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
            bias_init=tf.constant(value=bypass_bias_init_consts[i],

                                  shape=[bypass_layer_sizes[i]])))

    

          bypass_layer = model_ops.Dropout(bypass_layer, bypass_dropouts[i])

          bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i])

          prev_bypass_layer = bypass_layer

          prev_bypass_layer_size = bypass_layer_sizes[i]

        top_bypass_layer = prev_bypass_layer
@@ -127,8 +138,8 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
        print("task_layer_size")

        print(task_layer_size)

        #################################################### DEBUG

        self.output.append(tf.squeeze(

            model_ops.FullyConnectedLayer(

        output.append(tf.squeeze(

            model_ops.fully_connected_layer(

                tensor=task_layer,

                size=task_layer_size,

                weight_init=tf.truncated_normal(
@@ -136,6 +147,7 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
                    stddev=weight_init_stddevs[-1]),

                bias_init=tf.constant(value=bias_init_consts[-1],

                                      shape=[1]))))

      return output



  def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):

    """Construct a feed dictionary from minibatch data.
@@ -150,18 +162,18 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
    """ 

    orig_dict = {}

    orig_dict["mol_features"] = X_b

    for task in xrange(self.num_tasks):

    for task in xrange(self.n_tasks):

      if y_b is not None:

        orig_dict["labels_%d" % task] = y_b[:, task]

      else:

        # Dummy placeholders

        orig_dict["labels_%d" % task] = np.squeeze(

            np.zeros((self.model_params["batch_size"],)))

            np.zeros((self.batch_size,)))

      if w_b is not None:

        orig_dict["weights_%d" % task] = w_b[:, task]

      else:

        # Dummy placeholders

        orig_dict["weights_%d" % task] = np.ones(

            (self.model_params["batch_size"],)) 

    return self._get_feed_dict(orig_dict)

            (self.batch_size,)) 

    return TensorflowGraph.get_feed_dict(orig_dict)

deepchem/models/tests/test_overfit.py

+41 −16
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ from deepchem.models.keras_models.fcnet import MultiTaskDNN 
from deepchem.models.tensorflow_models import TensorflowModel

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskRegressor

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

from deepchem.models.tensorflow_models.robust_multitask import RobustMultitaskRegressor

from deepchem.models.multitask import SingletaskToMultitask

import tensorflow as tf

from keras import backend as K
@@ -580,22 +581,6 @@ class TestOverfitAPI(TestAPI): 
  

    dataset = NumpyDataset(X, y, w, ids)



    model_params = {

      "layer_sizes": [1000],

      "dropouts": [.0],

      "learning_rate": 0.0003,

      "momentum": .9,

      "batch_size": n_samples,

      "num_regression_tasks": n_tasks,

      "num_classes": n_classes,

      "num_features": n_features,

      "weight_init_stddevs": [.1],

      "bias_init_consts": [1.],

      "nb_epoch": 100,

      "penalty": 0.0,

      "optimizer": "adam"

    }



    verbosity = "high"

    regression_metric = Metric(metrics.mean_squared_error, verbosity=verbosity,

                               task_averager=np.mean, mode="regression")
@@ -615,3 +600,43 @@ class TestOverfitAPI(TestAPI): 
    scores = evaluator.compute_model_performance([regression_metric])



    assert scores[regression_metric.name] < .1



  def test_tf_robust_multitask_regression_overfit(self):

    """Test tf robust multitask overfits tiny data."""

    n_tasks = 10

    n_samples = 10

    n_features = 3

    n_classes = 2

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    dataset = NumpyDataset(X, y, w, ids)



    verbosity = "high"

    regression_metric = Metric(metrics.mean_squared_error, verbosity=verbosity,

                               task_averager=np.mean, mode="regression")

    tensorflow_model = RobustMultitaskRegressor(

        n_tasks, n_features, self.model_dir, layer_sizes=[100], dropouts=[0.],

        learning_rate=0.003, weight_init_stddevs=[.1],

        batch_size=n_samples, verbosity=verbosity)

    model = TensorflowModel(tensorflow_model, self.model_dir)



    # Fit trained model

    model.fit(dataset, nb_epoch=75)

    model.save()



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([regression_metric])

    ############################################### DEBUG

    print("scores[regression_metric.name]")

    print(scores[regression_metric.name])

    ############################################### DEBUG



    assert scores[regression_metric.name] < .5

CRA Git | Maintained and supported by SUSTech CRA and CCSE