Commit 89f3411e authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #287 from miaecle/Benchmark2 

Add n_tasks and n_samples
parents a3e0fb2a fd19aa77

README.md

+17 −17
Original line number Diff line number Diff line
@@ -203,23 +203,23 @@ passed a ``Featurizer`` object. DeepChem provides a number of 
different subclasses of ``Featurizer`` for convenience:



### Performances

|Dataset    |Model               |Train score/ROC-AUC|Valid score/ROC-AUC|Time(loading)/s |Time(running)/s|

|-----------|--------------------|-------------------|-------------------|----------------|---------------| 

|tox21      |logistic regression |0.910              |0.759              |30              |30             |

|           |tensorflow(MT-NN)   |0.987              |0.800              |30              |30             |

|           |graph convolution   |0.930              |0.819              |40              |40             |

|muv        |logistic regression |0.910              |0.744              |600             |800            |

|           |tensorflow(MT-NN)   |0.980              |0.710              |600             |800            |

|           |graph convolution   |0.881              |0.832              |800             |1200           |

|pcba       |logistic regression |0.759        	     |0.736              |1800            |5400           |                                         

|           |tensorflow(MT-NN)	 |0.949        	     |0.791              |1800            |7200           |                                         

|           |graph convolution   |0.866        	     |0.836              |2200            |20000          |                                         

|sider      |logistic regression |0.900        	     |0.620              |15              |40             |                                         

|           |tensorflow(MT-NN)	 |0.931        	     |0.647              |15              |60             |                                         

|           |graph convolution   |0.845        	     |0.646              |20              |60             |                                         

|toxcast    |logistic regression |0.762        	     |0.622              |80              |2000           |                                         

|           |tensorflow(MT-NN)	 |0.926        	     |0.705              |80              |2400           |                                         

|           |graph convolution   |0.906        	     |0.725              |80              |3000           |                                         

|Dataset    |N(tasks)	|N(samples) |Model               |Train score/ROC-AUC|Valid score/ROC-AUC|Time(loading)/s |Time(running)/s|

|-----------|-----------|-----------|--------------------|-------------------|-------------------|----------------|---------------| 

|tox21      |12         |8014       |logistic regression |0.910              |0.759              |30              |30             |

|           |           |           |tensorflow(MT-NN)   |0.987              |0.800              |30              |30             |

|           |           |           |graph convolution   |0.930              |0.819              |40              |40             |

|muv        |17         |93127      |logistic regression |0.910              |0.744              |600             |800            |

|           |           |           |tensorflow(MT-NN)   |0.980              |0.710              |600             |800            |

|           |           |           |graph convolution   |0.881              |0.832              |800             |1200           |

|pcba       |128        |439863     |logistic regression |0.794        	     |0.762              |1800            |15000          |                                         

|           |           |           |tensorflow(MT-NN)	 |0.949        	     |0.791              |1800            |15000          |                                         

|           |           |           |graph convolution   |0.866        	     |0.836              |2200            |20000          |                                         

|sider      |27         |1427       |logistic regression |0.900        	     |0.620              |15              |40             |                                         

|           |           |           |tensorflow(MT-NN)	 |0.931        	     |0.647              |15              |60             |                                         

|           |           |           |graph convolution   |0.845        	     |0.646              |20              |60             |                                         

|toxcast    |617        |8615       |logistic regression |0.762        	     |0.622              |80              |2000           |                                         

|           |           |           |tensorflow(MT-NN)	 |0.926        	     |0.705              |80              |2400           |                                         

|           |           |           |graph convolution   |0.906        	     |0.725              |80              |3000           |                                         





## Contributing to DeepChem

deepchem/models/__init__.py

+1 −0
Original line number Diff line number Diff line
@@ -20,4 +20,5 @@ from deepchem.models.keras_models.fcnet import MultiTaskDNN 
from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskRegressor

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

from deepchem.models.tensorflow_models.robust_multitask import RobustMultitaskRegressor

from deepchem.models.tensorflow_models.robust_multitask import RobustMultitaskClassifier

from deepchem.models.tensorflow_models.lr import TensorflowLogisticRegression
 
 No newline at end of file

deepchem/models/tensorflow_models/robust_multitask.py

+140 −32
Original line number Diff line number Diff line
@@ -6,11 +6,11 @@ import numpy as np 
import tensorflow as tf



from deepchem.models.tensorflow_models import TensorflowGraph

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskRegressor

from deepchem.models.tensorflow_models import model_ops





class RobustMultitaskRegressor(TensorflowMultiTaskRegressor):

class RobustMultitaskClassifier(TensorflowMultiTaskClassifier):

  """Implements a neural network for robust multitasking.

  

  Key idea is to have bypass layers that feed directly from features to task
@@ -25,7 +25,7 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
    self.bypass_weight_init_stddevs = bypass_weight_init_stddevs

    self.bypass_bias_init_consts = bypass_bias_init_consts

    self.bypass_dropouts = bypass_dropouts

    super(RobustMultitaskRegressor, self).__init__(

    super(RobustMultitaskClassifier, self).__init__(

        n_tasks, n_features, logdir, **kwargs)



  def build(self, graph, name_scopes, training):
@@ -71,8 +71,8 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
          len(bypass_bias_init_consts),

          len(bypass_dropouts),

          }

      assert (len(bypass_lengths_set) == 1,

              "All bypass_layer params must have same length.")

      assert len(bypass_lengths_set) == 1, ("All bypass_layer params"+

                                            " must have same length.")

      num_bypass_layers = bypass_lengths_set.pop()



      prev_layer = self.mol_features
@@ -139,41 +139,149 @@ class RobustMultitaskRegressor(TensorflowMultiTaskRegressor): 
        print(task_layer_size)

        #################################################### DEBUG

        output.append(tf.squeeze(

            model_ops.fully_connected_layer(

                tensor=task_layer,

                size=task_layer_size,

            model_ops.logits(task_layer, num_classes=2,

                weight_init=tf.truncated_normal(

                    shape=[task_layer_size, 1],

                    shape=[task_layer_size, 2],

                    stddev=weight_init_stddevs[-1]),

                bias_init=tf.constant(value=bias_init_consts[-1],

                                      shape=[1]))))

                                      shape=[2]))))

      return output



  def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):

    """Construct a feed dictionary from minibatch data.

class RobustMultitaskRegressor(TensorflowMultiTaskRegressor):

  """Implements a neural network for robust multitasking.

  

  Key idea is to have bypass layers that feed directly from features to task

  output. Hopefully will allow tasks to route around bad multitasking.

  """

  def __init__(self, n_tasks, n_features, logdir=None,

               bypass_layer_sizes=[100],

               bypass_weight_init_stddevs=[.02],

               bypass_bias_init_consts=[1.],

               bypass_dropouts=[.5], **kwargs):

    self.bypass_layer_sizes = bypass_layer_sizes

    self.bypass_weight_init_stddevs = bypass_weight_init_stddevs

    self.bypass_bias_init_consts = bypass_bias_init_consts

    self.bypass_dropouts = bypass_dropouts

    super(RobustMultitaskRegressor, self).__init__(

        n_tasks, n_features, logdir, **kwargs)



    TODO(rbharath): ids_b is not used here. Can we remove it?

  def build(self, graph, name_scopes, training):

    """Constructs the graph architecture as specified in its config.



    Args:

      X_b: np.ndarray of shape (batch_size, num_features)

      y_b: np.ndarray of shape (batch_size, num_tasks)

      w_b: np.ndarray of shape (batch_size, num_tasks)

      ids_b: List of length (batch_size) with datapoint identifiers.

    This method creates the following Placeholders:

      mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape

        batch_size x num_features.

    """

    orig_dict = {}

    orig_dict["mol_features"] = X_b

    num_features = self.n_features 

    placeholder_scope = TensorflowGraph.get_placeholder_scope(

        graph, name_scopes)

    with graph.as_default():

      with placeholder_scope:

        self.mol_features = tf.placeholder(

            tf.float32,

            shape=[None, num_features],

            name='mol_features')



      layer_sizes = self.layer_sizes

      weight_init_stddevs = self.weight_init_stddevs

      bias_init_consts = self.bias_init_consts

      dropouts = self.dropouts



      bypass_layer_sizes = self.bypass_layer_sizes

      bypass_weight_init_stddevs = self.bypass_weight_init_stddevs

      bypass_bias_init_consts = self.bypass_bias_init_consts

      bypass_dropouts = self.bypass_dropouts



      lengths_set = {

          len(layer_sizes),

          len(weight_init_stddevs),

          len(bias_init_consts),

          len(dropouts),

          }

      assert len(lengths_set) == 1, "All layer params must have same length."

      num_layers = lengths_set.pop()

      assert num_layers > 0, "Must have some layers defined."



      bypass_lengths_set = {

          len(bypass_layer_sizes),

          len(bypass_weight_init_stddevs),

          len(bypass_bias_init_consts),

          len(bypass_dropouts),

          }

      assert len(bypass_lengths_set) == 1, ("All bypass_layer params"+

                                            " must have same length.")

      num_bypass_layers = bypass_lengths_set.pop()



      prev_layer = self.mol_features

      prev_layer_size = num_features 

      for i in range(num_layers):

        # layer has shape [None, layer_sizes[i]]

        ########################################################## DEBUG

        print("Adding weights of shape %s" % str([prev_layer_size, layer_sizes[i]]))

        ########################################################## DEBUG

        layer = tf.nn.relu(model_ops.fully_connected_layer(

            tensor=prev_layer,

            size=layer_sizes[i],

            weight_init=tf.truncated_normal(

                shape=[prev_layer_size, layer_sizes[i]],

                stddev=weight_init_stddevs[i]),

            bias_init=tf.constant(value=bias_init_consts[i],

                                  shape=[layer_sizes[i]])))

        layer = model_ops.dropout(layer, dropouts[i], training)

        prev_layer = layer

        prev_layer_size = layer_sizes[i]



      output = []

      # top_multitask_layer has shape [None, layer_sizes[-1]]

      top_multitask_layer = prev_layer

      for task in range(self.n_tasks):

      if y_b is not None:

        orig_dict["labels_%d" % task] = y_b[:, task]

      else:

        # Dummy placeholders

        orig_dict["labels_%d" % task] = np.squeeze(

            np.zeros((self.batch_size,)))

      if w_b is not None:

        orig_dict["weights_%d" % task] = w_b[:, task]

        # TODO(rbharath): Might want to make it feasible to have multiple

        # bypass layers.

        # Construct task bypass layer

        prev_bypass_layer = self.mol_features

        prev_bypass_layer_size = num_features

        for i in range(num_bypass_layers):

          # bypass_layer has shape [None, bypass_layer_sizes[i]]

          ########################################################## DEBUG

          print("Adding bypass weights of shape %s"

                % str([prev_bypass_layer_size, bypass_layer_sizes[i]]))

          ########################################################## DEBUG

          bypass_layer = tf.nn.relu(model_ops.fully_connected_layer(

            tensor = prev_bypass_layer,

            size = bypass_layer_sizes[i],

            weight_init=tf.truncated_normal(

                shape=[prev_bypass_layer_size, bypass_layer_sizes[i]],

                stddev=bypass_weight_init_stddevs[i]),

            bias_init=tf.constant(value=bypass_bias_init_consts[i],

                                  shape=[bypass_layer_sizes[i]])))

    

          bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i])

          prev_bypass_layer = bypass_layer

          prev_bypass_layer_size = bypass_layer_sizes[i]

        top_bypass_layer = prev_bypass_layer



        if num_bypass_layers > 0:

          # task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]]

          task_layer = tf.concat(1, [top_multitask_layer, top_bypass_layer])

          task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1]

        else:

        # Dummy placeholders

        orig_dict["weights_%d" % task] = np.ones(

            (self.batch_size,)) 

    return TensorflowGraph.get_feed_dict(orig_dict)

          task_layer = top_multitask_layer

          task_layer_size = layer_sizes[-1]

        ########################################################## DEBUG

        print("Adding output weights of shape %s"

              % str([task_layer_size, 1]))

        ########################################################## DEBUG

        #################################################### DEBUG

        print("task_layer_size")

        print(task_layer_size)

        #################################################### DEBUG

        output.append(tf.squeeze(

            model_ops.fully_connected_layer(

                tensor=task_layer, size=1,

                weight_init=tf.truncated_normal(

                    shape=[task_layer_size, 1],

                    stddev=weight_init_stddevs[-1]),

                bias_init=tf.constant(value=bias_init_consts[-1],

                                      shape=[1]))))

      return output

deepchem/models/tests/test_overfit.py

+32 −0
Original line number Diff line number Diff line
@@ -463,6 +463,38 @@ class TestOverfit(test_util.TensorFlowTestCase): 
    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .9



  def test_tf_robust_multitask_classification_overfit(self):

    """Test tf robust multitask overfits tiny data."""

    n_tasks = 10

    n_samples = 10

    n_features = 3

    n_classes = 2

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)



    verbosity = "high"

    classification_metric = dc.metrics.Metric(

      dc.metrics.accuracy_score, verbosity=verbosity, task_averager=np.mean)

    tensorflow_model = dc.models.RobustMultitaskClassifier(

        n_tasks, n_features, layer_sizes=[50],

        bypass_layer_sizes=[10], dropouts=[0.],

        learning_rate=0.003, weight_init_stddevs=[.1],

        batch_size=n_samples, verbosity=verbosity)

    model = dc.models.TensorflowModel(tensorflow_model)



    # Fit trained model

    model.fit(dataset, nb_epoch=25)

    model.save()



    # Eval model on train

    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .9



  def test_tf_logreg_multitask_classification_overfit(self):

    """Test tf multitask overfits tiny data."""

examples/best_results.csv

+1 −0
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ tox21,logreg,train,0.910,valid,0.759,learning_rate,0.004~0.008,penalty(l1 or l2) 
sider,logreg,train,0.900,valid,0.620,learning_rate,0.004~0.008,penalty(l2),0.4~1

muv,logreg,train,0.910,valid,0.744,learning_rate,0.002~0.004,penalty(l2),0.4~0.7

toxcast,logreg,train,0.762,valid,0.622,learning_rate,0.004~0.1,penalty(l2),0.2~0.6

pcba,logreg,train,0.794,valid,0.762,learning_rate,0.004,penalty(l2),0.3



sider,tf,train,0.931,valid,0.647,learning_rate,0.0003~0.003,dropouts,0.2~0.3,layer_sizes,1000~1200

tox21,tf,train,0.996,valid,0.763,learning_rate,0.001~0.003,dropouts,0.4~0.5,layer_sizes,1000~1200

CRA Git | Maintained and supported by SUSTech CRA and CCSE