Commit e5475d1c authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #693 from patrickhop/master 

Approximate Bayesian Inference Using Dropout
parents c8c3b8bf 681df307

deepchem/models/tensorgraph/models/graph_models.py

+71 −26
Original line number Diff line number Diff line
@@ -7,11 +7,14 @@ from deepchem.metrics import to_one_hot, from_one_hot 
from deepchem.models.tensorgraph.graph_layers import WeaveLayer, WeaveGather, \

    Combine_AP, Separate_AP, DTNNEmbedding, DTNNStep, DTNNGather, DAGLayer, DAGGather, DTNNExtract

from deepchem.models.tensorgraph.layers import Dense, Concat, SoftMax, SoftMaxCrossEntropy, GraphConv, BatchNorm, \

    GraphPool, GraphGather, WeightedError, BatchNormalization

    GraphPool, GraphGather, WeightedError, Dropout, BatchNormalization

from deepchem.models.tensorgraph.layers import L2Loss, Label, Weights, Feature

from deepchem.models.tensorgraph.tensor_graph import TensorGraph

from deepchem.trans import undo_transforms

from deepchem.utils.evaluate import GeneratorEvaluator

from deepchem.data import NumpyDataset

from deepchem.data.data_loader import featurize_smiles_np

from deepchem.feat.graph_features import ConvMolFeaturizer





class WeaveTensorGraph(TensorGraph):
@@ -476,6 +479,7 @@ class GraphConvTensorGraph(TensorGraph): 


    """

    self.n_tasks = n_tasks

    self.error_bars = True if 'error_bars' in kwargs and kwargs['error_bars'] else False

    kwargs['use_queue'] = False

    super(GraphConvTensorGraph, self).__init__(**kwargs)

    self.build_graph()
@@ -507,20 +511,23 @@ class GraphConvTensorGraph(TensorGraph): 
    batch_norm2 = BatchNorm(in_layers=[gc2])

    gp2 = GraphPool(in_layers=[batch_norm2, self.degree_slice, self.membership]

                    + self.deg_adjs)

    dense = Dense(out_channels=128, activation_fn=None, in_layers=[gp2])

    dense = Dense(out_channels=128, activation_fn=tf.nn.relu, in_layers=[gp2])

    batch_norm3 = BatchNorm(in_layers=[dense])

    gg1 = GraphGather(

    readout = GraphGather(

        batch_size=self.batch_size,

        activation_fn=tf.nn.tanh,

        in_layers=[batch_norm3, self.degree_slice, self.membership] +

        self.deg_adjs)



    if self.error_bars == True:

      readout = Dropout(in_layers=[readout], dropout_prob=0.2)



    costs = []

    self.my_labels = []

    for task in range(self.n_tasks):

      if self.mode == 'classification':

        classification = Dense(

            out_channels=2, activation_fn=None, in_layers=[gg1])

            out_channels=2, activation_fn=None, in_layers=[readout])



        softmax = SoftMax(in_layers=[classification])

        self.add_output(softmax)
@@ -530,7 +537,8 @@ class GraphConvTensorGraph(TensorGraph): 
        cost = SoftMaxCrossEntropy(in_layers=[label, classification])

        costs.append(cost)

      if self.mode == 'regression':

        regression = Dense(out_channels=1, activation_fn=None, in_layers=[gg1])

        regression = Dense(

            out_channels=1, activation_fn=None, in_layers=[readout])

        self.add_output(regression)



        label = Label(shape=(None, 1))
@@ -583,6 +591,7 @@ class GraphConvTensorGraph(TensorGraph): 
              self.layers[k.name].out_tensor: v

              for k, v in six.iteritems(feed_dict)

          }

          feed_dict[self._training_placeholder] = 1.0  ##

          result = np.array(sess.run(out_tensors, feed_dict=feed_dict))

          if len(result.shape) == 3:

            result = np.transpose(result, axes=[1, 0, 2])
@@ -600,28 +609,64 @@ class GraphConvTensorGraph(TensorGraph): 
        labels=self.my_labels,

        weights=[self.my_task_weights])



  def predict_on_smiles(self, smiles, transformers):

    max_index = len(smiles)

    num_batches = max_index // self.batch_size

  def bayesian_predict(self,

                       dataset,

                       transformers=[],

                       n_passes=4,

                       untransform=False):

    """Generates predictions and confidences on a dataset object

     https://arxiv.org/pdf/1506.02142.pdf



    # Returns:

      mu: numpy ndarray of shape (n_samples, n_tasks)

      sigma: numpy ndarray of shape (n_samples, n_tasks)

    """

    X = dataset.X

    max_index = X.shape[0] - 1

    num_batches = (max_index // self.batch_size) + 1



    y_ = []

    mus = []

    sigmas = []

    for i in range(num_batches):

      smiles_batch = smiles[i * self.batch_size:(i + 1) * self.batch_size]

      y_.append(self.predict_on_smiles_batch(smiles_batch, transformers))

    smiles_batch = smiles[num_batches * self.batch_size:max_index]

    y_.append(self.predict_on_smiles_batch(smiles_batch, transformers))



    return np.concatenate(y_, axis=1)



  def predict_on_smiles_batch(self, smiles, transformers=[]):

    featurizer = ConvMolFeaturizer()

    convmols = featurize_smiles_np(smiles, featurizer)



    n_smiles = convmols.shape[0]

      start = i * self.batch_size

      end = min((i + 1) * self.batch_size, max_index + 1)

      batch = X[start:end]

      mu, sigma = self.bayesian_predict_on_batch(

          batch, transformers=[], n_passes=n_passes)

      mus.append(mu)

      sigmas.append(sigma)

    mu = np.concatenate(mus, axis=0)

    sigma = np.concatenate(sigmas, axis=0) + 0.55



    if untransform:

      mu = undo_transforms(mu, transformers)

      for i in range(sigma.shape[1]):

        sigma[:, i] = sigma[:, i] * transformers[0].y_stds[i]



    return mu[:max_index + 1], sigma[:max_index + 1]



  def predict_on_smiles(self, smiles, transformers=[], untransform=False):

    """Generates predictions on a numpy array of smile strings



    # Returns:

      y_: numpy ndarray of shape (n_samples, n_tasks)

    """

    max_index = len(smiles) - 1

    n_tasks = len(self.outputs)

    num_batches = (max_index // self.batch_size) + 1

    featurizer = ConvMolFeaturizer()



    dataset = NumpyDataset(X=convmols, y=None, n_tasks=n_tasks)

    generator = self.default_generator(dataset, predict=True, pad_batches=False)

    y_ = self.predict_on_generator(generator, transformers)



    return y_.reshape(-1, n_tasks)[:n_smiles]
 
    y_ = []

    for i in range(num_batches):

      start = i * self.batch_size

      end = min((i + 1) * self.batch_size, max_index + 1)

      smiles_batch = smiles[start:end]

      y_.append(

          self.predict_on_smiles_batch(smiles_batch, featurizer, transformers))

    y_ = np.concatenate(y_, axis=0)[:max_index + 1]

    y_ = y_.reshape(-1, n_tasks)



    if untransform:

      y_ = undo_transforms(y_, transformers)



    return y_

deepchem/models/tensorgraph/tensor_graph.py

+36 −0
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ from deepchem.models.models import Model 
from deepchem.models.tensorgraph.layers import InputFifoQueue, Label, Feature, Weights

from deepchem.trans import undo_transforms

from deepchem.utils.evaluate import GeneratorEvaluator

from deepchem.feat.graph_features import ConvMolFeaturizer

from deepchem.data.data_loader import featurize_smiles_np





class TensorGraph(Model):
@@ -280,6 +282,40 @@ class TensorGraph(Model): 
          results.append(result)

        return np.concatenate(results, axis=0)



  def bayesian_predict_on_batch(self, X, transformers=[], n_passes=4):

    """

    Returns:

      mu: numpy ndarray of shape (n_samples, n_tasks)

      sigma: numpy ndarray of shape (n_samples, n_tasks)

    """

    dataset = NumpyDataset(X=X, y=None, n_tasks=len(self.outputs))

    y_ = []

    for i in range(n_passes):

      generator = self.default_generator(

          dataset, predict=True, pad_batches=True)

      y_.append(self.predict_on_generator(generator, transformers))



    y_ = np.concatenate(y_, axis=2)

    mu = np.mean(y_, axis=2)

    sigma = np.std(y_, axis=2)



    return mu, sigma



  def predict_on_smiles_batch(self,

                              smiles,

                              featurizer,

                              n_tasks,

                              transformers=[]):

    """

    # Returns:

      A numpy ndarray of shape (n_samples, n_tasks)

    """

    convmols = featurize_smiles_np(smiles, featurizer)



    dataset = NumpyDataset(X=convmols, y=None, n_tasks=len(self.outputs))

    generator = self.default_generator(dataset, predict=True, pad_batches=True)

    return self.predict_on_generator(generator, transformers)



  def predict_on_batch(self, X, sess=None, transformers=[]):

    """Generates output predictions for the input samples,

      processing the samples in a batched way.

deepchem/trans/transformers.py

+6 −5
Original line number Diff line number Diff line
@@ -113,11 +113,6 @@ class NormalizationTransformer(Transformer): 
               dataset=None,

               transform_gradients=False):

    """Initialize normalization transformation."""

    super(NormalizationTransformer, self).__init__(

        transform_X=transform_X,

        transform_y=transform_y,

        transform_w=transform_w,

        dataset=dataset)

    if transform_X:

      X_means, X_stds = dataset.get_statistics(X_stats=True, y_stats=False)

      self.X_means = X_means
@@ -135,6 +130,12 @@ class NormalizationTransformer(Transformer): 
      self.grad = np.reshape(true_grad, (true_grad.shape[0], -1, 3))

      self.ydely_means = ydely_means



    super(NormalizationTransformer, self).__init__(

        transform_X=transform_X,

        transform_y=transform_y,

        transform_w=transform_w,

        dataset=dataset)



  def transform(self, dataset, parallel=False):

    return super(NormalizationTransformer, self).transform(

        dataset, parallel=parallel)

examples/delaney/delaney_datasets.py

+10 −7
Original line number Diff line number Diff line
@@ -10,13 +10,14 @@ import numpy as np 
import shutil

import deepchem as dc





def load_delaney(featurizer='ECFP', split='index'):

  """Load delaney datasets."""

  # Featurize Delaney dataset

  print("About to featurize Delaney dataset.")

  current_dir = os.path.dirname(os.path.realpath(__file__))

  dataset_file = os.path.join(

      current_dir, "../../datasets/delaney-processed.csv")

  dataset_file = os.path.join(current_dir,

                              "../../datasets/delaney-processed.csv")

  delaney_tasks = ['measured log solubility in mols per litre']

  if featurizer == 'ECFP':

    featurizer = dc.feat.CircularFingerprint(size=1024)
@@ -24,20 +25,22 @@ def load_delaney(featurizer='ECFP', split='index'): 
    featurizer = dc.feat.ConvMolFeaturizer()

  loader = dc.data.CSVLoader(

      tasks=delaney_tasks, smiles_field="smiles", featurizer=featurizer)

  dataset = loader.featurize(

      dataset_file, shard_size=8192)

  dataset = loader.featurize(dataset_file, shard_size=8192)



  # Initialize transformers

  transformers = [

      dc.trans.NormalizationTransformer(transform_y=True, dataset=dataset)]

      dc.trans.NormalizationTransformer(transform_y=True, dataset=dataset)

  ]



  print("About to transform data")

  for transformer in transformers:

    dataset = transformer.transform(dataset)



  splitters = {'index': dc.splits.IndexSplitter(),

  splitters = {

      'index': dc.splits.IndexSplitter(),

      'random': dc.splits.RandomSplitter(),

               'scaffold': dc.splits.ScaffoldSplitter()}

      'scaffold': dc.splits.ScaffoldSplitter()

  }

  splitter = splitters[split]

  train, valid, test = splitter.train_valid_test_split(dataset)

  return delaney_tasks, (train, valid, test), transformers

examples/delaney/delaney_graphconv_error_bars.py

0 → 100644
+78 −0
Original line number Diff line number Diff line 
"""

Script that trains graphconv models on delaney dataset.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



import numpy as np

np.random.seed(123)

import tensorflow as tf

tf.set_random_seed(123)

import deepchem as dc



import csv

from sklearn.metrics import r2_score

from deepchem.trans import undo_transforms



from delaney_datasets import load_delaney



MODEL_DIR = 'model_saves'

BATCH_SIZE = 64

LR = 1e-3

ERROR_BARS = True



delaney_tasks, delaney_datasets, transformers = dc.molnet.load_delaney(

    featurizer='GraphConv', split='scaffold')

train_dataset, valid_dataset, test_dataset = delaney_datasets

metric = dc.metrics.Metric(dc.metrics.pearson_r2_score, np.mean)



model = dc.models.GraphConvTensorGraph(

    len(delaney_tasks),

    batch_size=BATCH_SIZE,

    learning_rate=LR,

    use_queue=False,

    mode='regression',

    model_dir=MODEL_DIR,

    error_bars=ERROR_BARS)



model.fit(train_dataset, nb_epoch=8)

valid_scores = model.evaluate(valid_dataset, [metric], transformers)



model.save()

model.load_from_dir('model_saves')



mu, sigma = model.bayesian_predict(

    valid_dataset.X, transformers, untransform=True, n_passes=24)

print(mu[:4])

print(sigma[:4])



target = undo_transforms(valid_dataset.y, transformers)



print(r2_score(target, mu))



mu = mu[:, 0].tolist()

sigma = sigma[:, 0].tolist()

target = target[:, 0].tolist()



print(mu[:4])

print(sigma[:4])

print(target[:4])



in_one_sigma = 0

in_two_sigma = 0

in_four_sigma = 0



for i in xrange(0, len(mu)):

  if target[i] < (mu[i] + sigma[i]) and target[i] > (mu[i] - sigma[i]):

    in_one_sigma += 1

  if target[i] < (mu[i] + 2 * sigma[i]) and target[i] > (mu[i] - 2 * sigma[i]):

    in_two_sigma += 1

  if target[i] < (mu[i] + 4 * sigma[i]) and target[i] > (mu[i] - 4 * sigma[i]):

    in_four_sigma += 1



print('percent in 1 sigma [%f]' % (in_one_sigma / float(len(mu))))

print('percent in 2 sigma [%f]' % (in_two_sigma / float(len(mu))))

print('percent in 4 sigma [%f]' % (in_four_sigma / float(len(mu))))



print(sorted(sigma))

CRA Git | Maintained and supported by SUSTech CRA and CCSE