Commit 74d376e6 authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #266 from rbharath/low_data_models 

Experimental Low data models
parents 89f095ce 335545f1

deepchem/models/tf_keras_models/support_classifier.py

+63 −32
Original line number Diff line number Diff line
@@ -136,6 +136,25 @@ class SupportGenerator(object): 
  each task, and returns in a randomized order

  """

  def __init__(self, dataset, tasks, n_pos, n_neg, n_trials, replace):

    """

    Parameters

    ----------

    dataset: deepchem.datasets.Dataset

      Holds dataset from which support sets will be sampled.

    tasks: list

      Indices of tasks from which supports are sampled.

      TODO(rbharath): Can this be removed.

    n_pos: int

      Number of positive samples

    n_neg: int

      Number of negative samples.

    n_trials: int

      Number of passes over tasks to make. In total, n_tasks*n_trials

      support sets will be sampled by algorithm.

    replace: bool

      Whether to use sampling with or without replacement.

    """

      

    self.tasks = tasks

    self.n_tasks = len(tasks)

    self.n_trials = n_trials
@@ -274,18 +293,15 @@ class SupportGraphClassifier(Model): 
      feed_dict[K.learning_phase()] = training

    return feed_dict



  def fit(self, dataset, n_trials_per_epoch=1000, nb_epoch=10, n_pos=1,

  def fit(self, dataset, n_trials=1000, n_pos=1,

          n_neg=9, replace=True, **kwargs):

    """Fits model on dataset.



    Note that fitting for support models is quite different from fitting

    for other deep models. Fitting is a two-level process. During each epoch,

    we repeat n_trials_per_epoch, where for each trial, we randomply sample

    a support set for a given task, and independently a test set from that same

    task. The SupportGenerator class iterates over the tasks in random order.



    # TODO(rbharath): Is the concept of an epoch even meaningful here? There's

    # never a guarantee that the full dataset is covered as in usual fit.

    Note that fitting for support models is quite different from fitting for

    other deep models. Fitting is a two-level process.  We perform n_trials,

    where for each trial, we randomply sample a support set for each given

    task, and independently a test set from that same task. The

    SupportGenerator class iterates over the tasks in random order.



    # TODO(rbharath): Would it improve performance to sample multiple test sets

    for each support set or would that only harm performance?
@@ -296,10 +312,8 @@ class SupportGraphClassifier(Model): 
    ----------

    dataset: deepchem.datasets.Dataset

      Dataset to fit model on.

    n_trials_per_epoch: int, optional

      Number of (support, test) pairs to sample and train on per epoch.

    nb_epoch: int, optional

      Number of training epochs.

    n_trials: int, optional

      Number of (support, test) pairs to sample and train on.

    n_pos: int, optional

      Number of positive examples per support.

    n_neg: int, optional
@@ -308,18 +322,17 @@ class SupportGraphClassifier(Model): 
      Whether or not to use replacement when sampling supports/tests.

    """

    # Perform the optimization

    for epoch in range(nb_epoch):

    # TODO(rbharath): Try removing this learning rate.

      lr = self.learning_rate / (1 + float(epoch) / self.decay_T)

      print("Training epoch %d" % epoch)

    #lr = self.learning_rate / (1 + float(epoch) / self.decay_T)

    lr = self.learning_rate



    # Create different support sets

      for (task, support) in SupportGenerator(dataset, range(self.n_tasks),

          n_pos, n_neg, n_trials_per_epoch, replace):

        print("Sampled Support set")

    support_generator = SupportGenerator(dataset, range(self.n_tasks),

        n_pos, n_neg, n_trials, replace)

    for ind, (task, support) in enumerate(support_generator):

      print("Sample %d from task %s" % (ind, str(task)))

      # Get batch to try it out on

      test = get_task_test(dataset, self.test_batch_size, task, replace)

        print("Obtained batch")

      feed_dict = self.construct_feed_dict(test, support)

      # Train on support set, batch pair

      self.sess.run(self.train_op, feed_dict=feed_dict)
@@ -422,6 +435,12 @@ class SupportGraphClassifier(Model): 


    TODO(rbharath): Does not currently support any transforms.

    TODO(rbharath): Only for 1 task at a time currently. Is there a better way?

    Parameters

    ----------

    support: deepchem.datasets.Dataset

      The support dataset

    test: deepchem.datasets.Dataset

      The test dataset

    """

    y_preds = []

    for (X_batch, y_batch, w_batch, ids_batch) in test.iterbatches(
@@ -442,6 +461,10 @@ class SupportGraphClassifier(Model): 
    # Get scores

    pred, scores = self.sess.run([self.pred_op, self.scores_op], feed_dict=feed_dict)

    y_pred_batch = np.round(scores)

    ########################################################### DEBUG

    # Remove padded elements

    y_pred_batch = y_pred_batch[:n_samples]

    ########################################################### DEBUG

    return y_pred_batch



  def predict_proba_on_batch(self, support, test_batch):
@@ -454,6 +477,10 @@ class SupportGraphClassifier(Model): 
    # Get scores

    pred, scores = self.sess.run([self.pred_op, self.scores_op], feed_dict=feed_dict)

    y_pred_batch = to_one_hot(np.round(pred))

    ########################################################### DEBUG

    # Remove padded elements

    y_pred_batch = y_pred_batch[:n_samples]

    ########################################################### DEBUG

    return y_pred_batch

    

  def evaluate(self, dataset, test_tasks, metric, n_pos=1,
@@ -495,9 +522,10 @@ class SupportGraphClassifier(Model): 
    """

    # Get batches

    task_scores = {task: [] for task in test_tasks}

    for (task, support) in SupportGenerator(dataset, test_tasks,

         n_pos, n_neg, n_trials, replace):

      print("Sampled Support set.")

    support_generator = SupportGenerator(dataset, test_tasks,

        n_pos, n_neg, n_trials, replace)

    for ind, (task, support) in enumerate(support_generator):

      print("Eval sample %d from task %s" % (ind, str(task)))

      if exclude_support:

        print("Removing support datapoints for eval.")

        task_dataset = get_task_dataset_minus_support(dataset, support, task)
@@ -505,7 +533,10 @@ class SupportGraphClassifier(Model): 
        print("Keeping support datapoints for eval.")

        task_dataset = get_task_dataset(dataset, task)

      y_pred = self.predict_proba(support, task_dataset)



      ######################################################### DEBUG

      #print("task_dataset.y.shape, y_pred.shape, task_dataset.w.shape")

      #print(task_dataset.y.shape, y_pred.shape, task_dataset.w.shape)

      ######################################################### DEBUG

      task_scores[task].append(metric.compute_metric(

          task_dataset.y, y_pred, task_dataset.w))

examples/bace/bace_datasets.py

+1 −1
Original line number Diff line number Diff line
@@ -123,7 +123,7 @@ def load_bace(mode="regression", transform=True, split="20-80"): 
  transformers = input_transformers + output_transformers

  for dataset in [train_dataset, valid_dataset, test_dataset, crystal_dataset]:

    for transformer in transformers:

        transformer.transform(dataset)

        dataset = transformer.transform(dataset)



  return (bace_tasks, train_dataset, valid_dataset, test_dataset,

          crystal_dataset, output_transformers)

examples/low_data/datasets.py

0 → 100644
+119 −0
Original line number Diff line number Diff line 
"""

Load datasets for Low Data processing.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



import os

import shutil

import tempfile

import numpy as np

from deepchem.featurizers.graph_features import ConvMolFeaturizer

from deepchem.utils.save import load_from_disk

from deepchem.featurizers.featurize import DataLoader

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.datasets import DiskDataset

from deepchem.transformers import BalancingTransformer



def load_tox21_ecfp(base_dir=None, num_train=7200):

  """Load Tox21 datasets. Does not do train/test split"""

  # Set some global variables up top

  verbosity = "high"

  if base_dir is None:

    base_dir = tempfile.mkdtemp()



  # Create some directories for analysis

  # The base_dir holds the results of all analysis

  if os.path.exists(base_dir):

    shutil.rmtree(base_dir)

  if not os.path.exists(base_dir):

    os.makedirs(base_dir)

  current_dir = os.path.dirname(os.path.realpath(__file__))

  #Make directories to store the raw and featurized datasets.

  data_dir = os.path.join(base_dir, "dataset")

  train_dir = os.path.join(base_dir, "train")

  valid_dir = os.path.join(base_dir, "valid")



  # Load Tox21 dataset

  print("About to load Tox21 dataset.")

  dataset_file = os.path.join(

      current_dir, "../../datasets/tox21.csv.gz")

  dataset = load_from_disk(dataset_file)

  print("Columns of dataset: %s" % str(dataset.columns.values))

  print("Number of examples in dataset: %s" % str(dataset.shape[0]))



  # Featurize Tox21 dataset

  print("About to featurize Tox21 dataset.")

  featurizer = CircularFingerprint(size=1024)

  tox21_tasks = ['NR-AR', 'NR-AR-LBD', 'NR-AhR', 'NR-Aromatase', 'NR-ER',

                 'NR-ER-LBD', 'NR-PPAR-gamma', 'SR-ARE', 'SR-ATAD5',

                 'SR-HSE', 'SR-MMP', 'SR-p53']



  loader = DataLoader(tasks=tox21_tasks,

                      smiles_field="smiles",

                      featurizer=featurizer,

                      verbosity=verbosity)

  dataset = loader.featurize(

      dataset_file, data_dir, shard_size=8192)



  # Initialize transformers 

  transformers = [

      BalancingTransformer(transform_w=True, dataset=dataset)]



  print("About to transform data")

  for transformer in transformers:

      dataset = transformer.transform(dataset)



  return tox21_tasks, dataset, transformers



def load_tox21_convmol(base_dir=None, num_train=7200):

  """Load Tox21 datasets. Does not do train/test split"""

  # Set some global variables up top

  verbosity = "high"

  if base_dir is None:

    base_dir = tempfile.mkdtemp()



  # Create some directories for analysis

  # The base_dir holds the results of all analysis

  if os.path.exists(base_dir):

    shutil.rmtree(base_dir)

  if not os.path.exists(base_dir):

    os.makedirs(base_dir)

  current_dir = os.path.dirname(os.path.realpath(__file__))

  #Make directories to store the raw and featurized datasets.

  data_dir = os.path.join(base_dir, "dataset")

  train_dir = os.path.join(base_dir, "train")

  valid_dir = os.path.join(base_dir, "valid")



  # Load Tox21 dataset

  print("About to load Tox21 dataset.")

  dataset_file = os.path.join(

      current_dir, "../../datasets/tox21.csv.gz")

  dataset = load_from_disk(dataset_file)

  print("Columns of dataset: %s" % str(dataset.columns.values))

  print("Number of examples in dataset: %s" % str(dataset.shape[0]))



  # Featurize Tox21 dataset

  print("About to featurize Tox21 dataset.")

  featurizer = ConvMolFeaturizer()

  tox21_tasks = ['NR-AR', 'NR-AR-LBD', 'NR-AhR', 'NR-Aromatase', 'NR-ER',

                 'NR-ER-LBD', 'NR-PPAR-gamma', 'SR-ARE', 'SR-ATAD5',

                 'SR-HSE', 'SR-MMP', 'SR-p53']



  loader = DataLoader(tasks=tox21_tasks,

                      smiles_field="smiles",

                      featurizer=featurizer,

                      verbosity=verbosity)

  dataset = loader.featurize(

      dataset_file, data_dir, shard_size=8192)



  # Initialize transformers 

  transformers = [

      BalancingTransformer(transform_w=True, dataset=dataset)]



  print("About to transform data")

  for transformer in transformers:

      dataset = transformer.transform(dataset)



  return tox21_tasks, dataset, transformers

examples/low_data/tox_rf_K_fold.py

0 → 100644
+84 −0
Original line number Diff line number Diff line 
"""

Train low-data models on random forests.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



import tempfile

import numpy as np

import deepchem as dc

from datasets import load_tox21_ecfp

from sklearn.ensemble import RandomForestClassifier

from deepchem.metrics import Metric

from deepchem.splits.task_splitter import merge_fold_datasets

from deepchem.splits.task_splitter import TaskSplitter

from deepchem.models.sklearn_models import SklearnModel

from deepchem.models.tf_keras_models.support_classifier import SupportGenerator

from deepchem.models.tf_keras_models.support_classifier import get_task_dataset_minus_support



model_dir = tempfile.mkdtemp()



# 4-fold splits

K = 4

# 10 positive/negative ligands

n_pos = 10

n_neg = 10

# 10 trials on test-set

n_trials = 10

# Sample supports without replacement (all pos/neg should be different)

replace = False



tox21_tasks, dataset, transformers = load_tox21_ecfp()



# Define metric

metric = Metric(dc.metrics.roc_auc_score, verbosity="high",

                mode="classification")



task_splitter = TaskSplitter()

fold_datasets = task_splitter.k_fold_split(dataset, K)



all_scores = {}

for fold in range(K):

  train_inds = list(set(range(K)) - set([fold]))

  train_folds = [fold_datasets[ind] for ind in train_inds]

  train_dataset = merge_fold_datasets(train_folds)

  test_dataset = fold_datasets[fold]



  fold_tasks = range(fold * len(test_dataset.get_task_names()),

                     (fold+1) * len(test_dataset.get_task_names()))



  # Get supports on test-set

  support_generator = SupportGenerator(

      test_dataset, range(len(test_dataset.get_task_names())), n_pos, n_neg,

      n_trials, replace)



  # Compute accuracies

  task_scores = {task: [] for task in range(len(test_dataset.get_task_names()))}

  for (task, support) in support_generator:

    # Train model on support

    sklearn_model = RandomForestClassifier(

        class_weight="balanced", n_estimators=50)

    model = SklearnModel(sklearn_model, model_dir)

    model.fit(support)



    # Test model

    task_dataset = get_task_dataset_minus_support(test_dataset, support, task)

    y_pred = model.predict_proba(task_dataset)

    score = metric.compute_metric(

        task_dataset.y, y_pred, task_dataset.w)

    #print("Score on task %s is %s" % (str(task), str(score)))

    task_scores[task].append(score)



  # Join information for all tasks.

  mean_task_scores = {}

  for task in range(len(test_dataset.get_task_names())):

    mean_task_scores[task] = np.mean(np.array(task_scores[task]))

  print("Fold %s" % str(fold))

  print(mean_task_scores)



  for (fold_task, task) in zip(fold_tasks, range(len(test_dataset.get_task_names()))):

    all_scores[fold_task] = mean_task_scores[task]



print("All scores")

print(all_scores)

examples/low_data/tox_rf_one_fold.py

0 → 100644
+71 −0
Original line number Diff line number Diff line 
"""

Train low-data models with random forests. Test last fold only.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



import tempfile

import numpy as np

import deepchem as dc

from datasets import load_tox21_ecfp

from sklearn.ensemble import RandomForestClassifier

from deepchem.metrics import Metric

from deepchem.splits.task_splitter import merge_fold_datasets

from deepchem.splits.task_splitter import TaskSplitter

from deepchem.models.sklearn_models import SklearnModel

from deepchem.models.tf_keras_models.support_classifier import SupportGenerator

from deepchem.models.tf_keras_models.support_classifier import get_task_dataset_minus_support



model_dir = tempfile.mkdtemp()



# 4-fold splits

K = 4

# 10 positive/negative ligands

n_pos = 10

n_neg = 10

# 10 trials on test-set

n_trials = 10

# Sample supports without replacement (all pos/neg should be different)

replace = False



tox21_tasks, dataset, transformers = load_tox21_ecfp()



# Define metric

metric = Metric(dc.metrics.roc_auc_score, verbosity="high",

                mode="classification")



task_splitter = TaskSplitter()

fold_datasets = task_splitter.k_fold_split(dataset, K)



train_folds = fold_datasets[:-1] 

train_dataset = merge_fold_datasets(train_folds)

test_dataset = fold_datasets[-1]



# Get supports on test-set

support_generator = SupportGenerator(

    test_dataset, range(len(test_dataset.get_task_names())), n_pos, n_neg,

    n_trials, replace)



# Compute accuracies

task_scores = {task: [] for task in range(len(test_dataset.get_task_names()))}

for (task, support) in support_generator:

  # Train model on support

  sklearn_model = RandomForestClassifier(

      class_weight="balanced", n_estimators=50)

  model = SklearnModel(sklearn_model, model_dir)

  model.fit(support)



  # Test model

  task_dataset = get_task_dataset_minus_support(test_dataset, support, task)

  y_pred = model.predict_proba(task_dataset)

  score = metric.compute_metric(

      task_dataset.y, y_pred, task_dataset.w)

  #print("Score on task %s is %s" % (str(task), str(score)))

  task_scores[task].append(score)



# Join information for all tasks.

mean_task_scores = {}

for task in range(len(test_dataset.get_task_names())):

  mean_task_scores[task] = np.mean(np.array(task_scores[task]))

print(mean_task_scores)

CRA Git | Maintained and supported by SUSTech CRA and CCSE