Commit 94b91cf6 authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #261 from rbharath/task_splits 

Implementation of Task-based Dataset splits
parents e3d07edc ee4933b2

deepchem/datasets/__init__.py

+15 −15
Original line number Diff line number Diff line
@@ -237,7 +237,7 @@ class NumpyDataset(Dataset): 


  def get_task_names(self):

    """Get the names of the tasks associated with this dataset."""

    tasks = np.arange(self._y.shape[1])

    return np.arange(self._y.shape[1])



  @property

  def X(self):
@@ -486,7 +486,6 @@ class DiskDataset(Dataset): 
    """

    return self.metadata_df.shape[0]





  def itershards(self):

    """

    Return an object that iterates over all shards in dataset.
@@ -603,6 +602,7 @@ class DiskDataset(Dataset): 
  @staticmethod

  def from_numpy(data_dir, X, y, w=None, ids=None, tasks=None, verbosity=None,

                 compute_feature_statistics=True):

    """Creates a DiskDataset object from specified Numpy arrays."""

    n_samples = len(X)

    # The -1 indicates that y will be reshaped to have length -1

    if n_samples > 0:

deepchem/models/tests/test_overfit.py

+7 −1
Original line number Diff line number Diff line
@@ -632,6 +632,8 @@ class TestOverfitAPI(test_util.TensorFlowTestCase): 


  def test_tf_robust_multitask_regression_overfit(self):

    """Test tf robust multitask overfits tiny data."""

    np.random.seed(123)

    tf.set_random_seed(123)

    n_tasks = 10

    n_samples = 10

    n_features = 3
@@ -669,6 +671,8 @@ class TestOverfitAPI(test_util.TensorFlowTestCase): 


  def test_graph_conv_singletask_classification_overfit(self):

    """Test graph-conv multitask overfits tiny data."""

    np.random.seed(123)

    tf.set_random_seed(123)

    g = tf.Graph()

    sess = tf.Session(graph=g)

    K.set_session(sess)
@@ -725,10 +729,12 @@ class TestOverfitAPI(test_util.TensorFlowTestCase): 
      print("scores")

      print(scores)

      ######################################################### DEBUG

      assert scores[classification_metric.name] > .85

      assert scores[classification_metric.name] > .75



  def test_attn_lstm_singletask_classification_overfit(self):

    """Test support graph-conv multitask overfits tiny data."""

    np.random.seed(123)

    tf.set_random_seed(123)

    g = tf.Graph()

    sess = tf.Session(graph=g)

    K.set_session(sess)

deepchem/splits/__init__.py

+3 −6
Original line number Diff line number Diff line
@@ -41,7 +41,7 @@ class Splitter(object): 
    """Creates splitter object."""

    self.verbosity = verbosity



  def k_fold_split(self, dataset, directories, compute_feature_statistics=True):

  def k_fold_split(self, dataset, directories=None, compute_feature_statistics=True):

    """Does K-fold split of dataset."""

    log("Computing K-fold split", self.verbosity)

    k = len(directories)
@@ -59,8 +59,6 @@ class Splitter(object): 
      fold_dataset = rem_dataset.select( 

          fold_dir, fold_inds,

          compute_feature_statistics=compute_feature_statistics)

      # TODO(rbharath): Is making a tempfile the best way to handle remainders?

      # Would be  nice to be able to do in memory dataset construction...

      rem_dir = tempfile.mkdtemp()

      rem_dataset = rem_dataset.select( 

          rem_dir, rem_inds,
@@ -68,8 +66,8 @@ class Splitter(object): 
      fold_datasets.append(fold_dataset)

    return fold_datasets



  def train_valid_test_split(self, dataset, train_dir,

                             valid_dir, test_dir, frac_train=.8,

  def train_valid_test_split(self, dataset, train_dir=None,

                             valid_dir=None, test_dir=None, frac_train=.8,

                             frac_valid=.1, frac_test=.1, seed=None,

                             log_every_n=1000,

                             compute_feature_statistics=True):
@@ -243,7 +241,6 @@ class RandomStratifiedSplitter(Splitter): 
    return fold_datasets







class MolecularWeightSplitter(Splitter):

  """

  Class for doing data splits by molecular weight.

deepchem/splits/task_splitter.py

0 → 100644
+110 −0
Original line number Diff line number Diff line 
"""

Contains an abstract base class that supports chemically aware data splits.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "GPL"



import tempfile

import numpy as np

from rdkit import Chem

from deepchem.utils import ScaffoldGenerator

from deepchem.utils.save import log

from deepchem.datasets import NumpyDataset

from deepchem.featurizers.featurize import load_data

from deepchem.splits import Splitter



def merge_fold_datasets(fold_datasets):

  """Merges fold datasets together.



  Assumes that fold_datasets were outputted from k_fold_split. Specifically,

  assumes that each dataset contains the same datapoints, listed in the same

  ordering.

  """

  if not len(fold_datasets):

    return None



  # All datasets share features and identifiers by assumption.

  X = fold_datasets[0].X

  ids = fold_datasets[0].ids



  ys, ws = [], []

  for fold_dataset in fold_datasets:

    ys.append(fold_dataset.y)

    ws.append(fold_dataset.w)

  y = np.concatenate(ys, axis=1)

  w = np.concatenate(ws, axis=1)

  return NumpyDataset(X, y, w, ids)



class TaskSplitter(Splitter):

  """

  Provides a simple interface for splitting datasets task-wise.



  For some learning problems, the training and test datasets should

  have different tasks entirely. This is a different paradigm from the

  usual Splitter, which ensures that split datasets have different

  datapoints, not different tasks.

  """



  def __init__(self):

    "Creates Task Splitter object."

    pass



  def train_valid_test_split(self, dataset, frac_train=.8, frac_valid=.1,

                             frac_test=.1):

    """Performs a train/valid/test split of the tasks for dataset.



    Parameters

    ----------

    dataset: deepchem.datasets.Dataset

      Dataset to be split

    frac_train: float, optional

      Proportion of tasks to be put into train. Rounded to nearest int.

    frac_valid: float, optional

      Proportion of tasks to be put into valid. Rounded to nearest int.

    frac_test: float, optional

      Proportion of tasks to be put into test. Rounded to nearest int.

    """

    n_tasks = len(dataset.get_task_names())

    n_train = int(np.round(frac_train * n_tasks))

    n_valid = int(np.round(frac_valid * n_tasks))

    n_test = int(np.round(frac_test * n_tasks))

    if n_train + n_valid + n_test != n_tasks:

      raise ValueError("Train/Valid/Test fractions don't split tasks evenly.")



    X, y, w, ids = dataset.X, dataset.y, dataset.w, dataset.ids

    

    train_dataset = NumpyDataset(X, y[:,:n_train], w[:,:n_train], ids)

    valid_dataset = NumpyDataset(

        X, y[:,n_train:n_train+n_valid], w[:,n_train:n_train+n_valid], ids)

    test_dataset = NumpyDataset(

        X, y[:,n_train+n_valid:], w[:,n_train+n_valid:], ids)

    return train_dataset, valid_dataset, test_dataset



  def k_fold_split(self, dataset, K):

    """Performs a K-fold split of the tasks for dataset.



    Parameters

    ----------

    dataset: deepchem.datasets.Dataset

      Dataset to be split

    K: int

      Number of splits to be made

    """

    n_tasks = len(dataset.get_task_names())

    n_per_fold = int(np.round(n_tasks/float(K)))

    if K * n_per_fold != n_tasks:

      raise ValueError("Cannot perform a valid %d-way split" % K)

    

    X, y, w, ids = dataset.X, dataset.y, dataset.w, dataset.ids



    fold_datasets = []

    for fold in range(K):

      fold_tasks = range(fold*n_per_fold, (fold+1)*n_per_fold)

      fold_datasets.append(

          NumpyDataset(X, y[:, fold_tasks], w[:, fold_tasks], ids))

    return fold_datasets

deepchem/splits/tests/test_task_splitter.py

0 → 100644
+99 −0
Original line number Diff line number Diff line 


"""

Tests for splitter objects.

"""

from __future__ import division

from __future__ import print_function

from __future__ import unicode_literals



__author__ = "Bharath Ramsundar, Aneesh Pappu"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "GPL"



import tempfile

import numpy as np

from deepchem.splits.task_splitter import TaskSplitter

from deepchem.splits.task_splitter import merge_fold_datasets

from deepchem.datasets import NumpyDataset

from deepchem.datasets.tests import TestDatasetAPI





class TestTaskSplitters(TestDatasetAPI):

  """

  Test some basic splitters.

  """



  def test_multitask_train_valid_test_split(self):

    """

    Test TaskSplitter train/valid/test split on multitask dataset.

    """

    n_samples = 100

    n_features = 10

    n_tasks = 10

    X = np.random.rand(n_samples, n_features)

    p = .05 # proportion actives

    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    dataset = NumpyDataset(X, y)



    task_splitter = TaskSplitter()

    train, valid, test = task_splitter.train_valid_test_split(

        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)



    assert len(train.get_task_names()) == 4

    assert len(valid.get_task_names()) == 3

    assert len(test.get_task_names()) == 3



  def test_multitask_K_fold_split(self):

    """

    Test TaskSplitter K-fold split on multitask dataset.

    """

    n_samples = 100

    n_features = 10

    n_tasks = 10

    X = np.random.rand(n_samples, n_features)

    p = .05 # proportion actives

    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    dataset = NumpyDataset(X, y)

    K = 5



    task_splitter = TaskSplitter()

    fold_datasets = task_splitter.k_fold_split(dataset, K)



    for fold_dataset in fold_datasets:

      assert len(fold_dataset.get_task_names()) == 2



  def test_merge_fold_datasets(self):

    """

    Test that (K-1) folds can be merged into train dataset.

    """

    n_samples = 100

    n_features = 10

    n_tasks = 10

    X = np.random.rand(n_samples, n_features)

    p = .05 # proportion actives

    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = NumpyDataset(X, y, w)

    K = 5



    task_splitter = TaskSplitter()

    fold_datasets = task_splitter.k_fold_split(dataset, K)

    # Number tasks per fold

    n_per_fold = 2



    for fold in range(K):

      train_inds = list(set(range(K)) - set([fold]))

      train_fold_datasets = [fold_datasets[ind] for ind in train_inds]

      train_dataset = merge_fold_datasets(train_fold_datasets)



      # Find the tasks that correspond to this test fold

      train_tasks = list(

          set(range(10)) - set(range(fold*n_per_fold, (fold+1)*n_per_fold)))



      # Assert that all arrays look like they should

      np.testing.assert_array_equal(train_dataset.X, X)

      np.testing.assert_array_equal(

          train_dataset.y, y[:, train_tasks])

      np.testing.assert_array_equal(

          train_dataset.w, w[:, train_tasks])

      np.testing.assert_array_equal(train_dataset.X, X)

CRA Git | Maintained and supported by SUSTech CRA and CCSE