Commit 5ec720a7 authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #690 from lilleswing/k-fold 

New K-Fold interface for Splitter
parents 59b3c67c a6cf4fd7

deepchem/data/datasets.py

+17 −0
Original line number Diff line number Diff line
@@ -395,6 +395,23 @@ class NumpyDataset(Dataset): 
    ids = self.ids[indices]

    return NumpyDataset(X, y, w, ids)



  @staticmethod

  def from_DiskDataset(ds):

    """



    Parameters

    ----------

    ds : DiskDataset

    DiskDataset to transorm to NumpyDataset



    Returns

    -------

    NumpyDataset

      Data of ds as NumpyDataset



    """

    return NumpyDataset(ds.X, ds.y, ds.w, ds.ids)





class DiskDataset(Dataset):

  """

deepchem/splits/splitters.py

+53 −16
Original line number Diff line number Diff line
@@ -54,30 +54,67 @@ class Splitter(object): 
    self.verbose = verbose



  def k_fold_split(self, dataset, k, directories=None, **kwargs):

    """Does K-fold split of dataset."""

    """

    Parameters

    ----------

    dataset: Dataset

    Dataset to do a k-fold split



    k: int

    number of folds



    directories: list of str

    list of length 2*k filepaths to save the result disk-datasets



    kwargs



    Returns

    -------

    list of length k tuples of (train, cv)



    """

    """

    :param dataset: 

    :param k: 

    :param directories: 

    :param kwargs:

    :return: list of length k tuples of (train, cv)

    """

    log("Computing K-fold split", self.verbose)

    if directories is None:

      directories = [tempfile.mkdtemp() for _ in range(k)]

      directories = [tempfile.mkdtemp() for _ in range(2 * k)]

    else:

      assert len(directories) == k

    fold_datasets = []

      assert len(directories) == 2 * k

    cv_datasets = []

    train_ds_base = None

    train_datasets = []

    # rem_dataset is remaining portion of dataset

    rem_dataset = dataset

    for fold in range(k):

      # Note starts as 1/k since fold starts at 0. Ends at 1 since fold goes up

      # to k-1.

      frac_fold = 1. / (k - fold)

      fold_dir = directories[fold]

      train_dir, cv_dir = directories[2 * fold], directories[2 * fold + 1]

      fold_inds, rem_inds, _ = self.split(

          rem_dataset,

          frac_train=frac_fold,

          frac_valid=1 - frac_fold,

          frac_test=0)

      fold_dataset = rem_dataset.select(fold_inds, fold_dir)

      rem_dir = tempfile.mkdtemp()

      rem_dataset = rem_dataset.select(rem_inds, rem_dir)

      fold_datasets.append(fold_dataset)

    return fold_datasets

      cv_dataset = rem_dataset.select(fold_inds)

      cv_datasets.append(cv_dataset)

      rem_dataset = rem_dataset.select(rem_inds)



      train_ds_to_merge = filter(lambda x: x is not None,

                                 [train_ds_base, rem_dataset])

      train_ds_to_merge = filter(lambda x: len(x) > 0, train_ds_to_merge)

      train_dataset = DiskDataset.merge(train_ds_to_merge, merge_dir=train_dir)

      train_datasets.append(train_dataset)



      update_train_base_merge = filter(lambda x: x is not None,

                                       [train_ds_base, cv_dataset])

      train_ds_base = DiskDataset.merge(

          update_train_base_merge, merge_dir=cv_dir)

    return list(zip(train_datasets, cv_datasets))



  def train_valid_test_split(self,

                             dataset,
@@ -220,8 +257,8 @@ class RandomStratifiedSplitter(Splitter): 
      dataset_1 = NumpyDataset(dataset.X, dataset.y, dataset.w, dataset.ids)

      dataset_2 = None

      return dataset_1, dataset_2

    X, y, w, ids = randomize_arrays(

        (dataset.X, dataset.y, dataset.w, dataset.ids))

    X, y, w, ids = randomize_arrays((dataset.X, dataset.y, dataset.w,

                                     dataset.ids))

    split_indices = self.get_task_split_indices(y, w, frac_split)



    # Create weight matrices fpor two haves.
@@ -261,8 +298,8 @@ class RandomStratifiedSplitter(Splitter): 
    if test_dir is None:

      test_dir = tempfile.mkdtemp()

    # Obtain original x, y, and w arrays and shuffle

    X, y, w, ids = randomize_arrays(

        (dataset.X, dataset.y, dataset.w, dataset.ids))

    X, y, w, ids = randomize_arrays((dataset.X, dataset.y, dataset.w,

                                     dataset.ids))

    rem_dir = tempfile.mkdtemp()

    train_dataset, rem_dataset = self.split(dataset, frac_train,

                                            [train_dir, rem_dir])
@@ -720,8 +757,8 @@ class FingerprintSplitter(Splitter): 
    distances = np.ones(shape=(data_len, data_len))

    for i in range(data_len):

      for j in range(data_len):

        distances[i][j] = 1 - DataStructs.FingerprintSimilarity(fingerprints[i],

                                                                fingerprints[j])

        distances[i][j] = 1 - DataStructs.FingerprintSimilarity(

            fingerprints[i], fingerprints[j])



    train_cutoff = int(frac_train * len(dataset))

    valid_cutoff = int(frac_valid * len(dataset))

deepchem/splits/tests/test_splitter.py

+27 −12
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ import tempfile 
import unittest

import numpy as np

import deepchem as dc

from deepchem.data import NumpyDataset

from deepchem.splits import IndexSplitter

from rdkit import Chem, DataStructs
@@ -122,6 +124,22 @@ class TestSplitters(unittest.TestCase): 
    assert len(valid_data) == 3

    assert len(test_data) == 0



  def test_k_fold_splitter(self):

    """

    Test that we can 5 fold index wise over 5 points

    """

    ds = NumpyDataset(np.array(range(5)), np.array(range(5)))

    index_splitter = IndexSplitter()



    K = 5

    fold_datasets = index_splitter.k_fold_split(ds, K)



    for fold in range(K):

      self.assertTrue(fold_datasets[fold][1].X[0] == fold)

      train_data = set(list(fold_datasets[fold][0].X))

      self.assertFalse(fold in train_data)

      self.assertEqual(K - 1, len(train_data))



  def test_singletask_random_k_fold_split(self):

    """

    Test singletask RandomSplitter class.
@@ -133,7 +151,7 @@ class TestSplitters(unittest.TestCase): 
    K = 5

    fold_datasets = random_splitter.k_fold_split(solubility_dataset, K)

    for fold in range(K):

      fold_dataset = fold_datasets[fold]

      fold_dataset = fold_datasets[fold][1]

      # Verify lengths is 10/k == 2

      assert len(fold_dataset) == 2

      # Verify that compounds in this fold are subset of original compounds
@@ -143,14 +161,10 @@ class TestSplitters(unittest.TestCase): 
      for other_fold in range(K):

        if fold == other_fold:

          continue

        other_fold_dataset = fold_datasets[other_fold]

        other_fold_dataset = fold_datasets[other_fold][1]

        other_fold_ids_set = set(other_fold_dataset.ids)

        assert fold_ids_set.isdisjoint(other_fold_ids_set)



    merged_dataset = dc.data.DiskDataset.merge(fold_datasets)

    assert len(merged_dataset) == len(solubility_dataset)

    assert sorted(merged_dataset.ids) == (sorted(solubility_dataset.ids))



  def test_singletask_index_k_fold_split(self):

    """

    Test singletask IndexSplitter class.
@@ -163,7 +177,7 @@ class TestSplitters(unittest.TestCase): 
    fold_datasets = index_splitter.k_fold_split(solubility_dataset, K)



    for fold in range(K):

      fold_dataset = fold_datasets[fold]

      fold_dataset = fold_datasets[fold][1]

      # Verify lengths is 10/k == 2

      assert len(fold_dataset) == 2

      # Verify that compounds in this fold are subset of original compounds
@@ -173,11 +187,11 @@ class TestSplitters(unittest.TestCase): 
      for other_fold in range(K):

        if fold == other_fold:

          continue

        other_fold_dataset = fold_datasets[other_fold]

        other_fold_dataset = fold_datasets[other_fold][1]

        other_fold_ids_set = set(other_fold_dataset.ids)

        assert fold_ids_set.isdisjoint(other_fold_ids_set)



    merged_dataset = dc.data.DiskDataset.merge(fold_datasets)

    merged_dataset = dc.data.DiskDataset.merge([x[1] for x in fold_datasets])

    assert len(merged_dataset) == len(solubility_dataset)

    assert sorted(merged_dataset.ids) == (sorted(solubility_dataset.ids))
@@ -193,7 +207,7 @@ class TestSplitters(unittest.TestCase): 
    fold_datasets = scaffold_splitter.k_fold_split(solubility_dataset, K)



    for fold in range(K):

      fold_dataset = fold_datasets[fold]

      fold_dataset = fold_datasets[fold][1]

      # Verify lengths is 10/k == 2

      assert len(fold_dataset) == 2

      # Verify that compounds in this fold are subset of original compounds
@@ -203,11 +217,11 @@ class TestSplitters(unittest.TestCase): 
      for other_fold in range(K):

        if fold == other_fold:

          continue

        other_fold_dataset = fold_datasets[other_fold]

        other_fold_dataset = fold_datasets[other_fold][1]

        other_fold_ids_set = set(other_fold_dataset.ids)

        assert fold_ids_set.isdisjoint(other_fold_ids_set)



    merged_dataset = dc.data.DiskDataset.merge(fold_datasets)

    merged_dataset = dc.data.DiskDataset.merge([x[1] for x in fold_datasets])

    assert len(merged_dataset) == len(solubility_dataset)

    assert sorted(merged_dataset.ids) == (sorted(solubility_dataset.ids))
@@ -453,4 +467,5 @@ class TestSplitters(unittest.TestCase): 


if __name__ == "__main__":

  import nose



  nose.run(defaultTest=__name__)

CRA Git | Maintained and supported by SUSTech CRA and CCSE