Commit c6877dcc authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

More refactor

parent f5cfcea4

deepchem/datasets/datasets.py

+16 −8
Original line number Diff line number Diff line
@@ -684,9 +684,11 @@ class DiskDataset(Dataset): 
    self.save_to_disk()



  @staticmethod

  def from_numpy(data_dir, X, y, w=None, ids=None, tasks=None, verbosity=None,

                 compute_feature_statistics=True):

  def from_numpy(X, y, w=None, ids=None, data_dir=None, tasks=None,

                 verbosity=None, compute_feature_statistics=True):

    """Creates a DiskDataset object from specified Numpy arrays."""

    if data_dir is None:

      data_dir = tempfile.mkdtemp()

    n_samples = len(X)

    # The -1 indicates that y will be reshaped to have length -1

    if n_samples > 0:
@@ -706,10 +708,13 @@ class DiskDataset(Dataset): 
                   compute_feature_statistics=compute_feature_statistics)



  @staticmethod

  def merge(merge_dir, datasets):

  def merge(datasets, merge_dir=None):

    """Merges provided datasets into a merged dataset."""

    if merge_dir is not None:

      if not os.path.exists(merge_dir):

        os.makedirs(merge_dir)

    else:

      merge_dir = tempfile.mkdtemp()

    Xs, ys, ws, all_ids = [], [], [], []

    metadata_rows = []

    for ind, dataset in enumerate(datasets):
@@ -722,10 +727,13 @@ class DiskDataset(Dataset): 
                   metadata_rows=metadata_rows,

                   verbosity=dataset.verbosity)



  def subset(self, subset_dir, shard_nums):

  def subset(self, shard_nums, subset_dir=None):

    """Creates a subset of the original dataset on disk."""

    if subset_dir is not None:

      if not os.path.exists(subset_dir):

        os.makedirs(subset_dir)

    else:

      subset_dir = tempfile.mkdtemp()

    tasks = self.get_task_names()

    metadata_rows = []

    for shard_num, row in self.metadata_df.iterrows():

deepchem/datasets/tests/__init__.py

+50 −73
Original line number Diff line number Diff line
@@ -15,12 +15,6 @@ import os 
import shutil

import numpy as np

import deepchem as dc

#from deepchem.datasets import Dataset

#from deepchem.featurizers.featurize import DataLoader

#from deepchem.featurizers.fingerprints import CircularFingerprint

#from deepchem.featurizers import UserDefinedFeaturizer

#from deepchem.transformers import NormalizationTransformer

#from deepchem.models.tests import TestAPI



def load_solubility_data():

  """Loads solubility dataset"""
@@ -52,74 +46,57 @@ def load_multitask_data(): 
      verbosity="low")

  return loader.featurize(input_file)



class TestDatasets(unittest.TestCase):

  """

  Shared API for testing with dataset objects.

  """



def load_classification_data(self):

  """Loads classification data from example.csv"""

    if os.path.exists(self.data_dir):

      shutil.rmtree(self.data_dir)

    featurizer = CircularFingerprint(size=1024)

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.featurizers.CircularFingerprint(size=1024)

  tasks = ["outcome"]

  task_type = "classification"

  input_file = os.path.join(

        self.current_dir, "../../models/tests/example_classification.csv")

    loader = DataLoader(

        tasks=tasks,

        smiles_field=self.smiles_field,

        featurizer=featurizer,

        verbosity="low")

    return loader.featurize(input_file, self.data_dir)

      current_dir, "../../models/tests/example_classification.csv")

  loader = dc.loaders.DataLoader(

      tasks=tasks, smiles_field="smiles",

      featurizer=featurizer, verbosity="low")

  return loader.featurize(input_file)





  def load_sparse_multitask_dataset(self):

def load_sparse_multitask_dataset():

  """Load sparse tox multitask data, sample dataset."""

    if os.path.exists(self.data_dir):

      shutil.rmtree(self.data_dir)

    featurizer = CircularFingerprint(size=1024)

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.featurizers.CircularFingerprint(size=1024)

  tasks = ["task1", "task2", "task3", "task4", "task5", "task6",

           "task7", "task8", "task9"]

  input_file = os.path.join(

        self.current_dir, "../../models/tests/sparse_multitask_example.csv")

    loader = DataLoader(

        tasks=tasks,

        smiles_field="smiles",

        featurizer=featurizer,

        verbosity="low")

    return loader.featurize(input_file, self.data_dir)

      current_dir, "../../models/tests/sparse_multitask_example.csv")

  loader = dc.loaders.DataLoader(

      tasks=tasks, smiles_field="smiles",

      featurizer=featurizer, verbosity="low")

  return loader.featurize(input_file)

  

def load_feat_multitask_data(self):

  """Load example with numerical features, tasks."""

    if os.path.exists(self.data_dir):

      shutil.rmtree(self.data_dir)

  current_dir = os.path.dirname(os.path.abspath(__file__))

  features = ["feat0", "feat1", "feat2", "feat3", "feat4", "feat5"]

    featurizer = UserDefinedFeaturizer(features)

  featurizer = dc.featurizers.UserDefinedFeaturizer(features)

  tasks = ["task0", "task1", "task2", "task3", "task4", "task5"]

  input_file = os.path.join(

        self.current_dir, "../../models/tests/feat_multitask_example.csv")

    loader = DataLoader(

        tasks=tasks,

        featurizer=featurizer,

        id_field="id",

        verbosity="low")

    return loader.featurize(input_file, self.data_dir)

      current_dir, "../../models/tests/feat_multitask_example.csv")

  loader = dc.loaders.DataLoader(

      tasks=tasks, featurizer=featurizer,

      id_field="id", verbosity="low")

  return loader.featurize(input_file)



def load_gaussian_cdf_data(self):

  """Load example with numbers sampled from Gaussian normal distribution.

     Each feature and task is a column of values that is sampled

     from a normal distribution of mean 0, stdev 1."""

    if os.path.exists(self.data_dir):

      shutil.rmtree(self.data_dir)

  current_dir = os.path.dirname(os.path.abspath(__file__))

  features = ["feat0","feat1"]

    featurizer = UserDefinedFeaturizer(features)

  featurizer = dc.featurizers.UserDefinedFeaturizer(features)

  tasks = ["task0","task1"]

  input_file = os.path.join(

        self.current_dir, "../../models/tests/gaussian_cdf_example.csv")

    loader = DataLoader(

        tasks=tasks,

        featurizer=featurizer,

        id_field="id",

        verbosity=None)

    return loader.featurize(input_file, self.data_dir)
 
      current_dir, "../../models/tests/gaussian_cdf_example.csv")

  loader = dc.loaders.DataLoader(

      tasks=tasks, featurizer=featurizer,

      id_field="id", verbosity=None)

  return loader.featurize(input_file)

deepchem/datasets/tests/test_merge.py

+19 −29
Original line number Diff line number Diff line
@@ -12,14 +12,11 @@ __license__ = "GPL" 
import os

import shutil

import tempfile

import unittest

import deepchem as dc

import numpy as np

from deepchem.models.tests import TestAPI

from deepchem.utils.save import load_from_disk

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.featurizers.featurize import DataLoader

from deepchem.datasets import DiskDataset



class TestMerge(TestAPI):

class TestMerge(unittest.TestCase):

  """

  Test singletask/multitask dataset merging.

  """
@@ -27,26 +24,20 @@ class TestMerge(TestAPI): 
    """Test that datasets can be merged."""

    verbosity = "high"

    current_dir = os.path.dirname(os.path.realpath(__file__))

    first_data_dir = os.path.join(self.base_dir, "first_dataset")

    second_data_dir = os.path.join(self.base_dir, "second_dataset")

    merged_data_dir = os.path.join(self.base_dir, "merged_data")



    dataset_file = os.path.join(

        current_dir, "../../models/tests/example.csv")



    featurizer = CircularFingerprint(size=1024)

    featurizer = dc.featurizers.CircularFingerprint(size=1024)

    tasks = ["log-solubility"]

    loader = DataLoader(tasks=tasks,

                        smiles_field="smiles",

                        featurizer=featurizer,

                        verbosity=verbosity)

    first_dataset = loader.featurize(

        dataset_file, first_data_dir)

    second_dataset = loader.featurize(

        dataset_file, second_data_dir)

    loader = dc.loaders.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity=verbosity)

    first_dataset = loader.featurize(dataset_file)

    second_dataset = loader.featurize( dataset_file)



    merged_dataset = DiskDataset.merge(

        merged_data_dir, [first_dataset, second_dataset])

    merged_dataset = dc.datasets.DiskDataset.merge(

        [first_dataset, second_dataset])



    assert len(merged_dataset) == len(first_dataset) + len(second_dataset)
@@ -54,20 +45,19 @@ class TestMerge(TestAPI): 
    """Tests that subsetting of datasets works."""

    verbosity = "high"

    current_dir = os.path.dirname(os.path.realpath(__file__))

    data_dir = os.path.join(self.base_dir, "dataset")

    subset_dir = os.path.join(self.base_dir, "subset")

    #data_dir = os.path.join(self.base_dir, "dataset")

    #subset_dir = os.path.join(self.base_dir, "subset")



    dataset_file = os.path.join(

        current_dir, "../../models/tests/example.csv")



    featurizer = CircularFingerprint(size=1024)

    featurizer = dc.featurizers.CircularFingerprint(size=1024)

    tasks = ["log-solubility"]

    loader = DataLoader(tasks=tasks,

                        smiles_field="smiles",

                        featurizer=featurizer,

                        verbosity=verbosity)

    loader = dc.loaders.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity=verbosity)

    dataset = loader.featurize(

        dataset_file, data_dir, shard_size=2)

        dataset_file, shard_size=2)



    shard_nums = [1, 2]
@@ -75,7 +65,7 @@ class TestMerge(TestAPI): 
    _, _, _, ids_1 = dataset.get_shard(1)

    _, _, _, ids_2 = dataset.get_shard(2)



    subset = dataset.subset(subset_dir, shard_nums)

    subset = dataset.subset(shard_nums)

    after_ids = dataset.ids



    assert len(subset) == 4

deepchem/datasets/tests/test_reload.py

+16 −31
Original line number Diff line number Diff line
@@ -11,69 +11,54 @@ __license__ = "GPL" 


import os

import shutil

import unittest

import tempfile

import deepchem as dc

import numpy as np

from sklearn.linear_model import LogisticRegression

from deepchem.models.tests import TestAPI

from deepchem.utils.save import load_from_disk

from deepchem.datasets import Dataset

from deepchem.featurizers.featurize import DataLoader

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.splits import ScaffoldSplitter

from deepchem.datasets import Dataset

from deepchem.transformers import BalancingTransformer

from deepchem.hyperparameters import HyperparamOpt

from deepchem.models.multitask import SingletaskToMultitask

from deepchem import metrics

from deepchem.metrics import Metric

from deepchem.models.sklearn_models import SklearnModel

from deepchem.utils.evaluate import relative_difference



class TestReload(TestAPI):

class TestReload(unittest.TestCase):

  """

  Test reload for datasets.

  """

  def _run_muv_experiment(self, dataset_file, reload=False, verbosity=None):

    """Loads or reloads a small version of MUV dataset."""

    # Load MUV dataset

    raw_dataset = load_from_disk(dataset_file)

    print("Number of examples in dataset: %s" % str(raw_dataset.shape[0]))



    print("About to featurize compounds")

    featurizer = CircularFingerprint(size=1024)

    featurizer = dc.featurizers.CircularFingerprint(size=1024)

    raw_dataset = dc.utils.save.load_from_disk(dataset_file)

    MUV_tasks = ['MUV-692', 'MUV-689', 'MUV-846', 'MUV-859', 'MUV-644',

                 'MUV-548', 'MUV-852', 'MUV-600', 'MUV-810', 'MUV-712',

                 'MUV-737', 'MUV-858', 'MUV-713', 'MUV-733', 'MUV-652',

                 'MUV-466', 'MUV-832']

    loader = DataLoader(tasks=MUV_tasks,

                        smiles_field="smiles",

                        featurizer=featurizer,

                        verbosity=verbosity)

    dataset = loader.featurize(dataset_file, self.data_dir)

    loader = dc.loaders.DataLoader(

        tasks=MUV_tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity=verbosity)

    dataset = loader.featurize(dataset_file)

    assert len(dataset) == len(raw_dataset)



    print("About to split compounds into train/valid/test")

    splitter = ScaffoldSplitter(verbosity=verbosity)

    splitter = dc.splits.ScaffoldSplitter(verbosity=verbosity)

    frac_train, frac_valid, frac_test = .8, .1, .1

    train_dataset, valid_dataset, test_dataset = \

        splitter.train_valid_test_split(

            dataset, self.train_dir, self.valid_dir, self.test_dir,

            log_every_n=1000, frac_train=frac_train,

            dataset, log_every_n=1000, frac_train=frac_train,

            frac_test=frac_test, frac_valid=frac_valid)

    # Do an approximate comparison since splits are sometimes slightly off from

    # the exact fraction.

    assert relative_difference(

    assert dc.utils.evaluate.relative_difference(

        len(train_dataset), frac_train * len(dataset)) < 1e-3

    assert relative_difference(

    assert dc.utils.evaluate.relative_difference(

        len(valid_dataset), frac_valid * len(dataset)) < 1e-3

    assert relative_difference(

    assert dc.utils.evaluate.relative_difference(

        len(test_dataset), frac_test * len(dataset)) < 1e-3



    # TODO(rbharath): Transformers don't play nice with reload! Namely,

    # reloading will cause the transform to be reapplied. This is undesirable in

    # almost all cases. Need to understand a method to fix this.

    transformers = [

        BalancingTransformer(transform_w=True, dataset=train_dataset)]

        dc.transformers.BalancingTransformer(

            transform_w=True, dataset=train_dataset)]

    print("Transforming datasets")

    for dataset in [train_dataset, valid_dataset, test_dataset]:

      for transformer in transformers:

deepchem/datasets/tests/test_shuffle.py

+34 −35
Original line number Diff line number Diff line
@@ -12,14 +12,11 @@ __license__ = "GPL" 
import os

import shutil

import tempfile

import unittest

import deepchem as dc

import numpy as np

from deepchem.models.tests import TestAPI

from deepchem.utils.save import load_from_disk

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.featurizers.featurize import DataLoader

from deepchem.datasets import DiskDataset



class TestShuffle(TestAPI):

class TestShuffle(unittest.TestCase):

  """

  Test singletask/multitask dataset shuffling.

  """
@@ -27,25 +24,25 @@ class TestShuffle(TestAPI): 
    """Test that datasets can be merged."""

    verbosity = "high"

    current_dir = os.path.dirname(os.path.realpath(__file__))

    data_dir = os.path.join(self.base_dir, "dataset")



    dataset_file = os.path.join(

        current_dir, "../../models/tests/example.csv")



    featurizer = CircularFingerprint(size=1024)

    featurizer = dc.featurizers.CircularFingerprint(size=1024)

    tasks = ["log-solubility"]

    loader = DataLoader(tasks=tasks,

                        smiles_field="smiles",

                        featurizer=featurizer,

                        verbosity=verbosity)

    loader = dc.loaders.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity=verbosity)

    dataset = loader.featurize(

        dataset_file, data_dir, shard_size=2)

        dataset_file, shard_size=2)



    X_orig, y_orig, w_orig, orig_ids = (dataset.X, dataset.y, dataset.w, dataset.ids)

    X_orig, y_orig, w_orig, orig_ids = (dataset.X, dataset.y, dataset.w,

                                        dataset.ids)

    orig_len = len(dataset)



    dataset.shuffle(iterations=5)

    X_new, y_new, w_new, new_ids = (dataset.X, dataset.y, dataset.w, dataset.ids)

    X_new, y_new, w_new, new_ids = (dataset.X, dataset.y, dataset.w,

                                    dataset.ids)

    

    assert len(dataset) == orig_len

    # The shuffling should have switched up the ordering
@@ -61,25 +58,25 @@ class TestShuffle(TestAPI): 
    """Test that sparse datasets can be shuffled quickly."""

    verbosity = "high"

    current_dir = os.path.dirname(os.path.realpath(__file__))

    data_dir = os.path.join(self.base_dir, "dataset")



    dataset_file = os.path.join(

        current_dir, "../../models/tests/example.csv")



    featurizer = CircularFingerprint(size=1024)

    featurizer = dc.featurizers.CircularFingerprint(size=1024)

    tasks = ["log-solubility"]

    loader = DataLoader(tasks=tasks,

                        smiles_field="smiles",

                        featurizer=featurizer,

                        verbosity=verbosity)

    loader = dc.loaders.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity=verbosity)

    dataset = loader.featurize(

        dataset_file, data_dir, shard_size=2)

        dataset_file, shard_size=2)



    X_orig, y_orig, w_orig, orig_ids = (dataset.X, dataset.y, dataset.w, dataset.ids)

    X_orig, y_orig, w_orig, orig_ids = (dataset.X, dataset.y, dataset.w,

                                        dataset.ids)

    orig_len = len(dataset)



    dataset.sparse_shuffle()

    X_new, y_new, w_new, new_ids = (dataset.X, dataset.y, dataset.w, dataset.ids)

    X_new, y_new, w_new, new_ids = (dataset.X, dataset.y, dataset.w,

                                    dataset.ids)

    

    assert len(dataset) == orig_len

    # The shuffling should have switched up the ordering
@@ -95,25 +92,25 @@ class TestShuffle(TestAPI): 
    """Test that datasets can be merged."""

    verbosity = "high"

    current_dir = os.path.dirname(os.path.realpath(__file__))

    data_dir = os.path.join(self.base_dir, "dataset")



    dataset_file = os.path.join(

        current_dir, "../../models/tests/example.csv")



    featurizer = CircularFingerprint(size=1024)

    featurizer = dc.featurizers.CircularFingerprint(size=1024)

    tasks = ["log-solubility"]

    loader = DataLoader(tasks=tasks,

                        smiles_field="smiles",

                        featurizer=featurizer,

                        verbosity=verbosity)

    loader = dc.loaders.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity=verbosity)

    dataset = loader.featurize(

        dataset_file, data_dir, shard_size=2)

        dataset_file, shard_size=2)



    X_orig, y_orig, w_orig, orig_ids = (dataset.X, dataset.y, dataset.w, dataset.ids)

    X_orig, y_orig, w_orig, orig_ids = (dataset.X, dataset.y, dataset.w,

                                        dataset.ids)

    orig_len = len(dataset)



    dataset.reshard_shuffle(reshard_size=1)

    X_new, y_new, w_new, new_ids = (dataset.X, dataset.y, dataset.w, dataset.ids)

    X_new, y_new, w_new, new_ids = (dataset.X, dataset.y, dataset.w,

                                    dataset.ids)

    

    assert len(dataset) == orig_len

    # The shuffling should have switched up the ordering
@@ -135,7 +132,8 @@ class TestShuffle(TestAPI): 
    y = np.random.randint(2, size=(n_samples, n_tasks))

    w = np.random.randint(2, size=(n_samples, n_tasks))

    ids = np.arange(n_samples)

    dataset = DiskDataset.from_numpy(self.data_dir, X, y, w, ids)

    dataset = dc.datasets.DiskDataset.from_numpy(tempfile.mkdtemp(), X, y, w,

                                                 ids)

    dataset.reshard(shard_size=10)



    dataset.shuffle_each_shard()
@@ -163,7 +161,8 @@ class TestShuffle(TestAPI): 
    y = np.random.randint(2, size=(n_samples, n_tasks))

    w = np.random.randint(2, size=(n_samples, n_tasks))

    ids = np.arange(n_samples)

    dataset = DiskDataset.from_numpy(self.data_dir, X, y, w, ids)

    dataset = dc.datasets.DiskDataset.from_numpy(tempfile.mkdtemp(), X, y, w,

                                                 ids)

    dataset.reshard(shard_size=10)

    dataset.shuffle_shards()

CRA Git | Maintained and supported by SUSTech CRA and CCSE