Commit b4d052fd authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Debugging failing tests

parent c6877dcc

deepchem/datasets/datasets.py

+3 −2
Original line number Diff line number Diff line
@@ -684,8 +684,9 @@ class DiskDataset(Dataset): 
    self.save_to_disk()



  @staticmethod

  def from_numpy(X, y, w=None, ids=None, data_dir=None, tasks=None,

                 verbosity=None, compute_feature_statistics=True):

  def from_numpy(X, y, w=None, ids=None, tasks=None,

                 verbosity=None, compute_feature_statistics=True,

                 data_dir=None):

    """Creates a DiskDataset object from specified Numpy arrays."""

    if data_dir is None:

      data_dir = tempfile.mkdtemp()

deepchem/datasets/tests/test_datasets.py

+1 −2
Original line number Diff line number Diff line
@@ -224,8 +224,7 @@ class TestBasicDatasets(unittest.TestCase): 
    y = np.random.randint(2, size=(num_datapoints, num_tasks))

    w = np.ones((num_datapoints, num_tasks))

    ids = np.array(["id"] * num_datapoints)

    dataset = dc.datasets.DiskDataset.from_numpy(

        tempfile.mkdtemp(), X, y, w, ids)

    dataset = dc.datasets.DiskDataset.from_numpy(X, y, w, ids)



    indices = [0, 4, 5, 8]

    select_dataset = dataset.select(indices)

deepchem/datasets/tests/test_shuffle.py

+2 −4
Original line number Diff line number Diff line
@@ -132,8 +132,7 @@ class TestShuffle(unittest.TestCase): 
    y = np.random.randint(2, size=(n_samples, n_tasks))

    w = np.random.randint(2, size=(n_samples, n_tasks))

    ids = np.arange(n_samples)

    dataset = dc.datasets.DiskDataset.from_numpy(tempfile.mkdtemp(), X, y, w,

                                                 ids)

    dataset = dc.datasets.DiskDataset.from_numpy(X, y, w, ids)

    dataset.reshard(shard_size=10)



    dataset.shuffle_each_shard()
@@ -161,8 +160,7 @@ class TestShuffle(unittest.TestCase): 
    y = np.random.randint(2, size=(n_samples, n_tasks))

    w = np.random.randint(2, size=(n_samples, n_tasks))

    ids = np.arange(n_samples)

    dataset = dc.datasets.DiskDataset.from_numpy(tempfile.mkdtemp(), X, y, w,

                                                 ids)

    dataset = dc.datasets.DiskDataset.from_numpy(X, y, w, ids)

    dataset.reshard(shard_size=10)

    dataset.shuffle_shards()

deepchem/hyperparameters/tests/test_hyperparam_opt.py

+59 −70
Original line number Diff line number Diff line
@@ -14,58 +14,45 @@ import unittest 
import tempfile

import shutil

import numpy as np

from deepchem.models.tests import TestAPI

from deepchem.models.sklearn_models import SklearnModel

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.featurizers.featurize import DataLoader

from deepchem.transformers import NormalizationTransformer

from deepchem import metrics

from deepchem.metrics import Metric

from deepchem.models.multitask import SingletaskToMultitask 

from sklearn.ensemble import RandomForestClassifier

from sklearn.ensemble import RandomForestRegressor

from deepchem.datasets import DiskDataset, NumpyDataset

from deepchem.hyperparameters import HyperparamOpt

from deepchem.models.keras_models.fcnet import MultiTaskDNN

from deepchem.models.keras_models import KerasModel

from deepchem.models.tensorflow_models import TensorflowModel

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

from deepchem.splits import ScaffoldSplitter

import tensorflow as tf

from keras import backend as K

import deepchem as dc

from sklearn.ensemble import RandomForestClassifier

from sklearn.ensemble import RandomForestRegressor



class TestHyperparamOptAPI(TestAPI):

class TestHyperparamOptAPI(unittest.TestCase):

  """

  Test hyperparameter optimization API.

  """

  def test_singletask_sklearn_rf_ECFP_regression_hyperparam_opt(self):

    """Test of hyperparam_opt with singletask RF ECFP regression API."""

    featurizer = CircularFingerprint(size=1024)

    featurizer = dc.featurizers.CircularFingerprint(size=1024)

    tasks = ["log-solubility"]

    input_file = os.path.join(self.current_dir, "example.csv")

    loader = DataLoader(tasks=tasks,

                        smiles_field=self.smiles_field,

                        featurizer=featurizer,

                        verbosity="low")

    dataset = loader.featurize(input_file, self.data_dir)



    splitter = ScaffoldSplitter()

    current_dir = os.path.dirname(os.path.abspath(__file__))

    input_file = os.path.join(current_dir, "../../models/tests/example.csv")

    loader = dc.loaders.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity="low")

    dataset = loader.featurize(input_file)



    splitter = dc.splits.ScaffoldSplitter()

    train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(

        dataset, self.train_dir, self.valid_dir, self.test_dir)

        dataset)



    transformers = [

        NormalizationTransformer(transform_y=True, dataset=train_dataset)]

        dc.transformers.NormalizationTransformer(

            transform_y=True, dataset=train_dataset)]

    for dataset in [train_dataset, test_dataset]:

      for transformer in transformers:

        dataset = transformer.transform(dataset)



    params_dict = {"n_estimators": [10, 100]}

    metric = Metric(metrics.r2_score)

    metric = dc.metrics.Metric(dc.metrics.r2_score)

    def rf_model_builder(model_params, model_dir):

      sklearn_model = RandomForestRegressor(**model_params)

      return SklearnModel(sklearn_model, model_dir)

      return dc.models.SklearnModel(sklearn_model, model_dir)



    optimizer = HyperparamOpt(rf_model_builder, verbosity="low")

    optimizer = dc.hyperparameters.HyperparamOpt(rf_model_builder)

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, transformers,

      metric, logdir=None)
@@ -86,9 +73,8 @@ class TestHyperparamOptAPI(TestAPI): 
    w_train = np.ones_like(y_train)

    ids_train = ["C"] * n_train



    train_dataset = DiskDataset.from_numpy(self.train_dir,

                                           X_train, y_train, w_train, ids_train,

                                           tasks) 

    train_dataset = dc.datasets.DiskDataset.from_numpy(

        X_train, y_train, w_train, ids_train, tasks) 



    # Define validation dataset

    n_valid = 10
@@ -96,21 +82,20 @@ class TestHyperparamOptAPI(TestAPI): 
    y_valid = np.random.randint(2, size=(n_valid, n_tasks))

    w_valid = np.ones_like(y_valid)

    ids_valid = ["C"] * n_valid

    valid_dataset = DiskDataset.from_numpy(self.valid_dir,

                                           X_valid, y_valid, w_valid, ids_valid,

                                           tasks)

    valid_dataset = dc.datasets.DiskDataset.from_numpy(

        X_valid, y_valid, w_valid, ids_valid, tasks)



    transformers = []

    classification_metric = Metric(metrics.matthews_corrcoef, np.mean,

                                   mode="classification")

    classification_metric = dc.metrics.Metric(

        dc.metrics.matthews_corrcoef, np.mean, mode="classification")

    params_dict = {"n_estimators": [1, 10]}

    def multitask_model_builder(model_params, model_dir):

      def model_builder(model_dir):

        sklearn_model = RandomForestClassifier(**model_params)

        return SklearnModel(sklearn_model, model_dir)

      return SingletaskToMultitask(tasks, model_builder, model_dir)

        return dc.models.SklearnModel(sklearn_model, model_dir)

      return dc.models.SingletaskToMultitask(tasks, model_builder, model_dir)



    optimizer = HyperparamOpt(multitask_model_builder, verbosity="low")

    optimizer = dc.hyperparameters.HyperparamOpt(multitask_model_builder)

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, transformers,

      classification_metric, logdir=None)
@@ -118,32 +103,34 @@ class TestHyperparamOptAPI(TestAPI): 
  def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self):

    """Straightforward test of Keras multitask deepchem classification API."""

    task_type = "classification"

    input_file = os.path.join(self.current_dir, "multitask_example.csv")

    current_dir = os.path.dirname(os.path.abspath(__file__))

    input_file = os.path.join(

        current_dir, "../../models/tests/multitask_example.csv")

    tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",

             "task7", "task8", "task9", "task10", "task11", "task12",

             "task13", "task14", "task15", "task16"]



    n_features = 1024

    featurizer = CircularFingerprint(size=n_features)

    loader = DataLoader(tasks=tasks,

                        smiles_field=self.smiles_field,

                        featurizer=featurizer,

                        verbosity="low")

    dataset = loader.featurize(input_file, self.data_dir)



    splitter = ScaffoldSplitter()

    featurizer = dc.featurizers.CircularFingerprint(size=n_features)

    loader = dc.loaders.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity="low")

    dataset = loader.featurize(input_file)



    splitter = dc.splits.ScaffoldSplitter()

    train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(

        dataset, self.train_dir, self.valid_dir, self.test_dir)

        dataset)



    transformers = []

    metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification")

    metric = dc.metrics.Metric(

        dc.metrics.matthews_corrcoef, np.mean, mode="classification")

    params_dict= {"n_hidden": [5, 10]}

      

    def model_builder(model_params, model_dir):

      keras_model = MultiTaskDNN(

      keras_model = dc.models.MultiTaskDNN(

          len(tasks), n_features, task_type, dropout=0., **model_params)

      return KerasModel(keras_model, model_dir)

    optimizer = HyperparamOpt(model_builder, verbosity="low")

      return dc.models.KerasModel(keras_model, model_dir)

    optimizer = dc.hyperparameters.HyperparamOpt(model_builder)

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, transformers,

      metric, logdir=None)
@@ -152,34 +139,36 @@ class TestHyperparamOptAPI(TestAPI): 
    """Straightforward test of Tensorflow multitask deepchem classification API."""

    task_type = "classification"



    input_file = os.path.join(self.current_dir, "multitask_example.csv")

    current_dir = os.path.dirname(os.path.abspath(__file__))

    input_file = os.path.join(

        current_dir, "../../models/tests/multitask_example.csv")

    tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",

             "task7", "task8", "task9", "task10", "task11", "task12",

             "task13", "task14", "task15", "task16"]



    n_features = 1024

    featurizer = CircularFingerprint(size=n_features)

    featurizer = dc.featurizers.CircularFingerprint(size=n_features)



    loader = DataLoader(tasks=tasks,

                        smiles_field=self.smiles_field,

                        featurizer=featurizer,

                        verbosity="low")

    dataset = loader.featurize(input_file, self.data_dir)

    loader = dc.loaders.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer, verbosity="low")

    dataset = loader.featurize(input_file)



    splitter = ScaffoldSplitter()

    splitter = dc.splits.ScaffoldSplitter()

    train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(

        dataset, self.train_dir, self.valid_dir, self.test_dir)

        dataset)





    transformers = []

    metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification")

    metric = dc.metrics.Metric(

        dc.metrics.matthews_corrcoef, np.mean, mode="classification")

    params_dict = {"layer_sizes": [(10,), (100,)]}



    def model_builder(model_params, model_dir):

        tensorflow_model = TensorflowMultiTaskClassifier(

        tensorflow_model = dc.models.TensorflowMultiTaskClassifier(

            len(tasks), n_features, model_dir, **model_params)

        return TensorflowModel(tensorflow_model, model_dir)

    optimizer = HyperparamOpt(model_builder, verbosity="low")

        return dc.models.TensorflowModel(tensorflow_model)

    optimizer = dc.hyperparameters.HyperparamOpt(model_builder)

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, transformers, metric,

      logdir=None)

deepchem/metrics/tests/metrics_test.py

+14 −21
Original line number Diff line number Diff line 
#!/usr/bin/python

#

# Copyright 2015 Google Inc.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""Tests for metrics."""

"""

Tests for metricsT.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "GPL"





import numpy as np

import deepchem as dc

from tensorflow.python.platform import googletest

from deepchem.metrics import kappa_score 

from deepchem.metrics import Metric

from deepchem import metrics
@@ -28,7 +21,7 @@ class MetricsTest(googletest.TestCase): 
  def test_kappa_score(self):

    y_true = [1, 0, 1, 0]

    y_pred = [0.8, 0.2, 0.3, 0.4]  # [1, 0, 0, 0] with 0.5 threshold

    kappa = kappa_score(y_true, np.greater(y_pred, 0.5))

    kappa = dc.metrics.kappa_score(y_true, np.greater(y_pred, 0.5))

    observed_agreement = 3.0 / 4.0

    expected_agreement = ((2 * 1) + (2 * 3)) / 4.0 ** 2

    expected_kappa = np.true_divide(observed_agreement - expected_agreement,
@@ -42,8 +35,8 @@ class MetricsTest(googletest.TestCase): 
    n_samples = 10

    y_true = np.random.rand(n_samples,)

    y_pred = np.random.rand(n_samples,)

    regression_metric = Metric(metrics.r2_score, verbosity=verbosity)

    assert np.isclose(metrics.r2_score(y_true, y_pred),

    regression_metric = dc.metrics.Metric(dc.metrics.r2_score)

    assert np.isclose(dc.metrics.r2_score(y_true, y_pred),

                      regression_metric.compute_metric(y_true, y_pred))

CRA Git | Maintained and supported by SUSTech CRA and CCSE