Commit 44841f23 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Moving code in to start testing process

parent 241ce853

deepchem/datasets/__init__.py

+31 −6
Original line number Diff line number Diff line
@@ -210,11 +210,14 @@ class Dataset(object): 
      w = np.ones_like(y)

    if tasks is None:

      tasks = np.arange(n_tasks)

    ########### DEBUG

    #print("ids.shape, X.shape, y.shape, w.shape")

    #print(ids.shape, X.shape, y.shape, w.shape)

    ########### DEBUG

    raw_data = (ids, X, y, w)

    ###################  DEBUG

    #print("Dataset.from_numpy()")

    #print("type(y)")

    #print(type(y))

    #print("type(X)")

    #print(type(X))

    ###################  DEBUG

    return Dataset(data_dir=data_dir, tasks=tasks, raw_data=raw_data)

    

  def to_numpy(self):
@@ -335,6 +338,13 @@ def write_dataset_single(val, data_dir, feature_types=None, tasks=None, 
    ids, X, y, w = _df_to_numpy(df, feature_types, tasks)

  else:

    ids, X, y, w = raw_data

    ###################  DEBUG

    #print("write_dataset_single()")

    #print("type(y)")

    #print(type(y))

    #print("type(X)")

    #print(type(X))

    ###################  DEBUG

    df_file = ""

    assert X.shape[0] == y.shape[0]

    assert y.shape == w.shape
@@ -388,8 +398,23 @@ def _df_to_numpy(df, feature_types, tasks): 
  # perform common train/test split across all tasks

  n_samples = df.shape[0]

  n_tasks = len(tasks)

  y = np.array([df[task].values for task in tasks])

  y = np.reshape(y, (n_samples, n_tasks))

  y = np.hstack([np.reshape(np.array(df[task].values), (n_samples, 1)) for task in tasks])

  ############ DEBUG

  #print("Dataset._df_to_numpy()")

  #print("tasks")

  #print(tasks)

  #print("y.shape")

  #print(y.shape)

  #print("Before reshape")

  #print("y")

  #print(y)

  ########### DEBUG

  #y = np.reshape(y, (n_samples, n_tasks))

  ############ DEBUG

  #print("After reshape")

  #print("y")

  #print(y)

  ############ DEBUG

  w = np.ones((n_samples, n_tasks))

  missing = np.zeros_like(y).astype(int)

  tensors = []

deepchem/datasets/tests/test_load.py

0 → 100644
+130 −0
Original line number Diff line number Diff line 
"""

Testing singletask/multitask data loading capabilities.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "LGPL"



import os

import shutil

import tempfile

import numpy as np

from deepchem.models.tests import TestAPI

from deepchem.utils.save import load_from_disk

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.featurizers.featurize import DataFeaturizer

from deepchem.datasets import Dataset



## task0: 1,1,0,-,0,-,1,-,-,1



class TestLoad(TestAPI):

  """

  Test singletask/multitask data loading.

  """



  def test_singletask_matches_multitask_load(self):

    """Check that singletask load and multitask load of dataset are same."""

    # Only for debug!

    np.random.seed(123)



    # Set some global variables up top

    reload = True

    verbosity = "high"



    base_dir = tempfile.mkdtemp()



    current_dir = os.path.dirname(os.path.realpath(__file__))

    #Make directories to store the raw and featurized datasets.

    feature_dir = os.path.join(base_dir, "features")

    samples_dir = os.path.join(base_dir, "samples")

    full_dir = os.path.join(base_dir, "full_dataset")

    train_dir = os.path.join(base_dir, "train_dataset")

    valid_dir = os.path.join(base_dir, "valid_dataset")

    test_dir = os.path.join(base_dir, "test_dataset")

    model_dir = os.path.join(base_dir, "model")



    # Load dataset

    print("About to load dataset.")

    dataset_file = os.path.join(

        current_dir, "../../models/tests/multitask_example.csv")

    dataset = load_from_disk(dataset_file)

    print("Columns of dataset: %s" % str(dataset.columns.values))

    print("Number of examples in dataset: %s" % str(dataset.shape[0]))



    # Featurize tox21 dataset

    print("About to featurize dataset.")

    featurizers = [CircularFingerprint(size=1024)]

    all_tasks = ["task%d"%i for i in range(17)] 

    # For debugging purposes

    n_tasks = 17 

    tasks = all_tasks[0:n_tasks]

    valid_scores = {}



    ####### Do multitask load

    if os.path.exists(feature_dir):

      shutil.rmtree(feature_dir)

    featurizer = DataFeaturizer(tasks=tasks,

                                smiles_field="smiles",

                                compound_featurizers=featurizers,

                                verbosity=verbosity)

    featurized_samples = featurizer.featurize(

        dataset_file, feature_dir,

        samples_dir, shard_size=8192,

        reload=reload)

    if os.path.exists(full_dir):

      shutil.rmtree(full_dir)

    full_dataset = Dataset(data_dir=full_dir, samples=featurized_samples, 

                            featurizers=featurizers, tasks=tasks,

                            verbosity=verbosity, reload=reload)



    # Do train/valid split.

    X_multi, y_multi, w_multi, ids_multi = full_dataset.to_numpy()





    ####### Do singletask load

    X_tasks, y_tasks, w_tasks, ids_tasks = [], [], [], []

    for task in tasks:

      print("Processing task %s" % task)

      if os.path.exists(feature_dir):

        shutil.rmtree(feature_dir)

      featurizer = DataFeaturizer(tasks=[task],

                                  smiles_field="smiles",

                                  compound_featurizers=featurizers,

                                  verbosity=verbosity)

      featurized_samples = featurizer.featurize(

          dataset_file, feature_dir,

          samples_dir, shard_size=8192,

          reload=reload)

      if os.path.exists(full_dir):

        shutil.rmtree(full_dir)

      full_dataset = Dataset(data_dir=full_dir, samples=featurized_samples, 

                              featurizers=featurizers, tasks=[task],

                              verbosity=verbosity, reload=reload)



      X_task, y_task, w_task, ids_task = full_dataset.to_numpy()

      X_tasks.append(X_task)

      y_tasks.append(y_task)

      w_tasks.append(w_task)

      ids_tasks.append(ids_task)



    ################## Do comparison

    for ind, task in enumerate(tasks):

      #X_multi_task = X_multi[ind]

      y_multi_task = y_multi[:, ind]

      w_multi_task = w_multi[:, ind]

      #ids_multi_task = ids_multi[ind]



      #X_task = X_tasks[ind]

      y_task = y_tasks[ind]

      w_task = w_tasks[ind]

      ids_task = ids_tasks[ind]



      #np.testing.assert_allclose(X_multi_task, X_task)

      np.testing.assert_allclose(y_multi_task.flatten(), y_task.flatten())

      np.testing.assert_allclose(w_multi_task.flatten(), w_task.flatten())

      #np.testing.assert_allclose(ids_multi_task, ids_task)

    shutil.rmtree(base_dir)

deepchem/featurizers/featurize.py

+0 −1
Original line number Diff line number Diff line
@@ -350,7 +350,6 @@ class FeaturizedSamples(object): 
    self.compounds_df = compounds_df

    self.num_samples = len(compounds_df)





  def _get_compounds_filename(self):

    """

    Get standard location for file listing compounds in this dataframe.

deepchem/models/tests/test_generalize.py

0 → 100644
+245 −0
Original line number Diff line number Diff line 
"""

Tests to make sure deepchem models can fit models on easy datasets.

"""



from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "LGPL"



import sklearn

import sklearn.datasets

import numpy as np

from sklearn.ensemble import RandomForestRegressor

from sklearn.linear_model import LinearRegression 

from sklearn.linear_model import LogisticRegression

from deepchem.datasets import Dataset

from deepchem.metrics import Metric

from deepchem.models.tests import TestAPI

from deepchem.models.sklearn_models import SklearnModel

from deepchem import metrics

from deepchem.utils.evaluate import Evaluator

from deepchem.models.multitask import SingletaskToMultitask



class TestGeneralization(TestAPI):

  """

  Test that models can learn generalizable models on simple datasets.

  """



  def test_sklearn_regression(self):

    """Test that sklearn models can learn on simple regression datasets."""

    np.random.seed(123)

    dataset = sklearn.datasets.load_diabetes()

    X, y = dataset.data, dataset.target



    frac_train = .7

    n_samples = len(X)

    

    X_train, y_train = X[:frac_train*n_samples], y[:frac_train*n_samples]

    X_test, y_test = X[frac_train*n_samples:], y[frac_train*n_samples:]



    train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train)

    test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test)



    tasks = train_dataset.get_task_names()

    task_types = {task: "regression" for task in tasks}



    model_params = {

      "batch_size": None,

      "data_shape": train_dataset.get_data_shape()

    }



    verbosity = "high"

    regression_metric = Metric(metrics.r2_score, verbosity=verbosity)

    model = SklearnModel(tasks, task_types, model_params, self.model_dir,

                         mode="regression",

                         model_instance=LinearRegression())



    # Fit trained model

    model.fit(train_dataset)

    model.save()



    # Eval model on train

    transformers = []

    train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity)

    train_scores = train_evaluator.compute_model_performance([regression_metric])

    print("train_scores")

    print(train_scores)



    # Eval model on test

    transformers = []

    evaluator = Evaluator(model, test_dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([regression_metric])

    print("scores")

    print(scores)



    assert scores[regression_metric.name] > .5



  def test_sklearn_multitask_regression(self):

    """Test that sklearn models can learn on simple multitask regression."""

    np.random.seed(123)

    n_tasks = 4

    dataset = sklearn.datasets.load_diabetes()

    X, y = dataset.data, dataset.target

    y = np.reshape(y, (len(y), 1))

    y = np.hstack([y] * n_tasks)

    

    frac_train = .7

    n_samples = len(X)

    

    X_train, y_train = X[:frac_train*n_samples], y[:frac_train*n_samples]

    X_test, y_test = X[frac_train*n_samples:], y[frac_train*n_samples:]



    train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train)

    test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test)



    tasks = train_dataset.get_task_names()

    task_types = {task: "regression" for task in tasks}



    model_params = {

      "batch_size": None,

      "data_shape": train_dataset.get_data_shape()

    }



    verbosity = "high"

    regression_metric = Metric(metrics.r2_score, verbosity=verbosity)

    def model_builder(tasks, task_types, model_params, model_dir, verbosity=None):

      return SklearnModel(tasks, task_types, model_params, model_dir,

                          mode="regression",

                          model_instance=LinearRegression(),

                          verbosity=verbosity)

    model = SingletaskToMultitask(tasks, task_types, model_params, self.model_dir,

                                  model_builder, verbosity=verbosity)



    # Fit trained model

    model.fit(train_dataset)

    model.save()



    # Eval model on train

    transformers = []

    train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity)

    train_scores = train_evaluator.compute_model_performance([regression_metric])

    print("train_scores")

    print(train_scores)



    # Eval model on test

    transformers = []

    evaluator = Evaluator(model, test_dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([regression_metric])

    print("scores")

    print(scores)



    for score in scores[regression_metric.name]:

      assert score > .5



  def test_sklearn_classification(self):

    """Test that sklearn models can learn on simple classification datasets."""

    np.random.seed(123)

    dataset = sklearn.datasets.load_digits(n_class=2)

    X, y = dataset.data, dataset.target



    frac_train = .7

    n_samples = len(X)

    

    X_train, y_train = X[:frac_train*n_samples], y[:frac_train*n_samples]

    X_test, y_test = X[frac_train*n_samples:], y[frac_train*n_samples:]



    print("X_train.shape, y_train.shape, X_test.shape, y_test.shape")

    print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)



    train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train)

    test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test)



    tasks = train_dataset.get_task_names()

    task_types = {task: "classification" for task in tasks}



    model_params = {

      "batch_size": None,

      "data_shape": train_dataset.get_data_shape()

    }



    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)

    model = SklearnModel(tasks, task_types, model_params, self.model_dir,

                         mode="classification",

                         model_instance=LogisticRegression())



    # Fit trained model

    model.fit(train_dataset)

    model.save()



    # Eval model on train

    transformers = []

    train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity)

    train_scores = train_evaluator.compute_model_performance([classification_metric])

    print("train_scores")

    print(train_scores)



    # Eval model on test

    transformers = []

    evaluator = Evaluator(model, test_dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([classification_metric])

    print("scores")

    print(scores)



    assert scores[classification_metric.name] > .5



  def test_sklearn_multitask_classification(self):

    """Test that sklearn models can learn on simple multitask classification."""

    np.random.seed(123)

    n_tasks = 4

    dataset = sklearn.datasets.load_digits(n_class=2)

    X, y = dataset.data, dataset.target

    y = np.reshape(y, (len(y), 1))

    y = np.hstack([y] * n_tasks)

    

    frac_train = .7

    n_samples = len(X)

    

    X_train, y_train = X[:frac_train*n_samples], y[:frac_train*n_samples]

    X_test, y_test = X[frac_train*n_samples:], y[frac_train*n_samples:]



    train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train)

    test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test)



    tasks = train_dataset.get_task_names()

    task_types = {task: "classification" for task in tasks}



    model_params = {

      "batch_size": None,

      "data_shape": train_dataset.get_data_shape()

    }



    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)

    def model_builder(tasks, task_types, model_params, model_dir, verbosity=None):

      return SklearnModel(tasks, task_types, model_params, model_dir,

                          mode="classification",

                          model_instance=LogisticRegression(),

                          verbosity=verbosity)

    model = SingletaskToMultitask(tasks, task_types, model_params, self.model_dir,

                                  model_builder, verbosity=verbosity)



    # Fit trained model

    model.fit(train_dataset)

    model.save()



    # Eval model on train

    transformers = []

    train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity)

    train_scores = train_evaluator.compute_model_performance([classification_metric])

    print("train_scores")

    print(train_scores)



    # Eval model on test

    transformers = []

    evaluator = Evaluator(model, test_dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([classification_metric])

    print("scores")

    print(scores)



    for score in scores[classification_metric.name]:

      assert score > .5

examples/muv/muv_sklearn_models.py

0 → 100644
+190 −0
Original line number Diff line number Diff line 
"""

Script that trains multitask models on MUV dataset.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



import os

import numpy as np

import shutil

from sklearn.linear_model import LogisticRegression

from sklearn.ensemble import RandomForestClassifier

from deepchem.utils.save import load_from_disk

from deepchem.datasets import Dataset

from deepchem.featurizers.featurize import DataFeaturizer

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.splits import ScaffoldSplitter

from deepchem.splits import RandomSplitter

from deepchem.datasets import Dataset

from deepchem.transformers import BalancingTransformer

from deepchem.hyperparameters import HyperparamOpt

from deepchem.models.multitask import SingletaskToMultitask

from deepchem import metrics

from deepchem.metrics import Metric

from deepchem.metrics import to_one_hot

from deepchem.models.sklearn_models import SklearnModel

from deepchem.utils.evaluate import relative_difference

from deepchem.utils.evaluate import Evaluator





np.random.seed(123)



# Set some global variables up top

reload = True

verbosity = "high"

model = "logistic"



# Create some directories for analysis

# The base_dir holds the results of all analysis

#base_dir = "/scratch/users/rbharath/muv_multitask_analysis"

base_dir = "/scratch/users/rbharath/small_muv_multitask_analysis"

current_dir = os.path.dirname(os.path.realpath(__file__))

#Make directories to store the raw and featurized datasets.

feature_dir = os.path.join(base_dir, "features")

samples_dir = os.path.join(base_dir, "samples")

full_dir = os.path.join(base_dir, "full_dataset")

train_dir = os.path.join(base_dir, "train_dataset")

valid_dir = os.path.join(base_dir, "valid_dataset")

test_dir = os.path.join(base_dir, "test_dataset")

model_dir = os.path.join(base_dir, "model")



if os.path.exists(model_dir):

  shutil.rmtree(model_dir)

os.makedirs(model_dir)



# Load MUV dataset

print("About to load MUV dataset.")

dataset_file = os.path.join(

    current_dir, "../../datasets/muv.csv.gz")

dataset = load_from_disk(dataset_file)

print("Columns of dataset: %s" % str(dataset.columns.values))

print("Number of examples in dataset: %s" % str(dataset.shape[0]))



# Featurize MUV dataset

print("About to featurize MUV dataset.")

featurizers = [CircularFingerprint(size=1024)]

MUV_tasks = sorted(['MUV-692', 'MUV-689', 'MUV-846', 'MUV-859', 'MUV-644',

                    'MUV-548', 'MUV-852', 'MUV-600', 'MUV-810', 'MUV-712',

                    'MUV-737', 'MUV-858', 'MUV-713', 'MUV-733', 'MUV-652',

                    'MUV-466', 'MUV-832'])

# For debugging purposes

MUV_tasks = MUV_tasks[0:2]

print("Using following tasks")

print(MUV_tasks)

if os.path.exists(feature_dir):

  shutil.rmtree(feature_dir)

featurizer = DataFeaturizer(tasks=MUV_tasks,

                            smiles_field="smiles",

                            compound_featurizers=featurizers,

                            verbosity=verbosity)

featurized_samples = featurizer.featurize(

    dataset_file, feature_dir,

    samples_dir, shard_size=8192,

    reload=reload)



assert len(featurized_samples) == len(dataset)

# Train/Valid/Test Split dataset

print("About to perform train/valid/test split.")

splitter = RandomSplitter(verbosity=verbosity)

frac_train, frac_valid, frac_test = .8, .1, .1

train_samples, valid_samples, test_samples = \

    splitter.train_valid_test_split(

        featurized_samples, train_dir, valid_dir, test_dir,

        log_every_n=1000, reload=reload)



len_train_samples, len_valid_samples, len_test_samples = \

  len(train_samples), len(valid_samples), len(test_samples)

assert relative_difference(

    len(train_samples), frac_train * len(featurized_samples)) < 1e-3

assert relative_difference(

    len(valid_samples), frac_valid * len(featurized_samples)) < 1e-3

assert relative_difference(

    len(test_samples), frac_test * len(featurized_samples)) < 1e-3



# Generate datasets

print("About to create datasets")

print("MUV_tasks")

print(MUV_tasks)

if os.path.exists(full_dir):

  shutil.rmtree(full_dir)

full_dataset = Dataset(data_dir=full_dir, samples=featurized_samples, 

                        featurizers=featurizers, tasks=MUV_tasks,

                        verbosity=verbosity, reload=reload)

print("full_dataset.get_task_names()")

print(full_dataset.get_task_names())

y = full_dataset.get_labels()

if os.path.exists(train_dir):

  shutil.rmtree(train_dir)

train_dataset = Dataset(data_dir=train_dir, samples=train_samples, 

                        featurizers=featurizers, tasks=MUV_tasks,

                        verbosity=verbosity, reload=reload)

y_train  = train_dataset.get_labels()

if os.path.exists(valid_dir):

  shutil.rmtree(valid_dir)

valid_dataset = Dataset(data_dir=valid_dir, samples=valid_samples, 

                        featurizers=featurizers, tasks=MUV_tasks,

                        verbosity=verbosity, reload=reload)

y_valid = valid_dataset.get_labels()

if os.path.exists(test_dir):

  shutil.rmtree(test_dir)

test_dataset = Dataset(data_dir=test_dir, samples=test_samples, 

                       featurizers=featurizers, tasks=MUV_tasks,

                       verbosity=verbosity, reload=reload)

y_test = test_dataset.get_labels()

len_train_dataset, len_valid_dataset, len_test_dataset = \

  len(train_dataset), len(valid_dataset), len(test_dataset)



# Transform data

print("About to transform data")

input_transformers = []

output_transformers = []

weight_transformers = [BalancingTransformer(transform_w=True, dataset=train_dataset)]

transformers = input_transformers + output_transformers + weight_transformers

for transformer in transformers:

    transformer.transform(train_dataset)

for transformer in transformers:

    transformer.transform(valid_dataset)

for transformer in transformers:

    transformer.transform(test_dataset)



# Fit Logistic Regression models

MUV_task_types = {task: "Classification" for task in MUV_tasks}





classification_metric = Metric(metrics.roc_auc_score, np.mean,

                               verbosity=verbosity,

                               mode="classification")

params_dict = { 

    "batch_size": None,

    "data_shape": train_dataset.get_data_shape(),

}   



if os.path.exists(model_dir):

  shutil.rmtree(model_dir)

os.makedirs(model_dir)

def model_builder(tasks, task_types, model_params, model_dir, verbosity=None):

  return SklearnModel(tasks, task_types, model_params, model_dir,

                      #model_instance=LogisticRegression(class_weight="balanced"),

                      model_instance=RandomForestClassifier(

                          class_weight="balanced",

                          n_estimators=500),

                      verbosity=verbosity)

model = SingletaskToMultitask(MUV_tasks, MUV_task_types, params_dict, model_dir,

                              model_builder, verbosity=verbosity)



# Fit trained model

model.fit(train_dataset)

model.save()



train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity)

train_scores = train_evaluator.compute_model_performance([classification_metric])



print("Train scores")

print(train_scores)



valid_evaluator = Evaluator(model, valid_dataset, transformers, verbosity=verbosity)

valid_scores = valid_evaluator.compute_model_performance([classification_metric])



print("Validation scores")

print(valid_scores)

CRA Git | Maintained and supported by SUSTech CRA and CCSE