Fixing multitask networks ipynb

%% Cell type:markdown id: tags:

This notebook walks through the creation of multitask models on MUV. The goal is to demonstrate that multitask methods outperform singletask methods on MUV.

%% Cell type:code id: tags:

``` python

%reload_ext autoreload

%autoreload 2

%pdb off

reload = True

```

%% Output

    Automatic pdb calling has been turned OFF

%% Cell type:code id: tags:

``` python

from deepchem.utils.save import load_from_disk

from deepchem.datasets import Dataset

import deepchem as dc

dataset_file= "../datasets/muv.csv.gz"

dataset = load_from_disk(dataset_file)

dataset_file= "../../datasets/muv.csv.gz"

dataset = dc.utils.save.load_from_disk(dataset_file)

print("Columns of dataset: %s" % str(dataset.columns.values))

print("Number of examples in dataset: %s" % str(dataset.shape[0]))

```

%% Output

    Columns of dataset: ['mol_id' 'smiles' 'MUV-692' 'MUV-689' 'MUV-846' 'MUV-859' 'MUV-644'

     'MUV-548' 'MUV-852' 'MUV-600' 'MUV-810' 'MUV-712' 'MUV-737' 'MUV-858'

     'MUV-713' 'MUV-733' 'MUV-652' 'MUV-466' 'MUV-832']

    Columns of dataset: ['MUV-466' 'MUV-548' 'MUV-600' 'MUV-644' 'MUV-652' 'MUV-689' 'MUV-692'

     'MUV-712' 'MUV-713' 'MUV-733' 'MUV-737' 'MUV-810' 'MUV-832' 'MUV-846'

     'MUV-852' 'MUV-858' 'MUV-859' 'mol_id' 'smiles']

    Number of examples in dataset: 93127

%% Cell type:markdown id: tags:

Now, let's visualize some compounds from our dataset

%% Cell type:code id: tags:

``` python

from itertools import islice

from rdkit import Chem

from deepchem.utils.visualization import mols_to_pngs

from deepchem.utils.visualization import display_images

from rdkit.Chem import Draw

from itertools import islice

from IPython.display import Image, display, HTML

def display_images(filenames):

    """Helper to pretty-print images."""

    imagesList=''.join(

        ["<img style='width: 140px; margin: 0px; float: left; border: 1px solid black;' src='%s' />"

         % str(s) for s in sorted(filenames)])

    display(HTML(imagesList))

def mols_to_pngs(mols, basename="test"):

    """Helper to write RDKit mols to png files."""

    filenames = []

    for i, mol in enumerate(mols):

        filename = "%s%d.png" % (basename, i)

        Draw.MolToFile(mol, filename)

        filenames.append(filename)

    return filenames

num_to_display = 12

molecules = []

for _, data in islice(dataset.iterrows(), num_to_display):

    molecules.append(Chem.MolFromSmiles(data["smiles"]))

display_images(mols_to_pngs(molecules))

```

%% Output

%% Cell type:code id: tags:

``` python

from deepchem.featurizers.fingerprints import CircularFingerprint

featurizers = [CircularFingerprint(size=1024)]

```

%% Cell type:code id: tags:

``` python

MUV_tasks = ['MUV-692', 'MUV-689', 'MUV-846', 'MUV-859', 'MUV-644',

             'MUV-548', 'MUV-852', 'MUV-600', 'MUV-810', 'MUV-712',

             'MUV-737', 'MUV-858', 'MUV-713', 'MUV-733', 'MUV-652',

             'MUV-466', 'MUV-832']

```

%% Cell type:code id: tags:

featurizer = dc.feat.CircularFingerprint(size=1024)

loader = dc.data.CSVLoader(

      tasks=MUV_tasks, smiles_field="smiles",

      featurizer=featurizer)

dataset = loader.featurize(dataset_file)

``` python

import os

from deepchem.featurizers.featurize import DataFeaturizer

# The base_dir holds the results of all analysis

base_dir = "/scratch/users/rbharath/muv_multitask_analysis"

#Make directories to store the raw and featurized datasets.

feature_dir = os.path.join(base_dir, "features")

samples_dir = os.path.join(base_dir, "samples")

featurizer = DataFeaturizer(tasks=MUV_tasks,

                            smiles_field="smiles",

                            compound_featurizers=featurizers,

                            verbosity="low")

#featurizer = DataFeaturizer(tasks=MUV_tasks,

#                            smiles_field="smiles",

#                            compound_featurizers=featurizers,

#                            verbosity="low")

# Setting reload=True directs the featurizer to use existing featurization on disk if such exists.

featurized_samples = featurizer.featurize(dataset_file, feature_dir, samples_dir, shard_size=4096,

                                          reload=reload)

```

%% Cell type:code id: tags:

``` python

splittype = "scaffold"

train_dir = os.path.join(base_dir, "train_dataset")

valid_dir = os.path.join(base_dir, "valid_dataset")

test_dir = os.path.join(base_dir, "test_dataset")

train_samples, valid_samples, test_samples = featurized_samples.train_valid_test_split(

    splittype, train_dir, valid_dir, test_dir, log_every_n=1000, reload=reload)

#featurized_samples = featurizer.featurize(dataset_file, feature_dir, samples_dir, shard_size=4096,

#                                          reload=reload)

```

%% Cell type:code id: tags:

%% Output

``` python

from deepchem.datasets import Dataset

print("Creating train dataset")

verbosity = None

train_dataset = Dataset(data_dir=train_dir, samples=train_samples,

                        featurizers=featurizers, tasks=MUV_tasks,

                        verbosity=verbosity, reload=reload)

print("Creating valid dataset")

valid_dataset = Dataset(data_dir=valid_dir, samples=valid_samples,

                        featurizers=featurizers, tasks=MUV_tasks,

                        verbosity=verbosity, reload=reload)

print("Creating test dataset")

test_dataset = Dataset(data_dir=test_dir, samples=test_samples,

                       featurizers=featurizers, tasks=MUV_tasks,

                       verbosity=verbosity, reload=reload)

    Loading raw samples now.

    shard_size: 8192

    About to start loading CSV from ../../datasets/muv.csv.gz

    Loading shard 1 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 0 took 22.680 s

    Loading shard 2 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 1 took 31.196 s

    Loading shard 3 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 2 took 29.828 s

    Loading shard 4 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 3 took 16.055 s

    Loading shard 5 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 4 took 21.150 s

    Loading shard 6 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 5 took 16.878 s

    Loading shard 7 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 6 took 16.032 s

    Loading shard 8 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 7 took 16.518 s

    Loading shard 9 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 8 took 17.401 s

    Loading shard 10 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 9 took 21.326 s

    Loading shard 11 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    Featurizing sample 4000

    Featurizing sample 5000

    Featurizing sample 6000

    Featurizing sample 7000

    Featurizing sample 8000

    TIMING: featurizing shard 10 took 23.441 s

    Loading shard 12 of size 8192.

    Featurizing sample 0

    Featurizing sample 1000

    Featurizing sample 2000

    Featurizing sample 3000

    TIMING: featurizing shard 11 took 8.318 s

    TIMING: dataset construction took 246.032 s

    Loading dataset from disk.

%% Cell type:code id: tags:

``` python

splitter = dc.splits.RandomSplitter(dataset_file)

train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(

    dataset)

#NOTE THE RENAMING:

valid_dataset, test_dataset = test_dataset, valid_dataset

```

%% Output

    Creating train dataset

    Creating valid dataset

    Creating test dataset

%% Cell type:code id: tags:

``` python

input_transformers = []

output_transformers = []

```

    Computing train/valid/test indices

    TIMING: dataset construction took 6.272 s

    Loading dataset from disk.

    TIMING: dataset construction took 3.243 s

    Loading dataset from disk.

    TIMING: dataset construction took 3.498 s

    Loading dataset from disk.

%% Cell type:code id: tags:

``` python

from deepchem.hyperparameters import HyperparamOpt

from deepchem.models.tensorflow_models import TensorflowModel

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

from deepchem import metrics

from deepchem.metrics import Metric

import numpy as np

import numpy.random

model_dir = os.path.join(base_dir, "model")

MUV_task_types = {task: "Classification" for task in MUV_tasks}

params_dict = {"activation": ["relu"],

               "momentum": [.9],

               "batch_size": [50],

               "init": ["glorot_uniform"],

               "data_shape": [train_dataset.get_data_shape()],

               "learning_rate": [1e-3],

               "decay": [1e-6],

               "nb_hidden": [1000],

               "nb_epoch": [1],

               "nesterov": [False],

               "dropouts": [(.5,)],

               "nb_layers": [1],

               "batchnorm": [False],

               "layer_sizes": [(1000,)],

               "weight_init_stddevs": [(.1,)],

               "bias_init_consts": [(1.,)],

               "num_classes": [2],

               "penalty": [0.],

               "optimizer": ["sgd"],

               "num_classification_tasks": [len(MUV_task_types)]

              }

def model_builder(task_types, params_dict, logdir, verbosity=None):

    return TensorflowModel(

        task_types, params_dict, logdir,

        tf_class=TensorflowMultiTaskClassifier,

        verbosity=verbosity)

metric = Metric(metrics.roc_auc_score, np.mean)

optimizer = HyperparamOpt(model_builder, MUV_task_types, verbosity="low")

n_features = train_dataset.get_data_shape()[0]

def model_builder(model_params, model_dir):

  model = dc.models.TensorflowMultiTaskClassifier(

    len(MUV_tasks), n_features, **model_params)

  return model

metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean)

optimizer = dc.hyper.HyperparamOpt(model_builder)

best_dnn, best_hyperparams, all_results = optimizer.hyperparam_search(

    params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=model_dir)

    params_dict, train_dataset, valid_dataset, [], metric)

```

%% Output

    Fitting model 1/1

    hyperparameters: {'learning_rate': 0.001, 'layer_sizes': (1000,), 'data_shape': (1024,), 'dropouts': (0.5,), 'activation': 'relu', 'decay': 1e-06, 'batch_size': 50, 'penalty': 0.0, 'nesterov': False, 'init': 'glorot_uniform', 'bias_init_consts': (1.0,), 'weight_init_stddevs': (0.1,), 'batchnorm': False, 'nb_layers': 1, 'nb_epoch': 1, 'momentum': 0.9}

    Training for 1 epochs

    Ending epoch 0: loss 0.00714142

    ys[0]

    [0.0 0.0 0.0 ..., 0.0 0.0 0.0]

    y_preds[0]

    [0 0 0 ..., 0 0 0]

    Saving predictions to <open file '<fdopen>', mode 'w+b' at 0x7f280c626c00>

    /home/rbharath/deepchem/deepchem/metrics/__init__.py:151: UserWarning: Error calculating metric mean-roc_auc_score: unknown format is not supported

      % (self.name, e))

    Saving model performance scores to <open file '<fdopen>', mode 'w+b' at 0x7f280c626b70>

    hyperparameters.compute

    valid_score

    nan

    hyperparameter_tuple

    ('sgd', 1e-06, (1024,), 1, 'relu', (1000,), 50, 0.0, False, 'glorot_uniform', (1.0,), (0.1,), 2, 1, 17, 1000, False, (0.5,), 0.001, 0.9)

    Model 0/1, Metric mean-roc_auc_score, Validation set 0: nan

    	best_validation_score so far: -inf

    No models trained correctly.

%% Cell type:code id: tags:

``` python

```

    On batch 0

    On batch 50

    On batch 100

    On batch 150

    On batch 200

    On batch 250

    On batch 300

    On batch 350

    On batch 400

    On batch 450

    On batch 500

    On batch 550

    On batch 600

    On batch 650

    On batch 700

    On batch 750

    On batch 800

    On batch 850

    On batch 900

    On batch 950

    On batch 1000

    On batch 1050

    On batch 1100

    On batch 1150

    On batch 1200

    On batch 1250

    On batch 1300

    On batch 1350

    On batch 1400

    On batch 1450

    Ending epoch 0: Average loss 0.0517954

    On batch 0

    TIMING: model fitting took 42.847 s True

    computed_metrics: [0.65548567435359884, 0.3938885157824043, 0.91541865214431595, 0.39478957915831664, 0.68465248721524885, 0.74978870858688307, 0.80529733424470273, 0.77885952712100137, 0.79980310400741261, 0.88896680691912111, 0.5185931899641576, 0.5150129017124091, 0.75240054869684503, 0.36638813096862211, 0.58238095238095244, 0.61817558299039777, 0.68414343983684567]

    Model 1/1, Metric mean-roc_auc_score, Validation set 0: 0.653179

    	best_validation_score so far: 0.653179

    computed_metrics: [0.9154553963735379, 0.93463278293773655, 0.98457975954502508, 0.93589327852250226, 0.97265932420872547, 0.9838091436190004, 0.97946596833011079, 0.92610806949042246, 0.88254565136882901, 0.94433074824070085, 0.93797633429015803, 0.94311234732601956, 0.93923263973993476, 0.94550404459919302, 0.90309495615889945, 0.87384089962146405, 0.99468966106036016]

    Best hyperparameters: (1e-06, (1024,), 1, 'relu', (1000,), 50, 0.0, False, 'glorot_uniform', (1.0,), (0.1,), 1, False, (0.5,), 0.001, 0.9)

    train_score: 0.940996

    validation_score: 0.653179