Benchmark modified

|tox21      |tensorflow(MT-DNN)  |0.987              |0.800              |35              |36             |

|muv        |tensorflow(MT-DNN)  |0.979              |0.660              |414             |255            |

|pcba       |tensorflow(MT-DNN)	 |0.949        	     |0.791              |1765            |7209           |                                         

|sider      |tensorflow(MT-DNN)	 |0.864        	     |0.627              |10              |63             |                                         

|toxcast    |tensorflow(MT-DNN)	 |0.944        	     |0.697              |75              |2374           |                                         

## Contributing to DeepChem

We actively encourage community contributions to DeepChem. The first place to start getting involved is by running our examples locally. Afterwards, we encourage contributors to give a shot to improving our documentation. While we take effort to provide good docs, there's plenty of room for improvement. All docs are hosted on Github, either in this `README.md` file, or in the `docs/` directory.

#!/usr/bin/env python3

# -*- coding: utf-8 -*-

"""

Created on Tue Oct 18 15:53:27 2016

@author: Michael Wu

Benchmark test

Giving performances of RF(scikit) and MultitaskDNN(TF)

on datasets: muv, nci, pcba, tox21

time estimation(on a nvidia tesla K20 GPU):

tox21 - dataloading: 30s

      - tf: 40s

muv   - dataloading: 400s

      - tf: 250s

pcba  - dataloading: 30min

      - tf: 2h

sider - dataloading: 10s

      - tf: 60s

toxcast dataloading: 70s

	tf: 70s

(will include more)

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals

import sys

import os

import numpy as np

import shutil

import time

import deepchem as dc

from sklearn.linear_model import LogisticRegression

from sklearn.ensemble import RandomForestClassifier

from muv.muv_datasets import load_muv

from nci.nci_datasets import load_nci

from pcba.pcba_datasets import load_pcba

from tox21.tox21_datasets import load_tox21

from toxcast.toxcast_datasets import load_toxcast

from sider.sider_datasets import load_sider

def benchmark_loading_datasets(base_dir_o, hyper_parameters, n_features = 1024, 

                               dataset_name='all',model='all',reload = True,

                               verbosity='high',out_path='/home/zqwu/deepchem/examples'):

  """

  Loading dataset for benchmark test

  Parameters

  ----------

  base_dir_o, string

      path of working folder, will be combined with '/dataset_name'

  hyper_parameters, dict of list

      hyper parameters including dropout rate, learning rate, etc.

  n_features, integer, optional (default=1024)

      number of features, or length of binary fingerprints

  dataset_name, string, optional (default='all')

      choice of which dataset to use, 'all' = computing all the datasets

  model string, optional (default='all')

      choice of which model to use, 'all' = running all models on the dataset

  out_path, string, optional(default='/tmp')

      path of result file

  """

  assert dataset_name in ['all', 'muv', 'nci', 'pcba', 'tox21','sider',

                          'toxcast']

  if dataset_name == 'all':

    #currently not including the nci dataset

    dataset_name = ['tox21','muv','pcba','sider','toxcast']

  else:

    dataset_name = [dataset_name]

  loading_functions = {'tox21':load_tox21, 'muv':load_muv,

                       'pcba':load_pcba, 'nci':load_nci,

                       'sider':load_sider, 'toxcast':load_toxcast}

  for dname in dataset_name:

    print('-------------------------------------')

    print('Benchmark test on dataset: '+dname)

    print('-------------------------------------')

    base_dir = os.path.join(base_dir_o, dname)

    time_start = time.time()

    #loading datasets     

    tasks,datasets,transformers = loading_functions[dname]()

    train_dataset, valid_dataset, test_dataset = datasets

    time_finish_loading = time.time()

    #time_finish_loading-time_start is the time(s) used for dataset loading

    #running model

    train_score,valid_score = benchmark_train_and_valid(base_dir,train_dataset,

                                                        valid_dataset, tasks,

                                                        transformers,

                                                        hyper_parameters,

                                                        n_features=n_features,

                                                        model = model,

                                                        verbosity = verbosity)

    time_finish_running = time.time()

    #time_finish_running-time_finish_loading is the time(s) used for fitting and evaluating

    with open(os.path.join(out_path,'results.csv'),'a') as f:

      f.write ('\n'+dname+',train')

      for i in train_score:

        f.write(','+i+','+str(train_score[i]['mean-roc_auc_score']))

      f.write('\n'+dname+',valid')

      for i in valid_score:

        f.write(','+i+','+str(valid_score[i]['mean-roc_auc_score'])) 

      #output timing data: running time include all the model

      f.write('\n'+dname+',time_for_loading,,'+

              str(time_finish_loading-time_start)+'seconds')

      f.write('\n'+dname+',time_for_running,,'+

              str(time_finish_running-time_finish_loading)+'seconds')

    #clear workspace         

    del tasks,datasets,transformers

    del train_dataset,valid_dataset, test_dataset

    del time_start,time_finish_loading,time_finish_running

  return None

def benchmark_train_and_valid(base_dir,train_dataset,valid_dataset,tasks,

                              transformers, hyper_parameters,

                              n_features = 1024,model = 'all',

                              verbosity = 'high'):

  """

  Calculate performance of different models on the specific dataset & tasks

  Parameters

  ----------

  base_dir, string

      path of working folder

  train_dataset, dataset struct

      loaded dataset using load_* or splitter function

  valid_dataset, dataset struct

      loaded dataset using load_* or splitter function

  tasks, list of string

      list of targets(tasks, datasets)

  transformers, BalancingTransformer struct

      loaded properties of dataset from load_* function

  hyper_parameters, dict of list

      hyper parameters including dropout rate, learning rate, etc.

  n_features, integer, optional (default=1024)

      number of features, or length of binary fingerprints

  model, string, optional (default='all')

      choice of which model to use, 'all' = running all models on the dataset

  """

  train_scores = {}

  valid_scores = {}

  # Initialize metrics

  classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean,

                                            verbosity=verbosity,

                                            mode="classification")

  assert model in ['all', 'tf', 'rf']

  if model == 'all' or model == 'tf':

    # Initialize model folder

    model_dir_tf = os.path.join(base_dir, "model_tf")

    dropouts = hyper_parameters['tf'][0]

    learning_rate = hyper_parameters['tf'][1]

    weight_init_stddevs = hyper_parameters['tf'][2]

    batch_size = hyper_parameters['tf'][3]

    # Building tensorflow MultiTaskDNN model

    tensorflow_model = dc.models.TensorflowMultiTaskClassifier(

        len(tasks), n_features, dropouts=[dropouts],

        learning_rate=learning_rate, weight_init_stddevs=[weight_init_stddevs],

        batch_size=batch_size, verbosity=verbosity)

    model_tf = dc.models.TensorflowModel(tensorflow_model)

    print('-------------------------------------')

    print('Start fitting by tensorflow')

    model_tf.fit(train_dataset)

    train_scores['tensorflow'] = model_tf.evaluate(train_dataset,

                                    [classification_metric],transformers)

    valid_scores['tensorflow'] = model_tf.evaluate(valid_dataset,

                                    [classification_metric],transformers)

  if model == 'all' or model == 'rf':

    # Initialize model folder

    model_dir_rf = os.path.join(base_dir, "model_rf")

    # Building scikit random forest model

    def model_builder(model_dir_rf):

      sklearn_model = RandomForestClassifier(

        class_weight="balanced", n_estimators=500,n_jobs=-1)

      return dc.models.sklearn_models.SklearnModel(sklearn_model, model_dir_rf)

    model_rf = dc.models.multitask.SingletaskToMultitask(

		tasks, model_builder, model_dir_rf)

    print('-------------------------------------')

    print('Start fitting by random forest')

    model_rf.fit(train_dataset)

    train_scores['random_forest'] = model_rf.evaluate(train_dataset,

                                    [classification_metric],transformers)

    valid_scores['random_forest'] = model_rf.evaluate(valid_dataset,

                                    [classification_metric],transformers)

  return train_scores, valid_scores

if __name__ == '__main__':

  # Global variables

  np.random.seed(123)

  verbosity = 'high'

  #Working folder initialization

  base_dir_o="/tmp/benchmark_test_"+time.strftime("%Y_%m_%d", time.localtime())

  if os.path.exists(base_dir_o):

    shutil.rmtree(base_dir_o)

  os.makedirs(base_dir_o)

  #Datasets and models used in the benchmark test, all=all the datasets(models)

  dataset_name = sys.argv[1]

  model = sys.argv[2]

  #input hyperparameters

  #tf: dropouts, learning rate, weight initial stddev, batch_size

  hyper_parameters = {'tf':[0.25, 0.0003, 0.1, 50]}

  benchmark_loading_datasets(base_dir_o,hyper_parameters,n_features = 1024,

                             dataset_name = dataset_name, model = model,

                             reload = reload, verbosity = verbosity)

sider - dataloading: 10s

      - tf: 60s

toxcast dataloading: 70s

	tf: 70s

	tf: 40min

(will include more)

Total time of running a benchmark test: 3~4h

"""

from __future__ import print_function

from __future__ import division

def benchmark_loading_datasets(base_dir_o, hyper_parameters, n_features = 1024, 

                               dataset_name='all',model='all',reload = True,

                               verbosity='high',out_path='/home/zqwu/deepchem/examples'):

                               verbosity='high',out_path='/tmp'):

  """

  Loading dataset for benchmark test

  Parameters

  ----------

  base_dir_o, string

  base_dir_o : string

      path of working folder, will be combined with '/dataset_name'

  hyper_parameters, dict of list

  hyper_parameters : dict of list

      hyper parameters including dropout rate, learning rate, etc.

  n_features, integer, optional (default=1024)

  n_features : integer, optional (default=1024)

      number of features, or length of binary fingerprints

  dataset_name, string, optional (default='all')

  dataset_name : string, optional (default='all')

      choice of which dataset to use, 'all' = computing all the datasets

  model string, optional (default='all')

  model : string,  optional (default='all')

      choice of which model to use, 'all' = running all models on the dataset

  out_path, string, optional(default='/tmp')

  out_path : string, optional(default='/tmp')

      path of result file

  """

  Parameters

  ----------

  base_dir, string

  base_dir : string

      path of working folder

  train_dataset, dataset struct

  train_dataset : dataset struct

      loaded dataset using load_* or splitter function

  valid_dataset, dataset struct

  valid_dataset : dataset struct

      loaded dataset using load_* or splitter function

  tasks, list of string

  tasks : list of string

      list of targets(tasks, datasets)

  transformers, BalancingTransformer struct

  transformers : BalancingTransformer struct

      loaded properties of dataset from load_* function

  hyper_parameters, dict of list

  hyper_parameters : dict of list

      hyper parameters including dropout rate, learning rate, etc.

  n_features, integer, optional (default=1024)

  n_features : integer, optional (default=1024)

      number of features, or length of binary fingerprints

  model, string, optional (default='all')

  model : string, optional (default='all')

      choice of which model to use, 'all' = running all models on the dataset

  Returns

  -------

  train_scores : dict

	predicting results(AUC, R2) on training set

  valid_scores : dict

	predicting results(AUC, R2) on valid set

  """

  train_scores = {}

  valid_scores = {}

tox21,train,tensorflow,0.986147958155

tox21,valid,tensorflow,0.79650731821

tox21,time_for_loading,,51.2364470959seconds

tox21,time_for_running,,55.1193900108seconds

muv,train,tensorflow,0.981334751563

muv,valid,tensorflow,0.675101782834

muv,time_for_loading,,628.282253027seconds

muv,time_for_running,,392.296582937seconds

sider,train,tensorflow,0.952611245074

sider,valid,tensorflow,0.643257514915

sider,time_for_loading,,9.53850197792seconds

sider,time_for_running,,39.4045360088seconds

toxcast,train,tensorflow,0.99627313286

toxcast,valid,tensorflow,0.572024010438

toxcast,time_for_loading,,72.5853919983seconds

toxcast,time_for_running,,66.7092349529seconds

tox21,train,tensorflow,{'mean-roc_auc_score': 0.98533716325495568}

tox21,valid,tensorflow,{'mean-roc_auc_score': 0.76022265548478984}

tox21,time_for_loading,,47.2354819775seconds

tox21,time_for_running,,48.8804900646seconds

muv,train,tensorflow,{'mean-roc_auc_score': 0.9675581031557573}

muv,valid,tensorflow,{'mean-roc_auc_score': 0.70061828516526314}

muv,time_for_loading,,539.499345064seconds

muv,time_for_running,,390.370105982seconds

sider,train,tensorflow,{'mean-roc_auc_score': 0.86420057746491896}

sider,valid,tensorflow,{'mean-roc_auc_score': 0.62745178986315908}

sider,time_for_loading,,9.28750491142seconds

sider,time_for_running,,63.1111950874seconds

toxcast,train,tensorflow,0.944461761777

toxcast,valid,tensorflow,0.697416232964

toxcast,time_for_loading,,74.6496529579seconds

toxcast,time_for_running,,2374.13329697seconds

tox21,train,tensorflow,0.95365232868

tox21,valid,tensorflow,0.772503681086

tox21,time_for_loading,,50.2263281345seconds

tox21,time_for_running,,53.9519109726seconds

 No newline at end of file

import shutil

import deepchem as dc

'''

from deepchem.utils.save import load_from_disk

from deepchem.datasets import DiskDataset

from deepchem.featurizers.featurize import DataLoader

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.transformers import BalancingTransformer

'''

def load_sider():

  current_dir = os.path.dirname(os.path.realpath(__file__))