add in nci dataset to benchmark

|Dataset    |Model               |Splitting   |Train score/R2|Valid score/R2|

|-----------|--------------------|------------|--------------|--------------|

|delaney    |MT-NN regression    |Index       |0.773         |0.574         |

|           |graphconv regression|Index       |0.964         |0.829         |

|           |graphconv regression|Index       |0.991         |0.825         |

|           |MT-NN regression    |Random      |0.769         |0.591         |

|           |graphconv regression|Random      |0.959         |0.821         |

|           |graphconv regression|Random      |0.996         |0.873         |

|           |MT-NN regression    |Scaffold    |0.782         |0.426         |

|           |graphconv regression|Scaffold    |0.976         |0.581         |

|           |graphconv regression|Scaffold    |0.994         |0.606         |

|kaggle     |MT-NN regression    |User-defined|0.748         |0.452         |

* General features

@author: Michael Wu

Benchmark test

Giving performances of: Random forest(rf), MultitaskDNN(tf), 

Benchmark test:

Giving classification performances of: 

    Random forest(rf), MultitaskDNN(tf), 

    RobustMultitaskDNN(tf_robust),

    Logistic regression(logreg),

    Graph convolution(graphconv)                 

on datasets: muv, pcba, tox21, sider, toxcast

on datasets: muv, nci, pcba, tox21, sider, toxcast

Giving regression performances of:

    MultitaskDNN(tf_regression),

    Graph convolution regression(graphconvreg)

on datasets: delaney, nci, kaggle

time estimation listed in README file

from sider.sider_datasets import load_sider

from kaggle.kaggle_datasets import load_kaggle

from delaney.delaney_datasets import load_delaney

from nci.nci_datasets import load_nci

def benchmark_loading_datasets(hyper_parameters, 

                               dataset='tox21', model='tf', split=None,

      hyper parameters including dropout rate, learning rate, etc.

  dataset: string, optional (default='tox21')

      choice of which dataset to use, should be: tox21, muv, sider, 

      toxcast, pcba

      toxcast, pcba, delaney, kaggle, nci

  model: string,  optional (default='tf')

      choice of which model to use, should be: rf, tf, tf_robust, logreg,

      graphconv

      graphconv, tf_regression, graphconvreg

  split: string,  optional (default=None)

      choice of splitter function, None = using the default splitter

  out_path: string, optional(default='.')

      path of result file

  """

  if dataset in ['muv','nci','pcba','tox21','sider','toxcast']:

  if dataset in ['muv', 'pcba', 'tox21', 'sider', 'toxcast']:

    mode = 'classification'

  elif dataset in ['kaggle', 'delaney']:

  elif dataset in ['kaggle', 'delaney', 'nci']:

    mode = 'regression'

  else:

    raise ValueError('Dataset not supported')

  #assigning featurizer

  if model in ['graphconv']:

  if model in ['graphconv', 'graphconvreg']:

    featurizer = 'GraphConv'

    n_features = 71

  elif model in ['tf', 'tf_robust', 'logreg', 'rf']:

  elif model in ['tf', 'tf_robust', 'logreg', 'rf', 'tf_regression']:

    featurizer = 'ECFP'

    n_features = 1024

  elif model in ['tf_regression']:

  else:

    raise ValueError('Model not supported')

  if dataset in ['kaggle']:

    featurizer = None #kaggle dataset use its own features

    split = None #kaggle dataset is already splitted

    featurizer = 'ECFP'

    n_features = 1024

  else:

    raise ValueError('Model not supported')

  if not split in [None, 'index','random','scaffold']:

    raise ValueError('Splitter function not supported')

  n_features: integer

      number of features, or length of binary fingerprints

  model: string,  optional (default='tf_regression')

      choice of which model to use, should be: tf_regression

      choice of which model to use, should be: tf_regression, graphconvreg

  Returns

  # Initialize metrics

  regression_metric = dc.metrics.Metric(dc.metrics.pearson_r2_score, np.mean)

  assert model in ['tf_regression']

  assert model in ['tf_regression', 'graphconvreg']

  if model == 'tf_regression':

    # Loading hyper parameters

    valid_scores['tf_regression'] = model_tf_regression.evaluate(

        valid_dataset, [regression_metric], transformers)

  if model == 'graphconvreg':

    # Initialize model folder

    # Loading hyper parameters

    batch_size = hyper_parameters['batch_size']

    nb_epoch = hyper_parameters['nb_epoch']    

    learning_rate = hyper_parameters['learning_rate']

    n_filters = hyper_parameters['n_filters']

    n_fully_connected_nodes = hyper_parameters['n_fully_connected_nodes']

    g = tf.Graph()

    sess = tf.Session(graph=g)

    K.set_session(sess)

    # Building graph convoluwtion model

    with g.as_default():

      tf.set_random_seed(seed)

      graph_model = dc.nn.SequentialGraph(n_features)

      graph_model.add(dc.nn.GraphConv(int(n_filters), activation='relu'))

      graph_model.add(dc.nn.BatchNormalization(epsilon=1e-5, mode=1))

      graph_model.add(dc.nn.GraphPool())

      graph_model.add(dc.nn.GraphConv(int(n_filters), activation='relu'))

      graph_model.add(dc.nn.BatchNormalization(epsilon=1e-5, mode=1))

      graph_model.add(dc.nn.GraphPool())

      # Gather Projection

      graph_model.add(dc.nn.Dense(int(n_fully_connected_nodes),

                                  activation='relu'))

      graph_model.add(dc.nn.BatchNormalization(epsilon=1e-5, mode=1))

      graph_model.add(dc.nn.GraphGather(batch_size, activation="tanh"))

      with tf.Session() as sess:

        model_graphconvreg = dc.models.MultitaskGraphRegressor(

          sess, graph_model, len(tasks), 

          batch_size=batch_size, learning_rate=learning_rate,

          optimizer_type="adam", beta1=.9, beta2=.999)

        print('-------------------------------------')

        print('Start fitting by graph convolution')

        # Fit trained model

        model_graphconvreg.fit(train_dataset, nb_epoch=nb_epoch)

        # Evaluating graph convolution model

        train_scores['graphconvreg'] = model_graphconvreg.evaluate(

            train_dataset, [regression_metric], transformers)

        valid_scores['graphconvreg'] = model_graphconvreg.evaluate(

            valid_dataset, [regression_metric], transformers)

  return train_scores, valid_scores

  parser.add_argument('-s', action='append', dest='splitter_args', default=[],

      help='Choice of splitting function: index, random, scaffold')

  parser.add_argument('-m', action='append', dest='model_args', default=[], 

      help='Choice of model: tf, tf_robust, logreg, graphconv')

      help='Choice of model: tf, tf_robust, logreg, graphconv, ' + 

           'tf_regression, graphconvreg')

  parser.add_argument('-d', action='append', dest='dataset_args', default=[], 

      help='Choice of dataset: tox21, sider, muv, toxcast, pcba, kaggle, delaney')

      help='Choice of dataset: tox21, sider, muv, toxcast, pcba, ' + 

           'kaggle, delaney, nci')

  args = parser.parse_args()

  #Datasets and models used in the benchmark test

  splitters = args.splitter_args

  if len(splitters) == 0:

    splitters = ['index', 'random', 'scaffold']

  if len(models) == 0:

    models = ['tf', 'tf_robust', 'logreg', 'graphconv', 'tf_regression']

    models = ['tf', 'tf_robust', 'logreg', 'graphconv', 

              'tf_regression', 'graphconvreg']

  if len(datasets) == 0:

    datasets = ['tox21', 'sider', 'muv', 'toxcast', 'pcba', 

                'delaney', 'kaggle']

                'delaney', 'kaggle', 'nci']

  #input hyperparameters

  #tf: dropouts, learning rate, layer_sizes, weight initial stddev,penalty,

                           'penalty': 0.0005, 'penalty_type': 'l2', 

                           'batch_size': 128, 'nb_epoch': 50, 

                           'learning_rate': 0.00008}]

  hps['graphconvreg'] = [{'batch_size': 128, 'nb_epoch': 50, 

                          'learning_rate': 0.0005, 'n_filters': 128, 

                          'n_fully_connected_nodes': 256, 'seed': 123}]

  for split in splitters:

    for dataset in datasets:

        if dataset in ['kaggle']:

          datasets.remove('kaggle') #kaggle only needs to be run once

        for model in models:

          if model in ['tf_regression']:

          if model in ['tf_regression', 'graphconvreg']:

            benchmark_loading_datasets(

                hps, dataset=dataset, model=model, split=split, out_path='.')

def low_data_benchmark_loading_datasets(hyper_parameters, cross_valid=False,

                               dataset='tox21', model='siamese', split='task',

                               verbosity='high', out_path='.'):

                               out_path='.'):

  """

  Loading dataset for low data benchmark test

      time_start_fitting = time.time()

      valid_scores = low_data_benchmark_classification(

                         train_dataset, valid_dataset, hp, n_feat,

                         model=model, verbosity=verbosity)

                         model=model)

      time_finish_fitting = time.time() 

      with open(os.path.join(out_path, 'results.csv'),'a') as f:

        f.write('\n'+str(count_hp)+','+str(count_iter)+',')

def low_data_benchmark_classification(train_dataset, valid_dataset, 

                                      hyper_parameters, n_features, 

                                      model='siamese', seed=123, 

                                      verbosity='high'):

                                      model='siamese', seed=123):

  """

  Calculate low data benchmark performance

  # Initialize metrics

  classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean,

                                            verbosity=verbosity,

                                            mode="classification")

  assert model in ['siamese','attn','res']

    with tf.Session() as sess:

      model_low_data = dc.models.SupportGraphClassifier(

          sess, support_graph, test_batch_size=test_batch_size,

          support_batch_size=support_batch_size, learning_rate=learning_rate,

          verbosity="high")

          support_batch_size=support_batch_size, learning_rate=learning_rate)

      print('-------------------------------------')

      print('Start fitting by graph convolution')

if __name__ == '__main__':

  # Global variables

  np.random.seed(123)

  verbosity = 'high'

  #Working folder initialization

  base_dir_o="/tmp/benchmark_test_"+time.strftime("%Y_%m_%d", time.localtime())

  if os.path.exists(base_dir_o):

    shutil.rmtree(base_dir_o)

  os.makedirs(base_dir_o)

  parser = argparse.ArgumentParser(description='Deepchem benchmark: '+

      'giving performances of different learning models on datasets')

      for model in models:

        low_data_benchmark_loading_datasets(hps, cross_valid=cross_valid, 

                                            dataset=dataset, model=model, 

                                            split=split, verbosity='high', 

                                            out_path='.')

                                            split=split, out_path='.')

import shutil

import deepchem as dc

def load_nci(featurizer='ECFP', shard_size=1000, 

             num_shards_per_batch=4, split='random'):

def load_nci(featurizer='ECFP', shard_size=1000, split='random'):

  current_dir = os.path.dirname(os.path.realpath(__file__))

  loader = dc.data.CSVLoader(

      tasks=all_nci_tasks, smiles_field="smiles", featurizer=featurizer)

  dataset = loader.featurize(dataset_paths, shard_size=shard_size,

                             num_shards_per_batch=num_shards_per_batch)

  dataset = loader.featurize(dataset_paths, shard_size=shard_size)

  # Initialize transformers

  print("About to transform data")