Merge pull request #354 from miaecle/pdbbind_fix

* Regression

|Dataset         |Model               |Splitting   |Train score/R2|Valid score/R2|

|-----------|--------------------|------------|--------------|--------------|

|----------------|--------------------|------------|--------------|--------------|

|delaney         |MT-NN regression    |Index       |0.773         |0.574         |

|                |graphconv regression|Index       |0.991         |0.825         |

|                |MT-NN regression    |Random      |0.769         |0.591         |

|                |graphconv regression|Random      |0.117         |0.076         |

|                |MT-NN regression    |Scaffold    |0.180         |0.052         |

|                |graphconv regression|Scaffold    |0.131         |0.046         |

|pdbbind(core)   |MT-NN regression    |Random      |0.973         |0.494         |

|pdbbind(refined)|MT-NN regression    |Random      |0.987         |0.503         |

|pdbbind(full)   |MT-NN regression    |Random      |0.983         |0.528         |

|kaggle          |MT-NN regression    |User-defined|0.748         |0.452         |

* General features

Number of tasks and examples in the datasets

|Dataset         |N(tasks)	|N(samples) |

|-----------|-----------|-----------| 

|----------------|-----------|-----------| 

|tox21           |12         |8014       |

|muv             |17         |93127      |

|pcba            |128        |439863     |

|delaney         |1          |1128       |

|kaggle          |15         |173065     |

|nci             |60         |19127      |

|pdbbind(core)   |1          |195        |

|pdbbind(refined)|1          |3706       |

|pdbbind(full)   |1          |11908      |

Time needed for benchmark test(~20h in total)

|Dataset         |Model               |Time(loading)/s |Time(running)/s|

|-----------|--------------------|----------------|---------------| 

|----------------|--------------------|----------------|---------------| 

|tox21           |logistic regression |30              |60             |

|                |Multitask network   |30              |60             |

|                |robust MT-NN        |30              |90             |

|                |graphconv regression|10              |40             |

|nci             |MT-NN regression    |400             |1200           |

|                |graphconv regression|400             |2500           |

|pdbbind(core)   |MT-NN regression    |0(featurized)   |30             |

|pdbbind(refined)|MT-NN regression    |0(featurized)   |40             |

|pdbbind(full)   |MT-NN regression    |0(featurized)   |60             |

|kaggle          |MT-NN regression    |2200            |3200           |

    """

    np.testing.assert_almost_equal(frac_train + frac_valid + frac_test, 1.)

    if not seed is None:

      np.random.seed(seed)

    mws = []

    Splits internal compounds randomly into train/validation/test.

    """

    np.testing.assert_almost_equal(frac_train + frac_valid + frac_test, 1.)

    if not seed is None:

      np.random.seed(seed)

    num_datapoints = len(dataset)

    train_cutoff = int(frac_train * num_datapoints)

Giving regression performances of:

    MultitaskDNN(tf_regression),

    Graph convolution regression(graphconvreg)

on datasets: delaney, nci, kaggle

on datasets: delaney, nci, kaggle, pdbbind

time estimation listed in README file

from kaggle.kaggle_datasets import load_kaggle

from delaney.delaney_datasets import load_delaney

from nci.nci_datasets import load_nci

from pdbbind.pdbbind_datasets import load_pdbbind_grid

def benchmark_loading_datasets(hyper_parameters, 

                               dataset='tox21', model='tf', split=None,

  if dataset in ['muv', 'pcba', 'tox21', 'sider', 'toxcast']:

    mode = 'classification'

  elif dataset in ['kaggle', 'delaney', 'nci']:

  elif dataset in ['kaggle', 'delaney', 'nci','pdbbind']:

    mode = 'regression'

  else:

    raise ValueError('Dataset not supported')

  if dataset in ['kaggle']:

    featurizer = None #kaggle dataset use its own features

    if split in ['random', 'scaffold']:

      return

    else:

      split = None #kaggle dataset is already splitted

    if not model in ['tf_regression']:

      return

  if dataset in ['pdbbind']:

    featurizer = 'grid' #pdbbind use grid featurizer

    if split in ['scaffold', 'index']:

      return #skip the scaffold and index splitting of pdbbind

    if not model in ['tf_regression']:

      return

  if not split in [None, 'index','random','scaffold']:

    raise ValueError('Splitter function not supported')

  loading_functions = {'tox21': load_tox21, 'muv': load_muv,

                       'pcba': load_pcba, 'nci': load_nci,

                       'sider': load_sider, 'toxcast': load_toxcast,

                       'kaggle': load_kaggle, 'delaney': load_delaney}

                       'kaggle': load_kaggle, 'delaney': load_delaney,

                       'pdbbind': load_pdbbind_grid}

  print('-------------------------------------')

  print('Benchmark %s on dataset: %s' % (model, dataset))

  train_dataset, valid_dataset, test_dataset = all_dataset

  time_finish_loading = time.time()

  #time_finish_loading-time_start is the time(s) used for dataset loading

  if dataset in ['kaggle']:

  if dataset in ['kaggle','pdbbind']:

    n_features = train_dataset.get_data_shape()[0]

    #kaggle dataset has customized features

           'tf_regression, graphconvreg')

  parser.add_argument('-d', action='append', dest='dataset_args', default=[], 

      help='Choice of dataset: tox21, sider, muv, toxcast, pcba, ' + 

           'kaggle, delaney, nci')

           'kaggle, delaney, nci, pdbbind')

  args = parser.parse_args()

  #Datasets and models used in the benchmark test

  splitters = args.splitter_args

              'tf_regression', 'graphconvreg']

  if len(datasets) == 0:

    datasets = ['tox21', 'sider', 'muv', 'toxcast', 'pcba', 

                'delaney', 'nci', 'kaggle']

                'delaney', 'nci', 'kaggle', 'pdbbind']

  #input hyperparameters

  #tf: dropouts, learning rate, layer_sizes, weight initial stddev,penalty,

            benchmark_loading_datasets(

                hps, dataset=dataset, model=model, split=split, out_path='.')

      else:

        if dataset in ['kaggle']:

          datasets.remove('kaggle') #kaggle only needs to be run once

        for model in models:

          if model in ['tf_regression', 'graphconvreg']:

            benchmark_loading_datasets(

  dataset = dc.data.DiskDataset.from_numpy(X, y, w, ids, data_dir=data_dir)

  return dataset, tasks

def load_pdbbind_grid(split="index", feat="grid", subset="core"):

def load_pdbbind_grid(split="index", featurizer="grid", subset="full"):

  """Load PDBBind datasets. Does not do train/test split"""

  dataset, tasks = featurize_pdbbind(feat=feat, subset=subset)

  dataset, tasks = featurize_pdbbind(feat=featurizer, subset=subset)

  splitters = {'index': dc.splits.IndexSplitter(),

               'random': dc.splits.RandomSplitter()}