Benchmark

@author: Michael Wu

Benchmark test

Giving performances of RF(scikit) and MultitaskDNN(Keras & TF)

Giving performances of RF(scikit) and MultitaskDNN(TF)

on datasets: muv, nci, pcba, tox21

time estimation(on a nvidia tesla K20 GPU):

tox21 - dataloading: 30s

      - tf: 40s

muv   - dataloading: 400s

      - tf: 250s

pcba  - dataloading: 30min

      - tf: 2h

(will include more)

"""

from __future__ import print_function

from __future__ import division

from pcba.pcba_datasets import load_pcba

from tox21.tox21_datasets import load_tox21

def benchmarkLoadingDatasets(base_dir_o, n_features = 1024, datasetName = 'all',

                             model = 'all', reload = True, verbosity = 'high'):

def benchmark_loading_datasets(base_dir_o, n_features = 1024, 

			      dataset_name = 'all',model = 'all',reload = True, 

                              verbosity = 'high', out_path = '/tmp'):

  """

  Loading dataset for benchmark test

  Parameters

  ----------

  base_dir_o, string

      path of working folder, will be combined with '/dataset_name'

  n_features, integer, optional (default=1024)

      number of features, or length of binary fingerprints

  dataset_name, string, optional (default='all')

      choice of which dataset to use, 'all' means evaluating performance of all the datasets

  model string, optional (default='all')

      choice of which model to use, 'all' means running all models on the dataset

  out_path, string, optional(default='/tmp')

      path of result file

  assert datasetName in ['all', 'muv', 'nci', 'pcba', 'tox21']

  """

  assert dataset_name in ['all', 'muv', 'nci', 'pcba', 'tox21']

  if datasetName == 'all':

  if dataset_name == 'all':

    #currently not including the nci dataset

    datasetName = ['muv','pcba','tox21']

    dataset_name = ['muv','pcba','tox21']

  else:

    datasetName = [datasetName]

    dataset_name = [dataset_name]

  if 'tox21' in datasetName:

  if 'tox21' in dataset_name:

    print('-------------------------------------')

    print('Benchmark test on dataset: tox21')

    print('-------------------------------------')

    base_dir = os.path.join(base_dir_o, "tox21")

    time1 = time.time()

    time_start = time.time()

    #loading datasets for tox21

    tasks_tox21,datasets_tox21,transformers_tox21 = load_tox21(base_dir,

                                                               reload=reload)

    time2 = time.time()

    #time2-time1 is the time(s) used for dataset loading

    time_finish_loading = time.time()

    #time_finish_loading-time_start is the time(s) used for dataset loading

    #dataset splitting, built-in method in load_tox21

    train_dataset, valid_dataset = datasets_tox21

    #running model

    tox21_train,tox21_valid = benchmarkTrainAndValid(base_dir,train_dataset,

    tox21_train,tox21_valid = benchmark_train_and_valid(base_dir,train_dataset,

                                                valid_dataset,

                                                tasks_tox21,transformers_tox21,

                                                n_features, model, verbosity)

    time3 = time.time()

    #time3-time2 is the time(s) used for fitting and evaluating

    time_finish_running = time.time()

    #time_finish_running-time_finish_loading is the time(s) used for fitting and evaluating

    with open('/home/zqwu/deepchem/examples/results.csv','a') as f:

    with open(out_path+'/results.csv','a') as f:

      f.write ('\n'+'tox21,train')

      for i in tox21_train:

        f.write(','+i+','+str(tox21_train[i])) #output train score

      for i in tox21_valid:

        f.write(','+i+','+str(tox21_valid[i])) #output valid score

      #output timing data: running time include all the model

      f.write('\n'+'tox21,time_for_loading,'+str(time2-time1)+'seconds')

      f.write('\n'+'tox21,time_for_running,'+str(time3-time2)+'seconds')

      f.write('\n'+'tox21,time_for_loading,'+

              str(time_finish_loading-time_start)+'seconds')

      f.write('\n'+'tox21,time_for_running,'+

              str(time_finish_running-time_finish_loading)+'seconds')

    #clear workspace         

    del tasks_tox21,datasets_tox21,transformers_tox21

    del train_dataset,valid_dataset,time1,time2,time3

    del train_dataset,valid_dataset

    del time_start,time_finish_loading,time_finish_running

  if 'muv' in datasetName:

  if 'muv' in dataset_name:

    print('-------------------------------------')

    print('Benchmark test on dataset: muv')

    print('-------------------------------------')

    base_dir = os.path.join(base_dir_o, "muv")

    time1 = time.time()

    time_start = time.time()

    #loading datasets for muv

    tasks_muv,datasets_muv,transformers_muv = load_muv(base_dir,reload=reload)

    time2 = time.time()    

    time_finish_loading = time.time()    

    #dataset splitting, built-in method in load_tox21

    train_dataset, valid_dataset = datasets_muv

    #running model

    muv_train,muv_valid = benchmarkTrainAndValid(base_dir,train_dataset,

    muv_train,muv_valid = benchmark_train_and_valid(base_dir,train_dataset,

                                                 valid_dataset,

                                                 tasks_muv, transformers_muv,

                                                 n_features, model, verbosity)

    time3 = time.time()

    time_finish_running = time.time()

    with open('/home/zqwu/deepchem/examples/results.csv','a') as f:

    with open(out_path+'/results.csv','a') as f:

      f.write ('\n'+'muv,train')

      for i in muv_train:

        f.write(','+i+','+str(muv_train[i]))

      f.write('\n'+'muv,valid')

      for i in muv_valid:

        f.write(','+i+','+str(muv_valid[i]))

      f.write('\n'+'muv,time_for_loading,'+str(time2-time1)+'seconds')

      f.write('\n'+'muv,time_for_running,'+str(time3-time2)+'seconds')

      f.write('\n'+'muv,time_for_loading,'+

              str(time_finish_loading-time_start)+'seconds')

      f.write('\n'+'muv,time_for_running,'+

              str(time_finish_running-time_finish_loading)+'seconds')

    del tasks_muv, datasets_muv, transformers_muv

    del train_dataset,valid_dataset,time1,time2,time3

    del train_dataset,valid_dataset

    del time_start,time_finish_loading,time_finish_running

  if 'pcba' in datasetName:

  if 'pcba' in dataset_name:

    print('-------------------------------------')

    print('Benchmark test on dataset: pcba')

    print('-------------------------------------')

    valid_dir = os.path.join(base_dir, "valid_dataset")

    test_dir = os.path.join(base_dir, "test_dataset")

    time1 = time.time()

    time_start = time.time()

    #loading datasets for pcba

    tasks_pcba,datasets_pcba,transformers_pcba = load_pcba(base_dir,

                                                           reload=reload)

    time2 = time.time()

    time_finish_loading = time.time()

    #dataset splitting, RandomSplitter function

    print("About to perform train/valid/test split.")

    train_dataset,valid_dataset,test_dataset = splitter.train_valid_test_split(

                                datasets_pcba, train_dir, valid_dir, test_dir)

    #running model

    pcba_train,pcba_valid = benchmarkTrainAndValid(base_dir,train_dataset,

    pcba_train,pcba_valid = benchmark_train_and_valid(base_dir,train_dataset,

                                                valid_dataset,

                                                tasks_pcba,transformers_pcba,

                                                n_features, model, verbosity)

    time3 = time.time()

    time_finish_running = time.time()

    with open('/home/zqwu/deepchem/examples/results.csv','a') as f:

    with open(out_path+'/results.csv','a') as f:

      f.write ('\n'+'pcba,train')

      for i in pcba_train:

        f.write(','+i+','+str(pcba_train[i]))

      f.write('\n'+'pcba,valid')

      for i in pcba_valid:

        f.write(','+i+','+str(pcba_valid[i]))

      f.write('\n'+'pcba,time_for_loading,'+str(time2-time1)+'seconds')

      f.write('\n'+'pcba,time_for_running,'+str(time3-time2)+'seconds')

      f.write('\n'+'pcba,time_for_loading,'+

              str(time_finish_loading-time_start)+'seconds')

      f.write('\n'+'pcba,time_for_running,'+

              str(time_finish_running-time_finish_loading)+'seconds')

    del tasks_pcba, datasets_pcba, transformers_pcba

    del train_dataset,valid_dataset,time1,time2,time3

    del train_dataset,valid_dataset

    del time_start,time_finish_loading,time_finish_running

  if 'nci' in datasetName:

  if 'nci' in dataset_name:

    print('-------------------------------------')

    print('Benchmark test on dataset: nci')

    print('-------------------------------------')

    valid_dir = os.path.join(base_dir, "valid_dataset")

    test_dir = os.path.join(base_dir, "test_dataset")

    time1 = time.time()

    time_start = time.time()

    #loading datasets for nci

    tasks_nci,datasets_nci,transformers_nci = load_nci(base_dir, reload=reload)

    time2 = time.time()

    time_finish_loading = time.time()

    #dataset splitting, RandomSplitter function

    print("About to perform train/valid/test split.")

    train_dataset,valid_dataset,test_dataset = splitter.train_valid_test_split(

                                datasets_nci, train_dir, valid_dir, test_dir)

    #running model

    nci_train,nci_valid = benchmarkTrainAndValid(base_dir,train_dataset,

    nci_train,nci_valid = benchmark_train_and_valid(base_dir,train_dataset,

                                                 valid_dataset,

                                                 tasks_nci,transformers_nci,

                                                 n_features,model,verbosity)

    time3 = time.time()

    time_finish_running = time.time()

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

        f.write('Time for running nci,'+model+','+str(time3-time2)+'\n')

    with open('/home/zqwu/deepchem/examples/results.csv','a') as f:

    with open(out_path+'/results.csv','a') as f:

      f.write ('\n'+'nci,train')

      for i in nci_train:

        f.write(','+i+','+str(nci_train[i]))

      f.write('\n'+'nci,valid')

      for i in nci_valid:

        f.write(','+i+','+str(nci_valid[i]))

      f.write('\n'+'nci,time_for_loading,'+str(time2-time1)+'seconds')

      f.write('\n'+'nci,time_for_running,'+str(time3-time2)+'seconds')

      f.write('\n'+'nci,time_for_loading,'+

              str(time_finish_loading-time_start)+'seconds')

      f.write('\n'+'nci,time_for_running,'+

              str(time_finish_running-time_finish_loading)+'seconds')

    del tasks_nci, datasets_nci, transformers_nci

    del train_dataset,valid_dataset,time1,time2,time3

    del train_dataset,valid_dataset

    del time_start,time_finish_loading,time_finish_running

  return None

def benchmarkTrainAndValid(base_dir,train_dataset,valid_dataset,tasks,

def benchmark_train_and_valid(base_dir,train_dataset,valid_dataset,tasks,

                              transformers,n_features = 1024,model = 'all',

                              verbosity = 'high'):

  """

  Calculate performance of different models on the specific dataset & tasks

  Parameters

  ----------

  base_dir, string

      path of working folder

  train_dataset, dataset struct

      loaded dataset using load_* or splitter function

  valid_dataset, dataset struct

      loaded dataset using load_* or splitter function

  tasks, list of string

      list of targets(tasks, datasets)

  transformers, BalancingTransformer struct

      loaded properties of dataset from load_* function

  n_features, integer, optional (default=1024)

      number of features, or length of binary fingerprints

  model, string, optional (default='all')

      choice of which model to use, 'all' means running all models on the dataset

  """

  train_scores = {}

  valid_scores = {}

    valid_scores['random_forest'] = valid_evaluator.compute_model_performance(

                                [classification_metric])['mean-roc_auc_score']

  #currently not going to use this model

  '''

  if model == 'all' or model == 'keras':

    # Initialize model folder

    model_dir_kr = os.path.join(base_dir, "model_kr")

    # Building keras MultiTaskDNN model

    keras_model = MultiTaskDNN(len(tasks), n_features, "classification",

                               dropout=.25, learning_rate=.001, decay=1e-4)

    model_kr = KerasModel(keras_model, model_dir_kr, verbosity=verbosity)

    print('-------------------------------------')

    print('Start fitting by keras')

    model_kr.fit(train_dataset)

    train_evaluator = Evaluator(model_kr, train_dataset, transformers, 

                                verbosity=verbosity)

    train_scores['keras'] = train_evaluator.compute_model_performance([classification_metric])['mean-roc_auc_score']

    valid_evaluator = Evaluator(model_kr, valid_dataset, transformers, 

                                verbosity=verbosity)

    valid_scores['keras'] = valid_evaluator.compute_model_performance([classification_metric])['mean-roc_auc_score']

  '''

  return train_scores, valid_scores

if __name__ == '__main__':

  os.makedirs(base_dir)

  #Datasets and models used in the benchmark test, all=all the datasets(models)

  datasetName = sys.argv[1]

  dataset_name = sys.argv[1]

  model = sys.argv[2]

  benchmarkLoadingDatasets(base_dir, n_features = 1024,

                           datasetName = datasetName, model = model,

  benchmark_loading_datasets(base_dir, n_features = 1024,

                             dataset_name = dataset_name, model = model,

                             reload = reload, verbosity = verbosity)