Commit 4c1eb0ba authored by ZHENQIN WU's avatar ZHENQIN WU
Browse files

Benchmark script added

parent f2bc4821

examples/Benchmark.py

+108 −89
Original line number Diff line number Diff line
@@ -45,149 +45,162 @@ def benchmarkLoadingDatasets(base_dir_o, n_features = 1024, datasetName = 'all', 
  

  assert datasetName in ['all', 'muv', 'nci', 'pcba', 'tox21']

  

  if datasetName == 'all':

    datasetName = ['muv','pcba','tox21']

  else:

    datasetName = [datasetName]

      

  if datasetName in ['all','tox21']:

  if 'tox21' in datasetName:

    print('-------------------------------------')

    print('Benchmark test on datasets: tox21')

    print('Benchmark test on dataset: tox21')

    print('-------------------------------------')

    base_dir = os.path.join(base_dir_o, "tox21")

    time1 = time.time()

    #loading datasets for tox21

    tasks_tox21,datasets_tox21,transformers_tox21 = load_tox21(base_dir,

                                                               reload=reload)

    time2 = time.time()

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

      f.write('Time for loading tox21,'+str(time2-time1)+'\n')

    #time2-time1 is the time(s) used for dataset loading

    

    #dataset splitting, built-in method in load_tox21

    train_dataset, valid_dataset = datasets_tox21

    #running model

    tox21_train,tox21_valid = benchmarkTrainAndValid(base_dir,train_dataset,

                                                valid_dataset,

                                                tasks_tox21,transformers_tox21,

                                                n_features, model, verbosity)

    time3 = time.time()

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

        f.write('Time for running tox21,'+model+','+str(time3-time2)+'\n')

    #time3-time2 is the time(s) used for fitting and evaluating

        

    with open('/home/zqwu/deepchem/examples/results.csv','a') as f:

      f.write ('\n'+'tox21_train')

      f.write ('\n'+'tox21,train')

      for i in tox21_train:

        f.write(','+i+','+str(tox21_train[i]))

      f.write('\n'+'tox21_valid')

        f.write(','+i+','+str(tox21_train[i])) #output train score

      f.write('\n'+'tox21,valid')

      for i in tox21_valid:

        f.write(','+i+','+str(tox21_valid[i]))

        f.write(','+i+','+str(tox21_valid[i])) #output valid score

      #output timing data: running time include all the model

      f.write('\n'+'tox21,time_for_loading,'+str(time2-time1)+'seconds')

      f.write('\n'+'tox21,time_for_running,'+str(time3-time2)+'seconds')

    

    #clear workspace         

    del tasks_tox21,datasets_tox21,transformers_tox21

    del train_dataset,valid_dataset

    del train_dataset,valid_dataset,time1,time2,time3



  if datasetName in ['all','muv']:

  if 'muv' in datasetName:

    print('-------------------------------------')

    print('Benchmark test on datasets: muv')

    print('Benchmark test on dataset: muv')

    print('-------------------------------------')

    base_dir = os.path.join(base_dir_o, "muv")

    time1 = time.time()

    #loading datasets for muv

    tasks_muv,datasets_muv,transformers_muv = load_muv(base_dir,reload=reload)

    time2 = time.time()    

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

      f.write('Time for loading MUV,'+str(time2-time1)+'\n')

    

    

    # Spliting datasets

    #dataset splitting, built-in method in load_tox21

    train_dataset, valid_dataset = datasets_muv

    #running model

    muv_train,muv_valid = benchmarkTrainAndValid(base_dir,train_dataset,

                                                 valid_dataset,

                                                 tasks_muv, transformers_muv,

                                                 n_features, model, verbosity)

    time3 = time.time()

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

        f.write('Time for running MUV,'+model+','+str(time3-time2)+'\n')

    

    with open('/home/zqwu/deepchem/examples/results.csv','a') as f:

      f.write ('\n'+'muv_train')

      f.write ('\n'+'muv,train')

      for i in muv_train:

        f.write(','+i+','+str(muv_train[i]))

      f.write('\n'+'muv_valid')

      f.write('\n'+'muv,valid')

      for i in muv_valid:

        f.write(','+i+','+str(muv_valid[i]))

      f.write('\n'+'muv,time_for_loading,'+str(time2-time1)+'seconds')

      f.write('\n'+'muv,time_for_running,'+str(time3-time2)+'seconds')

      

    del tasks_muv, datasets_muv, transformers_muv

    del train_dataset,valid_dataset

    del train_dataset,valid_dataset,time1,time2,time3



  '''  

  if datasetName in ['all','nci']:

  if 'pcba' in datasetName:

    print('-------------------------------------')

    print('Benchmark test on datasets: nci')

    print('Benchmark test on dataset: pcba')

    print('-------------------------------------')

    base_dir = os.path.join(base_dir_o,  "nci")

    base_dir = os.path.join(base_dir_o, "pcba")

    train_dir = os.path.join(base_dir, "train_dataset")

    valid_dir = os.path.join(base_dir, "valid_dataset")

    test_dir = os.path.join(base_dir, "test_dataset")



    time1 = time.time()

    tasks_nci,datasets_nci,transformers_nci = load_nci(base_dir, reload=reload)

    #loading datasets for pcba

    tasks_pcba,datasets_pcba,transformers_pcba = load_pcba(base_dir,

                                                           reload=reload)

    time2 = time.time()

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

        f.write('Time for loading nci,'+str(time2-time1)+'\n')

   

    #dataset splitting, RandomSplitter function

    print("About to perform train/valid/test split.")

    splitter = RandomSplitter(verbosity=verbosity)

    print("Performing new split.")

    train_dataset,valid_dataset,test_dataset = splitter.train_valid_test_split(

                                datasets_nci, train_dir, valid_dir, test_dir)

    nci_train,nci_valid = benchmarkTrainAndValid(base_dir,train_dataset,

                                datasets_pcba, train_dir, valid_dir, test_dir)

    #running model

    pcba_train,pcba_valid = benchmarkTrainAndValid(base_dir,train_dataset,

                                                valid_dataset,

                                                 tasks_nci,transformers_nci,

                                                tasks_pcba,transformers_pcba,

                                                n_features, model, verbosity)

    time3 = time.time()

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

        f.write('Time for running nci,'+model+','+str(time3-time2)+'\n')



    with open('/home/zqwu/deepchem/examples/results.csv','a') as f:

      f.write ('\n'+'nci_train')

      for i in nci_train:

        f.write(','+i+','+str(nci_train[i]))

      f.write('\n'+'nci_valid')

      for i in nci_valid:

        f.write(','+i+','+str(nci_valid[i]))

      f.write ('\n'+'pcba,train')

      for i in pcba_train:

        f.write(','+i+','+str(pcba_train[i]))

      f.write('\n'+'pcba,valid')

      for i in pcba_valid:

        f.write(','+i+','+str(pcba_valid[i]))

      f.write('\n'+'pcba,time_for_loading,'+str(time2-time1)+'seconds')

      f.write('\n'+'pcba,time_for_running,'+str(time3-time2)+'seconds')

     

    del tasks_nci, datasets_nci, transformers_nci

    del train_dataset,valid_dataset

  '''

    del tasks_pcba, datasets_pcba, transformers_pcba

    del train_dataset,valid_dataset,time1,time2,time3



  if datasetName in ['all','pcba']:

  if 'nci' in datasetName:

    print('-------------------------------------')

    print('Benchmark test on datasets: pcba')

    print('Benchmark test on dataset: nci')

    print('-------------------------------------')

    base_dir = os.path.join(base_dir_o, "pcba")

    base_dir = os.path.join(base_dir_o,  "nci")

    train_dir = os.path.join(base_dir, "train_dataset")

    valid_dir = os.path.join(base_dir, "valid_dataset")

    test_dir = os.path.join(base_dir, "test_dataset")



    time1 = time.time()

    tasks_pcba,datasets_pcba,transformers_pcba = load_pcba(base_dir,

                                                           reload=reload)

    #loading datasets for nci

    tasks_nci,datasets_nci,transformers_nci = load_nci(base_dir, reload=reload)

    time2 = time.time()

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

      f.write('Time for loading pcba,'+str(time2-time1)+'\n')

    

    # Spliting datasets

    #dataset splitting, RandomSplitter function

    print("About to perform train/valid/test split.")

    splitter = RandomSplitter(verbosity=verbosity)

    print("Performing new split.")

    train_dataset,valid_dataset,test_dataset = splitter.train_valid_test_split(

                                datasets_pcba, train_dir, valid_dir, test_dir)

    

    pcba_train,pcba_valid = benchmarkTrainAndValid(base_dir,train_dataset,

                                datasets_nci, train_dir, valid_dir, test_dir)

    #running model

    nci_train,nci_valid = benchmarkTrainAndValid(base_dir,train_dataset,

                                                 valid_dataset,

                                                tasks_pcba,transformers_pcba,

                                                 tasks_nci,transformers_nci,

                                                 n_features,model,verbosity)

    time3 = time.time()

    

    with open('/home/zqwu/deepchem/examples/timing.csv','a') as f:

        f.write('Time for running pcba,'+model+','+str(time3-time2)+'\n')

        f.write('Time for running nci,'+model+','+str(time3-time2)+'\n')

    with open('/home/zqwu/deepchem/examples/results.csv','a') as f:

      f.write ('\n'+'pcba_train')

      for i in pcba_train:

        f.write(','+i+','+str(pcba_train[i]))

      f.write('\n'+'pcba_valid')

      for i in pcba_valid:

        f.write(','+i+','+str(pcba_valid[i]))

    del tasks_pcba, datasets_pcba, transformers_pcba

    del train_dataset,valid_dataset

      f.write ('\n'+'nci,train')

      for i in nci_train:

        f.write(','+i+','+str(nci_train[i]))

      f.write('\n'+'nci,valid')

      for i in nci_valid:

        f.write(','+i+','+str(nci_valid[i]))

      f.write('\n'+'nci,time_for_loading,'+str(time2-time1)+'seconds')

      f.write('\n'+'nci,time_for_running,'+str(time3-time2)+'seconds')



    del tasks_nci, datasets_nci, transformers_nci

    del train_dataset,valid_dataset,time1,time2,time3

  

  return None
@@ -197,7 +210,6 @@ def benchmarkTrainAndValid(base_dir,train_dataset,valid_dataset,tasks, 
  train_scores = {}

  valid_scores = {}

  

  

  # Initialize metrics

  classification_metric = Metric(metrics.roc_auc_score, np.mean,

                                 verbosity=verbosity,
@@ -221,10 +233,12 @@ def benchmarkTrainAndValid(base_dir,train_dataset,valid_dataset,tasks, 
    model_tf.fit(train_dataset)

    train_evaluator = Evaluator(model_tf, train_dataset, transformers,

                                verbosity=verbosity)

    train_scores['tensorflow'] = train_evaluator.compute_model_performance([classification_metric])['mean-roc_auc_score']

    train_scores['tensorflow'] = train_evaluator.compute_model_performance(

                                [classification_metric])['mean-roc_auc_score']

    valid_evaluator = Evaluator(model_tf, valid_dataset, transformers,

                                verbosity=verbosity)

    valid_scores['tensorflow'] = valid_evaluator.compute_model_performance([classification_metric])['mean-roc_auc_score']

    valid_scores['tensorflow'] = valid_evaluator.compute_model_performance(

                                [classification_metric])['mean-roc_auc_score']



  

  if model == 'all' or model == 'rf':
@@ -234,7 +248,7 @@ def benchmarkTrainAndValid(base_dir,train_dataset,valid_dataset,tasks, 
    # Building scikit random forest model

    def model_builder(model_dir_rf):

      sklearn_model = RandomForestClassifier(

        class_weight="balanced", n_estimators=500)

        class_weight="balanced", n_estimators=500,n_jobs=-1)

      return SklearnModel(sklearn_model, model_dir_rf)

    model_rf = SingletaskToMultitask(tasks, model_builder, model_dir_rf)
@@ -243,12 +257,15 @@ def benchmarkTrainAndValid(base_dir,train_dataset,valid_dataset,tasks, 
    model_rf.fit(train_dataset)

    train_evaluator = Evaluator(model_rf, train_dataset, transformers, 

                                verbosity=verbosity)

    train_scores['random_forest'] = train_evaluator.compute_model_performance([classification_metric])['mean-roc_auc_score']

    train_scores['random_forest'] = train_evaluator.compute_model_performance(

                                [classification_metric])['mean-roc_auc_score']

    valid_evaluator = Evaluator(model_rf, valid_dataset, transformers, 

                                verbosity=verbosity)

    valid_scores['random_forest'] = valid_evaluator.compute_model_performance([classification_metric])['mean-roc_auc_score']

  

    valid_scores['random_forest'] = valid_evaluator.compute_model_performance(

                                [classification_metric])['mean-roc_auc_score']

  

  #currently not going to use this model

  '''

  if model == 'all' or model == 'keras':

    # Initialize model folder

    model_dir_kr = os.path.join(base_dir, "model_kr")
@@ -266,6 +283,7 @@ def benchmarkTrainAndValid(base_dir,train_dataset,valid_dataset,tasks, 
    valid_evaluator = Evaluator(model_kr, valid_dataset, transformers, 

                                verbosity=verbosity)

    valid_scores['keras'] = valid_evaluator.compute_model_performance([classification_metric])['mean-roc_auc_score']

  '''

  

  return train_scores, valid_scores
@@ -276,7 +294,8 @@ if __name__ == '__main__': 
  verbosity = 'high'

  

  #Working folder initialization

  base_dir = "/scratch/users/zqwu/benchmark_test_"+time.strftime("%Y_%m_%d", time.localtime())

  base_dir = "/tmp/benchmark_test_"+time.strftime(

                                                "%Y_%m_%d", time.localtime())

  if os.path.exists(base_dir):

    shutil.rmtree(base_dir)

  os.makedirs(base_dir)

examples/hardcopy.0

deleted100644 → 0
+0 −24
Original line number Diff line number Diff line 
Loading shard 24 of size 1000 from file.

About to featurize shard.

Currently featurizing feature_type: CircularFingerprint

Featurizing sample 0

TIMING: convert_df_to_numpy y computation took 0.008 s

TIMING: convert_df_to_numpy x computation took 0.072 s

TIMING: convert_df_to_numpy missing elts computation took 0.015 s

TIMING: convert_df_to_numpy took 0.104 s

TIMING: writing metadata row took 0.277 s

TIMING: shard featurization took 6.428 s

TIMING: featurization map function took 6.428 s

TIMING: map call on batch took 25.844 s

Featurized 24000 datapoints



About to start processing next batch of shards

Loading shard 25 of size 1000.

Loading shard 26 of size 1000.

Loading shard 27 of size 1000.

Loading shard 28 of size 1000.

Loading shard 25 of size 1000 from file.

About to featurize shard.

Currently featurizing feature_type: CircularFingerprint

Featurizing sample 0

examples/nci/Benchmark.py

0 → 100644
+309 −0

File added.

Preview size limit exceeded, changes collapsed.

examples/nci_keras_fail.0

deleted100644 → 0
+0 −13851

File deleted.

Preview size limit exceeded, changes collapsed.

examples/pcba_keras.0

deleted100644 → 0
+0 −1208

File deleted.

Preview size limit exceeded, changes collapsed.

CRA Git | Maintained and supported by SUSTech CRA and CCSE