Remove verbose

               weight_init_stddevs=[.02], bias_init_consts=[1.], penalty=0.0,

               penalty_type="l2", dropouts=[0.5], learning_rate=.001,

               momentum=.9, optimizer="adam", batch_size=50, n_classes=2,

               verbosity="high", seed=None, **kwargs):

               verbose=True, seed=None, **kwargs):

    """Constructs the computational graph.

    This function constructs the computational graph for the model. It relies

    n_classes: int

      Number of classes if this is for classification.

      TODO(rbharath): Move this argument to TensorflowClassifier

    verbosity: str

      Must be one of ['high', 'low', None]. Amount of logging to do.

    verbose: True 

      Perform logging.

    seed: int

      If not none, is used as random seed for tensorflow. 

    """

    self.optimizer = optimizer

    self.batch_size = batch_size

    self.n_classes = n_classes

    self.verbosity = verbosity

    self.verbose= verbose

    self.seed = seed

    if logdir is not None:

    ############################################################## TIMING

    time1 = time.time()

    ############################################################## TIMING

    log("Training for %d epochs" % nb_epoch, self.verbosity)

    log("Training for %d epochs" % nb_epoch, self.verbose)

    with self.train_graph.graph.as_default():

      train_op = self.get_training_op(

          self.train_graph.graph, self.train_graph.loss)

              #dataset.iterbatches(batch_size, pad_batches=True)):

              dataset.iterbatches(self.batch_size, pad_batches=pad_batches)):

            if ind % log_every_N_batches == 0:

              log("On batch %d" % ind, self.verbosity)

              log("On batch %d" % ind, self.verbose)

            # Run training op.

            feed_dict = self.construct_feed_dict(X_b, y_b, w_b, ids_b)

            fetches = self.train_graph.output + [

            n_batches += 1

          saver.save(sess, self._save_path, global_step=epoch)

          avg_loss = float(avg_loss)/n_batches

          log('Ending epoch %d: Average loss %g' % (epoch, avg_loss), self.verbosity)

          log('Ending epoch %d: Average loss %g' % (epoch, avg_loss), self.verbose)

        # Always save a final checkpoint when complete.

        saver.save(sess, self._save_path, global_step=epoch+1)

    ############################################################## TIMING

    time2 = time.time()

    print("TIMING: model fitting took %0.3f s" % (time2-time1),

          self.verbosity)

          self.verbose)

    ############################################################## TIMING

  def add_output_ops(self, graph, output):

  def __init__(self, sess, model, n_tasks, logdir=None, batch_size=50,

               final_loss='cross_entropy', learning_rate=.001,

               optimizer_type="adam", learning_rate_decay_time=1000,

               beta1=.9, beta2=.999, verbosity=None):

               beta1=.9, beta2=.999, verbose=True):

    self.verbosity = verbosity

    self.verbose = verbose

    self.sess = sess

    self.n_tasks = n_tasks

    self.final_loss = final_loss

  def fit(self, dataset, nb_epoch=10, 

          max_checkpoints_to_keep=5, log_every_N_batches=50, **kwargs):

    # Perform the optimization

    log("Training for %d epochs" % nb_epoch, self.verbosity)

    log("Training for %d epochs" % nb_epoch, self.verbose)

    # TODO(rbharath): Disabling saving for now to try to debug.

    for epoch in range(nb_epoch):

      log("Starting epoch %d" % epoch, self.verbosity)

      log("Starting epoch %d" % epoch, self.verbose)

      for batch_num, (X_b, y_b, w_b, ids_b) in enumerate(dataset.iterbatches(

          self.batch_size, pad_batches=True)):

        if batch_num % log_every_N_batches == 0:

          log("On batch %d" % batch_num, self.verbosity)

          log("On batch %d" % batch_num, self.verbose)

        self.sess.run(

            self.train_op,

            feed_dict=self.construct_feed_dict(X_b, y_b, w_b))

from kaggle.kaggle_datasets import load_kaggle

from delaney.delaney_datasets import load_delaney

def benchmark_loading_datasets(base_dir_o, hyper_parameters, 

def benchmark_loading_datasets(hyper_parameters, 

                               dataset='tox21', model='tf', split=None,

                               reload=True, verbosity='high', 

                               out_path='.'):

                               reload=True, out_path='.'):

  """

  Loading dataset for benchmark test

  Parameters

  ----------

  base_dir_o : string

      path of working folder, will be combined with '/dataset_name'

  hyper_parameters: dict of list

      hyper parameters including dropout rate, learning rate, etc.

  dataset: string, optional (default='tox21')

      choice of which dataset to use, should be: tox21, muv, sider, 

      toxcast, pcba

  model: string,  optional (default='tf')

      choice of which model to use, should be: rf, tf, tf_robust, logreg,

      graphconv

  split: string,  optional (default=None)

      choice of splitter function, None = using the default splitter

  out_path: string, optional(default='.')

      path of result file

  """

  if dataset in ['muv','nci','pcba','tox21','sider','toxcast']:

  print('-------------------------------------')

  print('Benchmark %s on dataset: %s' % (model, dataset))

  print('-------------------------------------')

  base_dir = os.path.join(base_dir_o, dataset)

  time_start = time.time()

  #loading datasets

  if split is not None:

  for count, hp in enumerate(hyper_parameters[model]):

    time_start_fitting = time.time()

    if mode == 'classification':

      train_score, valid_score = benchmark_classification(base_dir,

      train_score, valid_score = benchmark_classification(

          train_dataset, valid_dataset, tasks, 

          transformers, hp, n_features,

                                     model=model, verbosity=verbosity)      

          model=model)      

    elif mode == 'regression':

      train_score, valid_score = benchmark_regression(base_dir,

      train_score, valid_score = benchmark_regression(

          train_dataset, valid_dataset, tasks, 

          transformers, hp, n_features,

                                     model=model, verbosity=verbosity)  

          model=model)  

    time_finish_fitting = time.time()

    with open(os.path.join(out_path, 'results.csv'),'a') as f:

      f.write('time_for_running,'+

              str(time_finish_fitting-time_start_fitting)+',')

  return None

def benchmark_classification(base_dir, train_dataset, valid_dataset, tasks,

def benchmark_classification(train_dataset, valid_dataset, tasks,

                             transformers, hyper_parameters, 

                            n_features, model='tf', seed=123, 

                            verbosity='high'):

                             n_features, model='tf', seed=123):

  """

  Calculate performance of different models on the specific dataset & tasks

  Parameters

  ----------

  base_dir : string

      path of working folder

  train_dataset: dataset struct

      loaded dataset using load_* or splitter function

  valid_dataset: dataset struct

      loaded dataset using load_* or splitter function

  tasks: list of string

      list of targets(tasks, datasets)

  transformers: BalancingTransformer struct

      loaded properties of dataset from load_* function

  hyper_parameters: dict

      hyper parameters including dropout rate, learning rate, etc.

  n_features: integer

      number of features, or length of binary fingerprints

  model: string,  optional (default='tf')

      choice of which model to use, should be: rf, tf, tf_robust, logreg,

      graphconv

  valid_scores = {}

  # Initialize metrics

  classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean,

                                            verbosity=verbosity,

                                            mode="classification")

  classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean)

  assert model in ['rf', 'tf', 'tf_robust', 'logreg', 'graphconv']

        weight_init_stddevs=weight_init_stddevs,

        bias_init_consts=bias_init_consts, dropouts=dropouts, penalty=penalty, 

        penalty_type=penalty_type, batch_size=batch_size, 

        learning_rate=learning_rate, verbosity=verbosity, seed=seed)

        learning_rate=learning_rate, seed=seed)

    print('-------------------------------------')

    print('Start fitting by multitask DNN')

        bypass_bias_init_consts=bypass_bias_init_consts,

        bypass_dropouts=bypass_dropouts, penalty=penalty, 

        penalty_type=penalty_type, batch_size=batch_size,

        learning_rate=learning_rate, verbosity=verbosity, seed=seed)

        learning_rate=learning_rate, seed=seed)

    print('--------------------------------------------')

    print('Start fitting by robust multitask DNN')

    model_logreg = dc.models.TensorflowLogisticRegression(len(tasks),

        n_features, penalty=penalty, penalty_type=penalty_type, 

        batch_size=batch_size, learning_rate=learning_rate, 

        verbosity=verbosity, seed=seed)

        seed=seed)

    print('-------------------------------------')

    print('Start fitting by logistic regression')

  if model == 'graphconv':

    # Initialize model folder

    model_dir_graphconv = os.path.join(base_dir, "model_graphconv")

    # Loading hyper parameters

    batch_size = hyper_parameters['batch_size']

      graph_model.add(dc.nn.GraphGather(batch_size, activation="tanh"))

      with tf.Session() as sess:

        model_graphconv = dc.models.MultitaskGraphClassifier(

          sess, graph_model, len(tasks), model_dir_graphconv, 

          sess, graph_model, len(tasks), 

          batch_size=batch_size, learning_rate=learning_rate,

          optimizer_type="adam", beta1=.9, beta2=.999, verbosity="high")

          optimizer_type="adam", beta1=.9, beta2=.999)

        print('-------------------------------------')

        print('Start fitting by graph convolution')

  if model == 'rf':

    # Initialize model folder

    model_dir_rf = os.path.join(base_dir, "model_rf")

    # Loading hyper parameters

    n_estimators = hyper_parameters['n_estimators']

        class_weight="balanced", n_estimators=n_estimators,n_jobs=-1)

      return dc.models.sklearn_models.SklearnModel(sklearn_model, model_dir_rf)

    model_rf = dc.models.multitask.SingletaskToMultitask(

		tasks, model_builder, model_dir_rf)

        tasks, model_builder)

    print('-------------------------------------')

    print('Start fitting by random forest')

  return train_scores, valid_scores

def benchmark_regression(base_dir, train_dataset, valid_dataset, tasks,

def benchmark_regression(train_dataset, valid_dataset, tasks,

                         transformers, hyper_parameters, 

                         n_features, model='tf_regression', seed=123, 

                         verbosity='high'):

                         n_features, model='tf_regression', seed=123):

  """

  Calculate performance of different models on the specific dataset & tasks

  Parameters

  ----------

  base_dir : string

      path of working folder

  train_dataset: dataset struct

      loaded dataset using load_* or splitter function

  valid_dataset: dataset struct

      loaded dataset using load_* or splitter function

  tasks: list of string

      list of targets(tasks, datasets)

  transformers: BalancingTransformer struct

      loaded properties of dataset from load_* function

  hyper_parameters: dict

      hyper parameters including dropout rate, learning rate, etc.

  n_features: integer

      number of features, or length of binary fingerprints

  model: string,  optional (default='tf_regression')

      choice of which model to use, should be: tf_regression

      predicting results(R2) on training set

  valid_scores: dict

      predicting results(R2) on valid set

  """

  train_scores = {}

  valid_scores = {}

  # Initialize metrics

  regression_metric = dc.metrics.Metric(dc.metrics.pearson_r2_score, np.mean,

                                        verbosity=verbosity)

  regression_metric = dc.metrics.Metric(dc.metrics.pearson_r2_score, np.mean)

  assert model in ['tf_regression']

        weight_init_stddevs=weight_init_stddevs,

        bias_init_consts=bias_init_consts, dropouts=dropouts, penalty=penalty, 

        penalty_type=penalty_type, batch_size=batch_size, 

        learning_rate=learning_rate, verbosity=verbosity, seed=seed)

        learning_rate=learning_rate, seed=seed)

    print('-----------------------------------------')

    print('Start fitting by multitask DNN regression')

if __name__ == '__main__':

  # Global variables

  np.random.seed(123)

  verbosity = 'high'

  #Working folder initialization

  base_dir_o="/tmp/benchmark_test_"+time.strftime("%Y_%m_%d", time.localtime())

  if os.path.exists(base_dir_o):

    shutil.rmtree(base_dir_o)

  os.makedirs(base_dir_o)

  parser = argparse.ArgumentParser(description='Deepchem benchmark: '+

      'giving performances of different learning models on datasets')

      if dataset in ['tox21', 'sider', 'muv', 'toxcast', 'pcba']:

        for model in models:

          if model in ['tf', 'tf_robust', 'logreg', 'graphconv']:

            benchmark_loading_datasets(base_dir_o, hps, dataset=dataset, 

                                       model=model, split=split, 

                                       verbosity='high', out_path='.')

            benchmark_loading_datasets(

                hps, dataset=dataset, model=model, split=split, out_path='.')

      else:

        for model in models:

          if model in ['tf_regression']:

             benchmark_loading_datasets(base_dir_o, hps, dataset=dataset, 

                                        model=model, split=split, 

                                        verbosity='high', out_path='.')

            benchmark_loading_datasets(

                hps, dataset=dataset, model=model, split=split, out_path='.')