Commit 24e55ba4 authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #718 from miaecle/temp 

Benchmark script clean-up
parents bee3b453 ce8a61b6

deepchem/models/tensorgraph/models/graph_models.py

+3 −3
Original line number Diff line number Diff line
@@ -139,7 +139,7 @@ class WeaveTensorGraph(TensorGraph): 
              feed_dict[label] = to_one_hot(y_b[:, index])

            if self.mode == "regression":

              feed_dict[label] = y_b[:, index:index + 1]

        if w_b is not None and not predict:

        if w_b is not None:

          feed_dict[self.weights] = w_b



        atom_feat = []
@@ -289,7 +289,7 @@ class DTNNTensorGraph(TensorGraph): 
        if y_b is not None and not predict:

          for index, label in enumerate(self.labels_fd):

            feed_dict[label] = y_b[:, index:index + 1]

        if w_b is not None and not predict:

        if w_b is not None:

          feed_dict[self.weights] = w_b

        distance = []

        atom_membership = []
@@ -440,7 +440,7 @@ class DAGTensorGraph(TensorGraph): 
              feed_dict[label] = to_one_hot(y_b[:, index])

            if self.mode == "regression":

              feed_dict[label] = y_b[:, index:index + 1]

        if w_b is not None and not predict:

        if w_b is not None:

          feed_dict[self.weights] = w_b



        atoms_per_mol = [mol.get_num_atoms() for mol in X_b]

deepchem/molnet/run_benchmark.py

+2 −2
Original line number Diff line number Diff line
@@ -71,7 +71,6 @@ def run_benchmark(datasets, 
      if metric == None:

        metric = [

            deepchem.metrics.Metric(deepchem.metrics.roc_auc_score, np.mean),

            deepchem.metrics.Metric(deepchem.metrics.prc_auc_score, np.mean)

        ]

    elif dataset in [

        'bace_r', 'chembl', 'clearance', 'delaney', 'hopv', 'kaggle', 'lipo',
@@ -187,7 +186,8 @@ def run_benchmark(datasets, 
      model_name = list(train_score.keys())[0]

      for i in train_score[model_name]:

        output_line = [

            dataset, str(split), mode, model_name, i, 'train',

            dataset,

            str(split), mode, model_name, i, 'train',

            train_score[model_name][i], 'valid', valid_score[model_name][i]

        ]

        if test:

devtools/jenkins/compare_results.py

+7 −6
Original line number Diff line number Diff line
@@ -38,11 +38,11 @@ def parse_test_results(test_results): 
  for line in test_results:

    vars = line.split(',')

    retval.append({

        "split": BENCHMARK_TO_DESIRED_KEY_MAP[vars[2]],

        "data_set": vars[1],

        "model": BENCHMARK_TO_DESIRED_KEY_MAP[vars[5]],

        "split": BENCHMARK_TO_DESIRED_KEY_MAP[vars[1]],

        "data_set": vars[0],

        "model": BENCHMARK_TO_DESIRED_KEY_MAP[vars[3]],

        "train_score": float(vars[6]),

        "test_score": float(vars[9])

        "test_score": float(vars[8])

    })

  return retval
@@ -63,8 +63,9 @@ def is_good_result(my_result, desired_result): 
    # Higher is Better

    desired_value = desired_result[key] - CUSHION_PERCENT

    if my_result[key] < desired_value or LOG_ALL_RESULTS:

      message_part = "%s,%s,%s,%s,%s,%s" % (

          my_result['data_set'], my_result['model'], my_result['split'], key,

      message_part = "%s,%s,%s,%s,%s,%s" % (my_result['data_set'],

                                            my_result['model'],

                                            my_result['split'], key,

                                            my_result[key], desired_result[key])

      message.append(message_part)

      retval = False

examples/benchmark.py

+35 −1022

File changed.

Preview size limit exceeded, changes collapsed.

examples/benchmark2.py

deleted100644 → 0
+0 −110
Original line number Diff line number Diff line 
#!/usr/bin/env python3

# -*- coding: utf-8 -*-

"""

Created on Tue Oct 18 15:53:27 2016



@author: Michael Wu



Benchmark test:



Giving classification performances of: 

    Random forest(rf), MultitaskDNN(tf), 

    RobustMultitaskDNN(tf_robust),

    Logistic regression(logreg), IRV(irv)

    Graph convolution(graphconv), xgboost(xgb),

    Directed acyclic graph(dag), Weave(weave) 

on datasets: bace_c, bbbp, clintox, hiv, muv, pcba, sider, tox21, toxcast  



Giving regression performances of:

    MultitaskDNN(tf_regression),

    Fit Transformer MultitaskDNN(tf_regression_ft),

    Random forest(rf_regression),

    Graph convolution regression(graphconvreg),

    xgboost(xgb_regression), Deep tensor neural net(dtnn),

    Directed acyclic graph(dag_regression),

    Weave(weave_regression)

on datasets: bace_r, chembl, clearance, delaney(ESOL), hopv, kaggle, lipo,

             nci, pdbbind, ppb, qm7, qm7b, qm8, qm9, sampl(FreeSolv)

                



time estimation listed in README file



"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



import os

import numpy as np

import deepchem as dc

import argparse



parser = argparse.ArgumentParser(

    description='Deepchem benchmark: ' +

    'giving performances of different learning models on datasets')

parser.add_argument(

    '-s',

    action='append',

    dest='splitter_args',

    default=[],

    help='Choice of splitting function: index, random, scaffold, stratified')

parser.add_argument(

    '-m',

    action='append',

    dest='model_args',

    default=[],

    help='Choice of model: tf, tf_robust, logreg, rf, irv, graphconv, xgb,' + \

         ' dag, weave, tf_regression, tf_regression_ft, rf_regression, ' + \

         'graphconvreg, xgb_regression, dtnn, dag_regression, weave_regression')

parser.add_argument(

    '-d',

    action='append',

    dest='dataset_args',

    default=[],

    help='Choice of dataset: bace_c, bace_r, bbbp, chembl, clearance, ' +

    'clintox, delaney, hiv, hopv, kaggle, lipo, muv, nci, pcba, ' +

    'pdbbind, ppb, qm7, qm7b, qm8, qm9, sampl, sider, tox21, toxcast')

parser.add_argument(

    '-t',

    action='store_true',

    dest='test',

    default=False,

    help='Evalute performance on test set')

parser.add_argument(

    '--seed',

    action='append',

    dest='seed_args',

    default=[],

    help='Choice of random seed')

args = parser.parse_args()

#Datasets and models used in the benchmark test

splitters = args.splitter_args

models = args.model_args

datasets = args.dataset_args

test = args.test

if len(args.seed_args) > 0:

  seed = int(args.seed_args[0])

else:

  seed = 123



if len(splitters) == 0:

  splitters = ['index', 'random', 'scaffold']

if len(models) == 0:

  models = [

      'tf', 'tf_robust', 'logreg', 'graphconv', 'tf_regression',

      'tf_regression_ft', 'graphconvreg'

  ]

  #irv, rf, rf_regression should be assigned manually

if len(datasets) == 0:

  datasets = [

      'bace_c', 'bace_r', 'bbbp', 'clearance', 'clintox', 'delaney', 'hiv',

      'hopv', 'lipo', 'muv', 'pdbbind', 'ppb', 'qm7b', 'qm8', 'qm9', 'sampl',

      'sider', 'tox21', 'toxcast'

  ]



for dataset in datasets:

  for split in splitters:

    for model in models:

      np.random.seed(seed)

      dc.molnet.run_benchmark(

          [dataset], str(model), split=split, test=test, seed=seed)

CRA Git | Maintained and supported by SUSTech CRA and CCSE