Commit 38763083 authored by leswing's avatar leswing
Browse files

Go to test it out

parent 3538c3b8

devtools/jenkins/compare_results.py

0 → 100644
+54 −0
Original line number Diff line number Diff line 
from nose.tools import assert_true



CUSHION_PERCENT = 0.01

DESIRED_TO_SAMPLE_MAP = {

  "index": "Index splitting",

  "random": "Random splitting",

  "scaffold": "Scaffold splitting",

  "logreg": "logistic regression",

  "tf": "Multitask network",

  "tf_robust": "robust MT-NN",

  "graphconv": "graph convolution",

}





def find_desired_result(result, desired_results):

  vars = result.split(',')

  data_set, split, model = vars[1], DESIRED_TO_SAMPLE_MAP[vars[2]], DESIRED_TO_SAMPLE_MAP[vars[5]]

  for line in desired_results:

    desired_vars = line.split(',')

    if data_set == desired_vars[1] and split == desired_vars[0] and model == desired_vars[2]:

      return float(desired_vars[-2]), float(desired_vars[-1])

  raise Exception("Unable to find desired result \n%s" % result)





def get_my_results(result):

  vars = result.split(',')

  return float(vars[6]), float(vars[9])





def is_good_result(my_result, desired_result):

  for i in range(2):

    my_value = my_result[i]

    desired_value = desired_result[i]

    if my_value > desired_value * (1.0 + CUSHION_PERCENT):

      return False

  return True





def test_compare_results():

  desired_results = open("devtools/jenkins/desired_results.csv").readlines()

  given_results = open("results.csv").readlines()

  exceptions = []

  for result in given_results:

    desired_result = find_desired_result(result, desired_results)

    my_result = get_my_results(result)

    if not is_good_result(my_result, desired_result):

      exceptions.append((result, my_result, desired_result))

    if len(exceptions) > 0:

      for exception in exceptions:

        print(exception)

    assert_true(len(exceptions) == 0, "Some performance benchmarks not passed")



  if __name__ == "__main__":

    test_compare_results()

devtools/jenkins/desired_results.csv

0 → 100644
+73 −0
Original line number Diff line number Diff line 
split,dataset,model,Train score/ROC-AUC,Valid score/ROC-AUC

Index splitting,tox21,logistic regression,0.903,0.705

Index splitting,tox21,Multitask network,0.856,0.763

Index splitting,tox21,robust MT-NN,0.857,0.767

Index splitting,tox21,graph convolution,0.872,0.798

Index splitting,muv,logistic regression,0.963,0.766

Index splitting,muv,Multitask network,0.904,0.764

Index splitting,muv,robust MT-NN,0.934,0.781

Index splitting,muv,graph convolution,0.840,0.823

Index splitting,pcba,logistic regression,0.809,0.776

Index splitting,pcba,Multitask network,0.826,0.802

Index splitting,pcba,robust MT-NN,0.809,0.783

Index splitting,pcba,graph convolution,0.876,0.852

Index splitting,sider,logistic regression,0.933,0.620

Index splitting,sider,Multitask network,0.775,0.634

Index splitting,sider,robust MT-NN,0.803,0.632

Index splitting,sider,graph convolution,0.708,0.594

Index splitting,toxcast,logistic regression,0.721,0.575

Index splitting,toxcast,Multitask network,0.830,0.678

Index splitting,toxcast,robust MT-NN,0.825,0.680

Index splitting,toxcast,graph convolution,0.821,0.720

Index splitting,clintox,logistic regression,0.967,0.676

Index splitting,clintox,Multitask network,0.934,0.830

Index splitting,clintox,robust MT-NN,0.949,0.827

Index splitting,clintox,graph convolution,0.946,0.860

Random splitting,tox21,logistic regression,0.903,0.735

Random splitting,tox21,Multitask network,0.856,0.783

Random splitting,tox21,robust MT-NN,0.855,0.773

Random splitting,tox21,graph convolution,0.865,0.827

Random splitting,muv,logistic regression,0.957,0.719

Random splitting,muv,Multitask network,0.902,0.734

Random splitting,muv,robust MT-NN,0.933,0.732

Random splitting,muv,graph convolution,0.860,0.730

Random splitting,pcba,logistic regression,0.808,0.776

Random splitting,pcba,Multitask network,0.811,0.778

Random splitting,pcba,robust MT-NN,0.811,0.771

Random splitting,pcba,graph convolution,0.872,0.844

Random splitting,sider,logistic regression,0.929,0.656

Random splitting,sider,Multitask network,0.777,0.655

Random splitting,sider,robust MT-NN,0.804,0.630

Random splitting,sider,graph convolution,0.705,0.618

Random splitting,toxcast,logistic regression,0.725,0.586

Random splitting,toxcast,Multitask network,0.836,0.684

Random splitting,toxcast,robust MT-NN,0.822,0.681

Random splitting,toxcast,graph convolution,0.820,0.717

Random splitting,clintox,logistic regression,0.972,0.725

Random splitting,clintox,Multitask network,0.951,0.834

Random splitting,clintox,robust MT-NN,0.959,0.830

Random splitting,clintox,graph convolution,0.975,0.876

Scaffold splitting,tox21,logistic regression,0.900,0.650

Scaffold splitting,tox21,Multitask network,0.863,0.703

Scaffold splitting,tox21,robust MT-NN,0.861,0.710

Scaffold splitting,tox21,graph convolution,0.885,0.732

Scaffold splitting,muv,logistic regression,0.947,0.767

Scaffold splitting,muv,Multitask network,0.899,0.762

Scaffold splitting,muv,robust MT-NN,0.944,0.726

Scaffold splitting,muv,graph convolution,0.872,0.795

Scaffold splitting,pcba,logistic regression,0.810,0.742

Scaffold splitting,pcba,Multitask network,0.814,0.760

Scaffold splitting,pcba,robust MT-NN,0.812,0.756

Scaffold splitting,pcba,graph convolution,0.874,0.817

Scaffold splitting,sider,logistic regression,0.926,0.592

Scaffold splitting,sider,Multitask network,0.776,0.557

Scaffold splitting,sider,robust MT-NN,0.797,0.560

Scaffold splitting,sider,graph convolution,0.722,0.583

Scaffold splitting,toxcast,logistic regression,0.716,0.492

Scaffold splitting,toxcast,Multitask network,0.828,0.617

Scaffold splitting,toxcast,robust MT-NN,0.830,0.614

Scaffold splitting,toxcast,graph convolution,0.832,0.638

Scaffold splitting,clintox,logistic regression,0.960,0.803

Scaffold splitting,clintox,Multitask network,0.947,0.862

Scaffold splitting,clintox,robust MT-NN,0.953,0.890

Scaffold splitting,clintox,graph convolution,0.957,0.823

devtools/jenkins/jenkins.sh

+5 −2
Original line number Diff line number Diff line 
#/bin/bash

#!/usr/bin/env bash

envname=`cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1`

conda create --name $envname

source activate $envname
@@ -17,6 +17,9 @@ python setup.py install 


cd examples

python benchmark.py -d tox21

cd ..

nosetests -v devtools/jenkins/compare_results.py --with-xunit || true



source deactivate

conda remove --name $envname --all

rm results.csv
 
 No newline at end of file

devtools/jenkins/results.table

0 → 100644
+86 −0
Original line number Diff line number Diff line 
Index splitting



|Dataset    |Model               |Train score/ROC-AUC|Valid score/ROC-AUC|

|-----------|--------------------|-------------------|-------------------|

|tox21      |logistic regression |0.903              |0.705              |

|           |Multitask network   |0.856              |0.763              |

|           |robust MT-NN        |0.857              |0.767              |

|           |graph convolution   |0.872              |0.798              |

|muv        |logistic regression |0.963              |0.766              |

|           |Multitask network   |0.904              |0.764              |

|           |robust MT-NN        |0.934              |0.781              |

|           |graph convolution   |0.840              |0.823              |

|pcba       |logistic regression |0.809              |0.776              |

|           |Multitask network   |0.826              |0.802              |

|           |robust MT-NN        |0.809              |0.783              |

|           |graph convolution   |0.876              |0.852              |

|sider      |logistic regression |0.933              |0.620              |

|           |Multitask network   |0.775              |0.634              |

|           |robust MT-NN        |0.803              |0.632              |

|           |graph convolution   |0.708              |0.594              |

|toxcast    |logistic regression |0.721              |0.575              |

|           |Multitask network   |0.830              |0.678              |

|           |robust MT-NN        |0.825              |0.680              |

|           |graph convolution   |0.821              |0.720              |

|clintox    |logistic regression |0.967              |0.676              |

|           |Multitask network   |0.934              |0.830              |

|           |robust MT-NN        |0.949              |0.827              |

|           |graph convolution   |0.946              |0.860              |



Random splitting



|Dataset    |Model               |Train score/ROC-AUC|Valid score/ROC-AUC|

|-----------|--------------------|-------------------|-------------------|

|tox21      |logistic regression |0.903              |0.735              |

|           |Multitask network   |0.856              |0.783              |

|           |robust MT-NN        |0.855              |0.773              |

|           |graph convolution   |0.865              |0.827              |

|muv        |logistic regression |0.957              |0.719              |

|           |Multitask network   |0.902              |0.734              |

|           |robust MT-NN        |0.933              |0.732              |

|           |graph convolution   |0.860              |0.730              |

|pcba       |logistic regression |0.808        	     |0.776              |

|           |Multitask network   |0.811        	     |0.778              |

|           |robust MT-NN        |0.811              |0.771              |

|           |graph convolution   |0.872       	     |0.844              |

|sider      |logistic regression |0.929        	     |0.656              |

|           |Multitask network   |0.777        	     |0.655              |

|           |robust MT-NN        |0.804              |0.630              |

|           |graph convolution   |0.705        	     |0.618              |

|toxcast    |logistic regression |0.725        	     |0.586              |

|           |Multitask network   |0.836        	     |0.684              |

|           |robust MT-NN        |0.822              |0.681              |

|           |graph convolution   |0.820        	     |0.717              |

|clintox    |logistic regression |0.972              |0.725              |

|           |Multitask network   |0.951              |0.834              |

|           |robust MT-NN        |0.959              |0.830              |

|           |graph convolution   |0.975              |0.876              |



Scaffold splitting



|Dataset    |Model               |Train score/ROC-AUC|Valid score/ROC-AUC|

|-----------|--------------------|-------------------|-------------------|

|tox21      |logistic regression |0.900              |0.650              |

|           |Multitask network   |0.863              |0.703              |

|           |robust MT-NN        |0.861              |0.710              |

|           |graph convolution   |0.885              |0.732              |

|muv        |logistic regression |0.947              |0.767              |

|           |Multitask network   |0.899              |0.762              |

|           |robust MT-NN        |0.944              |0.726              |

|           |graph convolution   |0.872              |0.795              |

|pcba       |logistic regression |0.810              |0.742              |

|           |Multitask network   |0.814              |0.760              |

|           |robust MT-NN        |0.812              |0.756              |

|           |graph convolution   |0.874              |0.817              |

|sider      |logistic regression |0.926              |0.592              |

|           |Multitask network   |0.776              |0.557              |

|           |robust MT-NN        |0.797              |0.560              |

|           |graph convolution   |0.722              |0.583              |

|toxcast    |logistic regression |0.716              |0.492              |

|           |Multitask network   |0.828              |0.617              |

|           |robust MT-NN        |0.830              |0.614              |

|           |graph convolution   |0.832              |0.638              |

|clintox    |logistic regression |0.960              |0.803              |

|           |Multitask network   |0.947              |0.862              |

|           |robust MT-NN        |0.953              |0.890              |

|           |graph convolution   |0.957              |0.823              |
 
 No newline at end of file

devtools/jenkins/sample_results.csv

0 → 100644
+12 −0
Original line number Diff line number Diff line 
0,tox21,index,classification,train,tf,0.857374210957,valid,tf,0.763705248461,time_for_running,62.337130069732666

0,tox21,index,classification,train,tf_robust,0.851230951439,valid,tf_robust,0.761026632635,time_for_running,103.45717263221741

0,tox21,index,classification,train,logreg,0.902582168307,valid,logreg,0.704655144339,time_for_running,58.77318286895752

0,tox21,index,classification,train,graphconv,0.885263409928,valid,graphconv,0.810359597317,time_for_running,253.0348720550537

0,tox21,random,classification,train,tf,0.84454312661,valid,tf,0.79544852983,time_for_running,61.921756744384766

0,tox21,random,classification,train,tf_robust,0.850847847035,valid,tf_robust,0.771402095982,time_for_running,103.8645749092102

0,tox21,random,classification,train,logreg,0.90284653346,valid,logreg,0.715422756626,time_for_running,56.38301730155945

0,tox21,random,classification,train,graphconv,0.89114965469,valid,graphconv,0.829789213747,time_for_running,251.0880537033081

0,tox21,scaffold,classification,train,tf,0.860443872891,valid,tf,0.729043342895,time_for_running,60.44110417366028

0,tox21,scaffold,classification,train,tf_robust,0.858757182477,valid,tf_robust,0.747690569439,time_for_running,103.23447942733765

0,tox21,scaffold,classification,train,logreg,0.903081839621,valid,logreg,0.702282040447,time_for_running,59.35971403121948

0,tox21,scaffold,classification,train,graphconv,0.911592979851,valid,graphconv,0.764893014635,time_for_running,243.28665375709534

CRA Git | Maintained and supported by SUSTech CRA and CCSE