Loading examples/benchmark.py +133 −90 Original line number Diff line number Diff line Loading @@ -6,8 +6,12 @@ Created on Tue Oct 18 15:53:27 2016 @author: Michael Wu Benchmark test Giving performances of RF(scikit) and MultitaskDNN(TF) on datasets: muv, nci, pcba, tox21 Giving performances of: Random forest(rf), MultitaskDNN(tf), RobustMultitaskDNN(tf_robust), Logistic regression(logreg), Graph convolution(graphconv) on datasets: muv, nci, pcba, tox21, sider, toxcast time estimation(on a nvidia tesla K20 GPU): tox21 - dataloading: 30s Loading Loading @@ -64,7 +68,8 @@ def benchmark_loading_datasets(base_dir_o, hyper_parameters, choice of which dataset to use, 'all' = computing all the datasets model : string, optional (default='tf') choice of which model to use, should be: tf, logreg, graphconv choice of which model to use, should be: rf, tf, tf_robust, logreg, graphconv out_path : string, optional(default='/tmp') path of result file Loading Loading @@ -117,15 +122,15 @@ def benchmark_loading_datasets(base_dir_o, hyper_parameters, time_finish_fitting = time.time() with open(os.path.join(out_path, 'results.csv'),'a') as f: f.write('\n\n'+str(count)) f.write('\n'+dname+',train') f.write('\n'+str(count)+',') f.write(dname+',train,') for i in train_score: f.write(','+i+','+str(train_score[i]['mean-roc_auc_score'])) f.write('\n'+dname+',valid') f.write(i+','+str(train_score[i]['mean-roc_auc_score'])+',') f.write('valid,') for i in valid_score: f.write(','+i+','+str(valid_score[i]['mean-roc_auc_score'])) f.write('\n'+dname+',time_for_running,'+ str(time_finish_fitting-time_start_fitting)) f.write(i+','+str(valid_score[i]['mean-roc_auc_score'])+',') f.write('time_for_running,'+ str(time_finish_fitting-time_start_fitting)+',') #clear workspace del tasks,datasets,transformers Loading Loading @@ -161,11 +166,12 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, hyper_parameters : dict of list hyper parameters including dropout rate, learning rate, etc. n_features : integer, optional (default=1024) n_features : integer number of features, or length of binary fingerprints model : string, optional (default='all') choice of which model to use, 'all' = running all models on the dataset model : string, optional (default='tf') choice of which model to use, should be: rf, tf, tf_robust, logreg, graphconv Returns Loading @@ -184,25 +190,33 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, verbosity=verbosity, mode="classification") assert model in ['graphconv', 'tf', 'tf_robust', 'rf','logreg'] assert model in ['rf', 'tf', 'tf_robust', 'logreg', 'graphconv'] if model == 'tf': # Building tensorflow MultiTaskDNN model dropouts = hyper_parameters['dropouts'] learning_rate = hyper_parameters['learning_rate'] # Loading hyper parameters layer_sizes = hyper_parameters['layer_sizes'] weight_init_stddevs = hyper_parameters['weight_init_stddevs'] bias_init_consts = hyper_parameters['bias_init_consts'] dropouts = hyper_parameters['dropouts'] penalty = hyper_parameters['penalty'] penalty_type = hyper_parameters['penalty_type'] batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] learning_rate = hyper_parameters['learning_rate'] model_tf = dc.models.TensorflowMultiTaskClassifier( len(tasks), n_features, learning_rate=learning_rate, layer_sizes=layer_sizes, dropouts=dropouts, batch_size=batch_size, seed=seed, verbosity=verbosity) # Building tensorflow MultiTaskDNN model model_tf = dc.models.TensorflowMultiTaskClassifier(len(tasks), n_features, layer_sizes=layer_sizes, weight_init_stddevs=weight_init_stddevs, bias_init_consts=bias_init_consts, dropouts=dropouts, penalty=penalty, penalty_type=penalty_type, batch_size=batch_size, learning_rate=learning_rate, verbosity=verbosity, seed=seed) print('-------------------------------------') print('Start fitting by tensorflow') print('Start fitting by multitask DNN') model_tf.fit(train_dataset, nb_epoch=nb_epoch) # Evaluating tensorflow MultiTaskDNN model train_scores['tensorflow'] = model_tf.evaluate( train_dataset, [classification_metric], transformers) Loading @@ -210,69 +224,86 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, valid_dataset, [classification_metric], transformers) if model == 'tf_robust': # Building tensorflow MultiTaskDNN model dropouts = hyper_parameters['dropouts'] bypass_dropouts = hyper_parameters['bypass_dropouts'] learning_rate = hyper_parameters['learning_rate'] # Loading hyper parameters layer_sizes = hyper_parameters['layer_sizes'] weight_init_stddevs = hyper_parameters['weight_init_stddevs'] bias_init_consts = hyper_parameters['bias_init_consts'] dropouts = hyper_parameters['dropouts'] bypass_layer_sizes = hyper_parameters['bypass_layer_sizes'] bypass_weight_init_stddevs = hyper_parameters['bypass_weight_init_stddevs'] bypass_bias_init_consts = hyper_parameters['bypass_bias_init_consts'] bypass_dropouts = hyper_parameters['bypass_dropouts'] penalty = hyper_parameters['penalty'] penalty_type = hyper_parameters['penalty_type'] batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] learning_rate = hyper_parameters['learning_rate'] model_robust = dc.models.RobustMultitaskClassifier( len(tasks), n_features, learning_rate=learning_rate, layer_sizes=layer_sizes, bypass_layer_sizes=bypass_layer_sizes, dropouts=dropouts, bypass_dropouts=bypass_dropouts, batch_size=batch_size, seed=seed, verbosity=verbosity) # Building tensorflow robust MultiTaskDNN model model_robust = dc.models.RobustMultitaskClassifier(len(tasks), n_features, layer_sizes=layer_sizes, weight_init_stddevs=weight_init_stddevs, bias_init_consts=bias_init_consts, dropouts=dropouts, bypass_layer_sizes=bypass_layer_sizes, bypass_weight_init_stddevs=bypass_weight_init_stddevs, bypass_bias_init_consts=bypass_bias_init_consts, bypass_dropouts=bypass_dropouts, penalty=penalty, penalty_type=penalty_type, batch_size=batch_size, learning_rate=learning_rate, verbosity=verbosity, seed=seed) print('-------------------------------------') print('Start fitting by tensorflow') print('--------------------------------------------') print('Start fitting by robust multitask DNN') model_robust.fit(train_dataset, nb_epoch=nb_epoch) # Evaluating tensorflow robust MultiTaskDNN model train_scores['tf_robust'] = model_robust.evaluate( train_dataset, [classification_metric], transformers) valid_scores['tf_robust'] = model_robust.evaluate( valid_dataset, [classification_metric], transformers) if model == 'logreg': # Building tensorflow logistic regression model learning_rate = hyper_parameters['learning_rate'] # Loading hyper parameters penalty = hyper_parameters['penalty'] penalty_type = hyper_parameters['penalty_type'] batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] learning_rate = hyper_parameters['learning_rate'] model_logreg = dc.models.TensorflowLogisticRegression( len(tasks), n_features, learning_rate=learning_rate, penalty=penalty, penalty_type=penalty_type, batch_size=batch_size, seed=seed, verbosity=verbosity) # Building tensorflow logistic regression model model_logreg = dc.models.TensorflowLogisticRegression(len(tasks), n_features, penalty=penalty, penalty_type=penalty_type, batch_size=batch_size, learning_rate=learning_rate, verbosity=verbosity, seed=seed) print('-------------------------------------') print('Start fitting by logistic regression') model_logreg.fit(train_dataset, nb_epoch=nb_epoch) train_scores['logreg'] = model_logreg.evaluate(train_dataset, [classification_metric],transformers) # Evaluating tensorflow logistic regression model train_scores['logreg'] = model_logreg.evaluate( train_dataset, [classification_metric], transformers) valid_scores['logreg'] = model_logreg.evaluate(valid_dataset, [classification_metric],transformers) valid_scores['logreg'] = model_logreg.evaluate( valid_dataset, [classification_metric], transformers) if model == 'graphconv': # Initialize model folder model_dir_graphconv = os.path.join(base_dir, "model_graphconv") # Loading hyper parameters batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] learning_rate = hyper_parameters['learning_rate'] n_filters = hyper_parameters['n_filters'] n_fully_connected_nodes = hyper_parameters['n_fully_connected_nodes'] batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) # Building graph convolution model with g.as_default(): tf.set_random_seed(seed) graph_model = dc.nn.SequentialGraph(n_features) Loading @@ -293,19 +324,22 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, batch_size=batch_size, learning_rate=learning_rate, optimizer_type="adam", beta1=.9, beta2=.999, verbosity="high") print('-------------------------------------') print('Start fitting by logistic regression') # Fit trained model model_graphconv.fit(train_dataset, nb_epoch=nb_epoch) # Evaluating graph convolution model train_scores['graphconv'] = model_graphconv.evaluate( train_dataset, [classification_metric], transformers) train_scores['graphconv'] = model_graphconv.evaluate(train_dataset, [classification_metric],transformers) valid_scores['graphconv'] = model_graphconv.evaluate(valid_dataset, [classification_metric],transformers) valid_scores['graphconv'] = model_graphconv.evaluate( valid_dataset, [classification_metric], transformers) if model == 'rf': # Initialize model folder model_dir_rf = os.path.join(base_dir, "model_rf") # Loading hyper parameters n_estimators = hyper_parameters['n_estimators'] # Building scikit random forest model Loading @@ -319,11 +353,13 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, print('-------------------------------------') print('Start fitting by random forest') model_rf.fit(train_dataset) train_scores['random_forest'] = model_rf.evaluate(train_dataset, [classification_metric],transformers) valid_scores['random_forest'] = model_rf.evaluate(valid_dataset, [classification_metric],transformers) # Evaluating scikit random forest model train_scores['random_forest'] = model_rf.evaluate( train_dataset, [classification_metric], transformers) valid_scores['random_forest'] = model_rf.evaluate( valid_dataset, [classification_metric], transformers) return train_scores, valid_scores Loading @@ -339,27 +375,34 @@ if __name__ == '__main__': os.makedirs(base_dir_o) #Datasets and models used in the benchmark test, all=all the datasets dataset_name = 'muv' dataset_name = 'tox21' model = 'tf' #input hyperparameters #tf: dropouts, learning rate, layer_sizes, weight initial stddev,penalty, # batch_size hps = {} hps['tf'] = [{'dropouts': [0.25], 'learning_rate': 0.001, 'layer_sizes': [1000], 'batch_size': 50, 'nb_epoch': 10}] hps['tf_robust'] = [{'dropouts': [0.5], 'bypass_dropouts': [0.5], 'learning_rate': 0.001, 'layer_sizes': [500], 'bypass_layer_sizes': [100], 'batch_size': 50, 'nb_epoch': 10}] hps['logreg'] = [{'learning_rate': 0.001, 'penalty': 0.05, 'penalty_type': 'l1', 'batch_size': 50, 'nb_epoch': 10}] hps['graphconv'] = [{'learning_rate': 0.001, 'n_filters': 64, 'n_fully_connected_nodes': 128, 'batch_size': 50, 'nb_epoch': 10}] hps = {} hps['tf'] = [{'layer_sizes': [500], 'weight_init_stddevs': [0.02], 'bias_init_consts': [1.], 'dropouts': [0.5], 'penalty': 0, 'penalty_type': 'l2', 'batch_size': 50, 'nb_epoch': 10, 'learning_rate': 0.001}] hps['tf_robust'] = [{'layer_sizes': [500], 'weight_init_stddevs': [0.02], 'bias_init_consts': [1.], 'dropouts': [0.5], 'bypass_layer_sizes': [100], 'bypass_weight_init_stddevs': [0.02], 'bypass_bias_init_consts': [1.], 'bypass_dropouts': [0.5], 'penalty': 0, 'penalty_type': 'l2', 'batch_size': 50, 'nb_epoch': 10, 'learning_rate': 0.001}] hps['logreg'] = [{'penalty': 0, 'penalty_type': 'l2', 'batch_size': 50, 'nb_epoch': 10, 'learning_rate': 0.001}] hps['graphconv'] = [{'batch_size': 50, 'nb_epoch': 10, 'learning_rate': 0.001, 'n_filters': 64, 'n_fully_connected_nodes': 128}] hps['rf'] = [{'n_estimators': 500}] Loading examples/run_benchmark.py +86 −39 Original line number Diff line number Diff line Loading @@ -11,61 +11,108 @@ import time import sys from scipy.stats import truncnorm layer_sizes = [800,1200] dropouts = [0.35,0.40,0.45,0.50] penalty_distri = truncnorm(-1,1,loc=0.05,scale=0.04) penalty_type = ['l1'] batch_size = [100] nb_epoch = [50] learning_rate_distri = truncnorm(-0.3,0.3,loc=-3,scale=0.5) n_filters = [64,96,128] n_fully_connected_nodes = [100,120,140,160,200,240,300] n_estimators = [500] # main layer layer_sizes_0 = [1200] weight_init_stddevs_0 = [0.02] bias_init_consts_0 = [1.] dropouts_0 = [0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,0.7] #bypass layer bypass_layer_sizes_0 = truncnorm(-1,1,loc=200,scale=150) bypass_weight_init_stddevs_0 = [.02] bypass_bias_init_consts_0 = [1.] bypass_dropouts_0 = [0.4,0.45,0.5,0.55,0.6,0.65,0.7] #penalty penalty_0 = truncnorm(-1,1,loc=0.3,scale=0.2) penalty_type_0 = ['l2'] #general figure batch_size_0 = [50] nb_epoch_0 = [12] #learning rate learning_rate_0 = truncnorm(-1,1,loc=-3.2,scale=1.2) #for graph-conv and random forest n_filters_0 = [64,96,128] n_fully_connected_nodes_0 = [100,120,140,160,200,240,300] n_estimators_0 = [500] seed = None out_path='.' base_dir_o="/tmp/benchmark_test_"+time.strftime("%Y_%m_%d", time.localtime()) dname = sys.argv[1] model = sys.argv[2] parameters_printed = {'tf': ['dropouts', 'learning_rate', 'layer_sizes', 'batch_size', 'nb_epoch'], 'logreg': ['learning_rate', 'penalty', 'penalty_type', 'batch_size', 'nb_epoch'], 'graphconv': ['learning_rate', 'n_filters', 'n_fully_connected_nodes', 'batch_size', 'nb_epoch'], parameters_printed = {'tf':['layer_sizes', 'weight_init_stddevs', 'bias_init_consts', 'dropouts', 'penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'], 'tf_robust':['layer_sizes', 'weight_init_stddevs', 'bias_init_consts', 'dropouts', 'bypass_layer_sizes', 'bypass_weight_init_stddevs', 'bypass_bias_init_consts', 'bypass_dropouts', 'penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'], 'logreg':['penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'], 'graphconv':['batch_size', 'nb_epoch', 'learning_rate', 'n_filters', 'n_fully_connected_nodes'], 'rf':['n_estimators']} hps = {} hps[model] = [] for i in range(int(sys.argv[3])): ls = np.random.choice(layer_sizes) dp = np.random.choice(dropouts) pn = penalty_distri.rvs() pt = np.random.choice(penalty_type) bs = np.random.choice(batch_size) layer_sizes = layer_sizes_0 weight_init_stddevs = weight_init_stddevs_0 bias_init_consts = bias_init_consts_0 dropouts = [np.random.choice(dropouts_0)] bypass_layer_sizes = [int(bypass_layer_sizes_0.rvs())] bypass_weight_init_stddevs = bypass_weight_init_stddevs_0 bypass_bias_init_consts = bypass_bias_init_consts_0 bypass_dropouts = [np.random.choice(bypass_dropouts_0)] lr = 10.**(learning_rate_distri.rvs()) ne = np.min([np.ceil(0.02/lr),nb_epoch[0]]) penalty = penalty_0.rvs() penalty_type = np.random.choice(penalty_type_0) nf = np.random.choice(n_filters) nn = np.random.choice(n_fully_connected_nodes) nest = np.random.choice(n_estimators) batch_size = np.random.choice(batch_size_0) nb_epoch = np.random.choice(nb_epoch_0) learning_rate = 10**(learning_rate_0.rvs()) n_filters = np.random.choice(n_filters_0) n_fully_connected_nodes = np.random.choice(n_fully_connected_nodes_0) n_estimators = np.random.choice(n_estimators_0) hps[model].append({'layer_sizes': layer_sizes, 'weight_init_stddevs': weight_init_stddevs, 'bias_init_consts': bias_init_consts, 'dropouts': dropouts, 'bypass_layer_sizes': bypass_layer_sizes, 'bypass_weight_init_stddevs': bypass_weight_init_stddevs, 'bypass_bias_init_consts': bypass_bias_init_consts, 'bypass_dropouts': bypass_dropouts, 'penalty': penalty, 'penalty_type': penalty_type, 'batch_size': batch_size, 'nb_epoch': nb_epoch, 'learning_rate': learning_rate, 'n_filters': n_filters, 'n_fully_connected_nodes': n_fully_connected_nodes, 'n_estimators': n_estimators, 'seed': seed}) hps[model].append({'dropouts': [dp], 'learning_rate': lr, 'layer_sizes': [int(ls)], 'penalty': pn, 'penalty_type': pt, 'batch_size': int(bs), 'nb_epoch': int(ne), 'n_filters': 64, 'n_fully_connected_nodes': 128, 'n_estimators': 500}) with open(os.path.join(out_path,'hps.csv'),'a') as f: f.write('\n\n'+str(i)) f.write('\n'+str(i)+','+dname+',') for item in hps[model][i]: if item in parameters_printed[model]: f.write('\n'+item+',') f.write(str(hps[model][i][item])) f.write(item+',') f.write(str(hps[model][i][item])+',') benchmark_loading_datasets(base_dir_o, hps, dataset_name=dname, model=model, reload=True, verbosity='high', out_path=out_path) Loading
examples/benchmark.py +133 −90 Original line number Diff line number Diff line Loading @@ -6,8 +6,12 @@ Created on Tue Oct 18 15:53:27 2016 @author: Michael Wu Benchmark test Giving performances of RF(scikit) and MultitaskDNN(TF) on datasets: muv, nci, pcba, tox21 Giving performances of: Random forest(rf), MultitaskDNN(tf), RobustMultitaskDNN(tf_robust), Logistic regression(logreg), Graph convolution(graphconv) on datasets: muv, nci, pcba, tox21, sider, toxcast time estimation(on a nvidia tesla K20 GPU): tox21 - dataloading: 30s Loading Loading @@ -64,7 +68,8 @@ def benchmark_loading_datasets(base_dir_o, hyper_parameters, choice of which dataset to use, 'all' = computing all the datasets model : string, optional (default='tf') choice of which model to use, should be: tf, logreg, graphconv choice of which model to use, should be: rf, tf, tf_robust, logreg, graphconv out_path : string, optional(default='/tmp') path of result file Loading Loading @@ -117,15 +122,15 @@ def benchmark_loading_datasets(base_dir_o, hyper_parameters, time_finish_fitting = time.time() with open(os.path.join(out_path, 'results.csv'),'a') as f: f.write('\n\n'+str(count)) f.write('\n'+dname+',train') f.write('\n'+str(count)+',') f.write(dname+',train,') for i in train_score: f.write(','+i+','+str(train_score[i]['mean-roc_auc_score'])) f.write('\n'+dname+',valid') f.write(i+','+str(train_score[i]['mean-roc_auc_score'])+',') f.write('valid,') for i in valid_score: f.write(','+i+','+str(valid_score[i]['mean-roc_auc_score'])) f.write('\n'+dname+',time_for_running,'+ str(time_finish_fitting-time_start_fitting)) f.write(i+','+str(valid_score[i]['mean-roc_auc_score'])+',') f.write('time_for_running,'+ str(time_finish_fitting-time_start_fitting)+',') #clear workspace del tasks,datasets,transformers Loading Loading @@ -161,11 +166,12 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, hyper_parameters : dict of list hyper parameters including dropout rate, learning rate, etc. n_features : integer, optional (default=1024) n_features : integer number of features, or length of binary fingerprints model : string, optional (default='all') choice of which model to use, 'all' = running all models on the dataset model : string, optional (default='tf') choice of which model to use, should be: rf, tf, tf_robust, logreg, graphconv Returns Loading @@ -184,25 +190,33 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, verbosity=verbosity, mode="classification") assert model in ['graphconv', 'tf', 'tf_robust', 'rf','logreg'] assert model in ['rf', 'tf', 'tf_robust', 'logreg', 'graphconv'] if model == 'tf': # Building tensorflow MultiTaskDNN model dropouts = hyper_parameters['dropouts'] learning_rate = hyper_parameters['learning_rate'] # Loading hyper parameters layer_sizes = hyper_parameters['layer_sizes'] weight_init_stddevs = hyper_parameters['weight_init_stddevs'] bias_init_consts = hyper_parameters['bias_init_consts'] dropouts = hyper_parameters['dropouts'] penalty = hyper_parameters['penalty'] penalty_type = hyper_parameters['penalty_type'] batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] learning_rate = hyper_parameters['learning_rate'] model_tf = dc.models.TensorflowMultiTaskClassifier( len(tasks), n_features, learning_rate=learning_rate, layer_sizes=layer_sizes, dropouts=dropouts, batch_size=batch_size, seed=seed, verbosity=verbosity) # Building tensorflow MultiTaskDNN model model_tf = dc.models.TensorflowMultiTaskClassifier(len(tasks), n_features, layer_sizes=layer_sizes, weight_init_stddevs=weight_init_stddevs, bias_init_consts=bias_init_consts, dropouts=dropouts, penalty=penalty, penalty_type=penalty_type, batch_size=batch_size, learning_rate=learning_rate, verbosity=verbosity, seed=seed) print('-------------------------------------') print('Start fitting by tensorflow') print('Start fitting by multitask DNN') model_tf.fit(train_dataset, nb_epoch=nb_epoch) # Evaluating tensorflow MultiTaskDNN model train_scores['tensorflow'] = model_tf.evaluate( train_dataset, [classification_metric], transformers) Loading @@ -210,69 +224,86 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, valid_dataset, [classification_metric], transformers) if model == 'tf_robust': # Building tensorflow MultiTaskDNN model dropouts = hyper_parameters['dropouts'] bypass_dropouts = hyper_parameters['bypass_dropouts'] learning_rate = hyper_parameters['learning_rate'] # Loading hyper parameters layer_sizes = hyper_parameters['layer_sizes'] weight_init_stddevs = hyper_parameters['weight_init_stddevs'] bias_init_consts = hyper_parameters['bias_init_consts'] dropouts = hyper_parameters['dropouts'] bypass_layer_sizes = hyper_parameters['bypass_layer_sizes'] bypass_weight_init_stddevs = hyper_parameters['bypass_weight_init_stddevs'] bypass_bias_init_consts = hyper_parameters['bypass_bias_init_consts'] bypass_dropouts = hyper_parameters['bypass_dropouts'] penalty = hyper_parameters['penalty'] penalty_type = hyper_parameters['penalty_type'] batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] learning_rate = hyper_parameters['learning_rate'] model_robust = dc.models.RobustMultitaskClassifier( len(tasks), n_features, learning_rate=learning_rate, layer_sizes=layer_sizes, bypass_layer_sizes=bypass_layer_sizes, dropouts=dropouts, bypass_dropouts=bypass_dropouts, batch_size=batch_size, seed=seed, verbosity=verbosity) # Building tensorflow robust MultiTaskDNN model model_robust = dc.models.RobustMultitaskClassifier(len(tasks), n_features, layer_sizes=layer_sizes, weight_init_stddevs=weight_init_stddevs, bias_init_consts=bias_init_consts, dropouts=dropouts, bypass_layer_sizes=bypass_layer_sizes, bypass_weight_init_stddevs=bypass_weight_init_stddevs, bypass_bias_init_consts=bypass_bias_init_consts, bypass_dropouts=bypass_dropouts, penalty=penalty, penalty_type=penalty_type, batch_size=batch_size, learning_rate=learning_rate, verbosity=verbosity, seed=seed) print('-------------------------------------') print('Start fitting by tensorflow') print('--------------------------------------------') print('Start fitting by robust multitask DNN') model_robust.fit(train_dataset, nb_epoch=nb_epoch) # Evaluating tensorflow robust MultiTaskDNN model train_scores['tf_robust'] = model_robust.evaluate( train_dataset, [classification_metric], transformers) valid_scores['tf_robust'] = model_robust.evaluate( valid_dataset, [classification_metric], transformers) if model == 'logreg': # Building tensorflow logistic regression model learning_rate = hyper_parameters['learning_rate'] # Loading hyper parameters penalty = hyper_parameters['penalty'] penalty_type = hyper_parameters['penalty_type'] batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] learning_rate = hyper_parameters['learning_rate'] model_logreg = dc.models.TensorflowLogisticRegression( len(tasks), n_features, learning_rate=learning_rate, penalty=penalty, penalty_type=penalty_type, batch_size=batch_size, seed=seed, verbosity=verbosity) # Building tensorflow logistic regression model model_logreg = dc.models.TensorflowLogisticRegression(len(tasks), n_features, penalty=penalty, penalty_type=penalty_type, batch_size=batch_size, learning_rate=learning_rate, verbosity=verbosity, seed=seed) print('-------------------------------------') print('Start fitting by logistic regression') model_logreg.fit(train_dataset, nb_epoch=nb_epoch) train_scores['logreg'] = model_logreg.evaluate(train_dataset, [classification_metric],transformers) # Evaluating tensorflow logistic regression model train_scores['logreg'] = model_logreg.evaluate( train_dataset, [classification_metric], transformers) valid_scores['logreg'] = model_logreg.evaluate(valid_dataset, [classification_metric],transformers) valid_scores['logreg'] = model_logreg.evaluate( valid_dataset, [classification_metric], transformers) if model == 'graphconv': # Initialize model folder model_dir_graphconv = os.path.join(base_dir, "model_graphconv") # Loading hyper parameters batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] learning_rate = hyper_parameters['learning_rate'] n_filters = hyper_parameters['n_filters'] n_fully_connected_nodes = hyper_parameters['n_fully_connected_nodes'] batch_size = hyper_parameters['batch_size'] nb_epoch = hyper_parameters['nb_epoch'] g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) # Building graph convolution model with g.as_default(): tf.set_random_seed(seed) graph_model = dc.nn.SequentialGraph(n_features) Loading @@ -293,19 +324,22 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, batch_size=batch_size, learning_rate=learning_rate, optimizer_type="adam", beta1=.9, beta2=.999, verbosity="high") print('-------------------------------------') print('Start fitting by logistic regression') # Fit trained model model_graphconv.fit(train_dataset, nb_epoch=nb_epoch) # Evaluating graph convolution model train_scores['graphconv'] = model_graphconv.evaluate( train_dataset, [classification_metric], transformers) train_scores['graphconv'] = model_graphconv.evaluate(train_dataset, [classification_metric],transformers) valid_scores['graphconv'] = model_graphconv.evaluate(valid_dataset, [classification_metric],transformers) valid_scores['graphconv'] = model_graphconv.evaluate( valid_dataset, [classification_metric], transformers) if model == 'rf': # Initialize model folder model_dir_rf = os.path.join(base_dir, "model_rf") # Loading hyper parameters n_estimators = hyper_parameters['n_estimators'] # Building scikit random forest model Loading @@ -319,11 +353,13 @@ def benchmark_train_and_valid(base_dir, train_dataset, valid_dataset, tasks, print('-------------------------------------') print('Start fitting by random forest') model_rf.fit(train_dataset) train_scores['random_forest'] = model_rf.evaluate(train_dataset, [classification_metric],transformers) valid_scores['random_forest'] = model_rf.evaluate(valid_dataset, [classification_metric],transformers) # Evaluating scikit random forest model train_scores['random_forest'] = model_rf.evaluate( train_dataset, [classification_metric], transformers) valid_scores['random_forest'] = model_rf.evaluate( valid_dataset, [classification_metric], transformers) return train_scores, valid_scores Loading @@ -339,27 +375,34 @@ if __name__ == '__main__': os.makedirs(base_dir_o) #Datasets and models used in the benchmark test, all=all the datasets dataset_name = 'muv' dataset_name = 'tox21' model = 'tf' #input hyperparameters #tf: dropouts, learning rate, layer_sizes, weight initial stddev,penalty, # batch_size hps = {} hps['tf'] = [{'dropouts': [0.25], 'learning_rate': 0.001, 'layer_sizes': [1000], 'batch_size': 50, 'nb_epoch': 10}] hps['tf_robust'] = [{'dropouts': [0.5], 'bypass_dropouts': [0.5], 'learning_rate': 0.001, 'layer_sizes': [500], 'bypass_layer_sizes': [100], 'batch_size': 50, 'nb_epoch': 10}] hps['logreg'] = [{'learning_rate': 0.001, 'penalty': 0.05, 'penalty_type': 'l1', 'batch_size': 50, 'nb_epoch': 10}] hps['graphconv'] = [{'learning_rate': 0.001, 'n_filters': 64, 'n_fully_connected_nodes': 128, 'batch_size': 50, 'nb_epoch': 10}] hps = {} hps['tf'] = [{'layer_sizes': [500], 'weight_init_stddevs': [0.02], 'bias_init_consts': [1.], 'dropouts': [0.5], 'penalty': 0, 'penalty_type': 'l2', 'batch_size': 50, 'nb_epoch': 10, 'learning_rate': 0.001}] hps['tf_robust'] = [{'layer_sizes': [500], 'weight_init_stddevs': [0.02], 'bias_init_consts': [1.], 'dropouts': [0.5], 'bypass_layer_sizes': [100], 'bypass_weight_init_stddevs': [0.02], 'bypass_bias_init_consts': [1.], 'bypass_dropouts': [0.5], 'penalty': 0, 'penalty_type': 'l2', 'batch_size': 50, 'nb_epoch': 10, 'learning_rate': 0.001}] hps['logreg'] = [{'penalty': 0, 'penalty_type': 'l2', 'batch_size': 50, 'nb_epoch': 10, 'learning_rate': 0.001}] hps['graphconv'] = [{'batch_size': 50, 'nb_epoch': 10, 'learning_rate': 0.001, 'n_filters': 64, 'n_fully_connected_nodes': 128}] hps['rf'] = [{'n_estimators': 500}] Loading
examples/run_benchmark.py +86 −39 Original line number Diff line number Diff line Loading @@ -11,61 +11,108 @@ import time import sys from scipy.stats import truncnorm layer_sizes = [800,1200] dropouts = [0.35,0.40,0.45,0.50] penalty_distri = truncnorm(-1,1,loc=0.05,scale=0.04) penalty_type = ['l1'] batch_size = [100] nb_epoch = [50] learning_rate_distri = truncnorm(-0.3,0.3,loc=-3,scale=0.5) n_filters = [64,96,128] n_fully_connected_nodes = [100,120,140,160,200,240,300] n_estimators = [500] # main layer layer_sizes_0 = [1200] weight_init_stddevs_0 = [0.02] bias_init_consts_0 = [1.] dropouts_0 = [0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,0.7] #bypass layer bypass_layer_sizes_0 = truncnorm(-1,1,loc=200,scale=150) bypass_weight_init_stddevs_0 = [.02] bypass_bias_init_consts_0 = [1.] bypass_dropouts_0 = [0.4,0.45,0.5,0.55,0.6,0.65,0.7] #penalty penalty_0 = truncnorm(-1,1,loc=0.3,scale=0.2) penalty_type_0 = ['l2'] #general figure batch_size_0 = [50] nb_epoch_0 = [12] #learning rate learning_rate_0 = truncnorm(-1,1,loc=-3.2,scale=1.2) #for graph-conv and random forest n_filters_0 = [64,96,128] n_fully_connected_nodes_0 = [100,120,140,160,200,240,300] n_estimators_0 = [500] seed = None out_path='.' base_dir_o="/tmp/benchmark_test_"+time.strftime("%Y_%m_%d", time.localtime()) dname = sys.argv[1] model = sys.argv[2] parameters_printed = {'tf': ['dropouts', 'learning_rate', 'layer_sizes', 'batch_size', 'nb_epoch'], 'logreg': ['learning_rate', 'penalty', 'penalty_type', 'batch_size', 'nb_epoch'], 'graphconv': ['learning_rate', 'n_filters', 'n_fully_connected_nodes', 'batch_size', 'nb_epoch'], parameters_printed = {'tf':['layer_sizes', 'weight_init_stddevs', 'bias_init_consts', 'dropouts', 'penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'], 'tf_robust':['layer_sizes', 'weight_init_stddevs', 'bias_init_consts', 'dropouts', 'bypass_layer_sizes', 'bypass_weight_init_stddevs', 'bypass_bias_init_consts', 'bypass_dropouts', 'penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'], 'logreg':['penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'], 'graphconv':['batch_size', 'nb_epoch', 'learning_rate', 'n_filters', 'n_fully_connected_nodes'], 'rf':['n_estimators']} hps = {} hps[model] = [] for i in range(int(sys.argv[3])): ls = np.random.choice(layer_sizes) dp = np.random.choice(dropouts) pn = penalty_distri.rvs() pt = np.random.choice(penalty_type) bs = np.random.choice(batch_size) layer_sizes = layer_sizes_0 weight_init_stddevs = weight_init_stddevs_0 bias_init_consts = bias_init_consts_0 dropouts = [np.random.choice(dropouts_0)] bypass_layer_sizes = [int(bypass_layer_sizes_0.rvs())] bypass_weight_init_stddevs = bypass_weight_init_stddevs_0 bypass_bias_init_consts = bypass_bias_init_consts_0 bypass_dropouts = [np.random.choice(bypass_dropouts_0)] lr = 10.**(learning_rate_distri.rvs()) ne = np.min([np.ceil(0.02/lr),nb_epoch[0]]) penalty = penalty_0.rvs() penalty_type = np.random.choice(penalty_type_0) nf = np.random.choice(n_filters) nn = np.random.choice(n_fully_connected_nodes) nest = np.random.choice(n_estimators) batch_size = np.random.choice(batch_size_0) nb_epoch = np.random.choice(nb_epoch_0) learning_rate = 10**(learning_rate_0.rvs()) n_filters = np.random.choice(n_filters_0) n_fully_connected_nodes = np.random.choice(n_fully_connected_nodes_0) n_estimators = np.random.choice(n_estimators_0) hps[model].append({'layer_sizes': layer_sizes, 'weight_init_stddevs': weight_init_stddevs, 'bias_init_consts': bias_init_consts, 'dropouts': dropouts, 'bypass_layer_sizes': bypass_layer_sizes, 'bypass_weight_init_stddevs': bypass_weight_init_stddevs, 'bypass_bias_init_consts': bypass_bias_init_consts, 'bypass_dropouts': bypass_dropouts, 'penalty': penalty, 'penalty_type': penalty_type, 'batch_size': batch_size, 'nb_epoch': nb_epoch, 'learning_rate': learning_rate, 'n_filters': n_filters, 'n_fully_connected_nodes': n_fully_connected_nodes, 'n_estimators': n_estimators, 'seed': seed}) hps[model].append({'dropouts': [dp], 'learning_rate': lr, 'layer_sizes': [int(ls)], 'penalty': pn, 'penalty_type': pt, 'batch_size': int(bs), 'nb_epoch': int(ne), 'n_filters': 64, 'n_fully_connected_nodes': 128, 'n_estimators': 500}) with open(os.path.join(out_path,'hps.csv'),'a') as f: f.write('\n\n'+str(i)) f.write('\n'+str(i)+','+dname+',') for item in hps[model][i]: if item in parameters_printed[model]: f.write('\n'+item+',') f.write(str(hps[model][i][item])) f.write(item+',') f.write(str(hps[model][i][item])+',') benchmark_loading_datasets(base_dir_o, hps, dataset_name=dname, model=model, reload=True, verbosity='high', out_path=out_path)
