Update quantum machine notebook (dddf1da3) · Commits · 钟慕尧 / deepchem

examples/quantum_machine.ipynb

+378 −138

Original line number	Diff line number	Diff line
		%% Cell type:markdown id: tags:

		Setting up imports

		%% Cell type:code id: tags:

		``` python
		%load_ext autoreload
		%autoreload 2
		%pdb off
		from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		__author__ = "Joseph Gomes"
		__copyright__ = "Copyright 2016, Stanford University"
		__license__ = "LGPL"

		import os
		import unittest
		import tempfile
		import shutil

		import numpy as np
		import numpy.random

		from deepchem.utils.evaluate import Evaluator
		from deepchem import metrics
		from deepchem.datasets import Dataset
		from deepchem.featurizers.featurize import DataFeaturizer
		from deepchem.featurizers.featurize import FeaturizedSamples
		from deepchem.hyperparameters import HyperparamOpt
		from deepchem.metrics import Metric
		from deepchem.models import Model
		from deepchem.models.deep import SingleTaskDNN
		from deepchem.models.keras_models.fcnet import SingleTaskDNN
		from deepchem.models.sklearn_models import SklearnModel
		from deepchem.transformers import NormalizationTransformer
		from deepchem.datasets import Dataset
		from deepchem.transformers import CoulombBinarizationTransformer
		from deepchem.transformers import CoulombRandomizationTransformer
		from deepchem.utils.evaluate import Evaluator

		from sklearn.ensemble import RandomForestRegressor
		from sklearn.kernel_ridge import KernelRidge
		```

		%% Output

		Automatic pdb calling has been turned OFF

		Using Theano backend.

		%% Cell type:markdown id: tags:

		Creating temporary directories

		%% Cell type:code id: tags:

		``` python
		feature_dir = tempfile.mkdtemp()
		samples_dir = tempfile.mkdtemp()
		train_dir = tempfile.mkdtemp()
		valid_dir = tempfile.mkdtemp()
		test_dir = tempfile.mkdtemp()
		model_dir = tempfile.mkdtemp()
		```

		%% Cell type:markdown id: tags:

		Setting up model variables

		%% Cell type:code id: tags:

		``` python
		splittype = "random"
		compound_featurizers = []
		complex_featurizers = []
		feature_types = ["user_specified_features"]
		user_specified_features = ["evals"]
		task_types = {"u0_atom": "regression"}
		#input_file = "../datasets/gdb7k.csv"
		input_file = "../datasets/gdb7k.pkl.gz"
		smiles_field = "smiles"
		protein_pdb_field = None
		ligand_pdb_field = None
		```

		%% Cell type:markdown id: tags:

		Load featurized data

		%% Cell type:code id: tags:

		``` python
		featurizers = compound_featurizers + complex_featurizers
		featurizer = DataFeaturizer(tasks=task_types.keys(),
		smiles_field=smiles_field,
		protein_pdb_field=protein_pdb_field,
		ligand_pdb_field=ligand_pdb_field,
		compound_featurizers=compound_featurizers,
		complex_featurizers=complex_featurizers,
		user_specified_features=user_specified_features,
		verbose=False)
		user_specified_features=user_specified_features)
		featurized_samples = featurizer.featurize(input_file, feature_dir, samples_dir)
		```

		%% Cell type:markdown id: tags:

		Perform Train, Validation, and Testing Split

		%% Cell type:code id: tags:

		``` python
		train_samples, valid_samples, test_samples = featurized_samples.train_valid_test_split(
		splittype, train_dir, valid_dir, test_dir)
		```

		%% Output

		FeaturizedSamples.train_valid_test_split
		dataset_files
		[u'/local-scratch/joegomes/8001366/tmpAUHlIC/features_shard0.joblib', u'/local-scratch/joegomes/8001366/tmpAUHlIC/features_shard1.joblib', u'/local-scratch/joegomes/8001366/tmpAUHlIC/features_shard2.joblib', u'/local-scratch/joegomes/8001366/tmpAUHlIC/features_shard3.joblib', u'/local-scratch/joegomes/8001366/tmpAUHlIC/features_shard4.joblib', u'/local-scratch/joegomes/8001366/tmpAUHlIC/features_shard5.joblib', u'/local-scratch/joegomes/8001366/tmpAUHlIC/features_shard6.joblib']

		%% Cell type:markdown id: tags:

		Creating datasets

		%% Cell type:code id: tags:

		``` python
		train_dataset = Dataset(data_dir=train_dir, samples=train_samples,
		featurizers=featurizers, tasks=task_types.keys(),
		use_user_specified_features=True)
		valid_dataset = Dataset(data_dir=valid_dir, samples=valid_samples,
		featurizers=featurizers, tasks=task_types.keys(),
		use_user_specified_features=True)
		test_dataset = Dataset(data_dir=test_dir, samples=test_samples,
		featurizers=featurizers, tasks=task_types.keys(),
		use_user_specified_features=True)
		```

		%% Output

		/home/joegomes/deepchem/deepchem/datasets/__init__.py:252: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
		/home/joegomes/deepchem/deepchem/datasets/__init__.py:290: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
		if features[feature_ind] == "":
		/home/joegomes/deepchem/deepchem/datasets/__init__.py:260: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
		/home/joegomes/deepchem/deepchem/datasets/__init__.py:298: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
		if y[ind, task] == "":

		%% Cell type:markdown id: tags:

		Transforming datasets

		%% Cell type:code id: tags:

		``` python
		input_transformers = [NormalizationTransformer(transform_X=True, dataset=train_dataset)]
		output_transformers = [NormalizationTransformer(transform_y=True, dataset=train_dataset)]
		transformers = input_transformers + output_transformers
		for transformer in transformers:
		transformer.transform(train_dataset)
		for transformer in transformers:
		transformer.transform(valid_dataset)
		for transformer in transformers:
		transformer.transform(test_dataset)
		```

		%% Output

		update_moments -- data_shape = (23,)
		update_moments -- data_shape = (23,)

		%% Cell type:markdown id: tags:

		Fit Random Forest with hyperparameter search

		%% Cell type:code id: tags:

		``` python
		np.random.seed()
		params_dict = {"activation": ["relu"],
		"momentum": [.9],
		"batch_size": [50],
		"init": ["glorot_uniform"],
		"data_shape": [train_dataset.get_data_shape()],
		"learning_rate": np.power(10., np.random.uniform(-5, -2, size=1)),
		"decay": np.power(10., np.random.uniform(-6, -4, size=1)),
		"nb_hidden": [1000],
		"nb_epoch": [50],
		"nesterov": [True],
		"dropout": [.0001,.001,.01],
		"nb_layers": [1, 2],
		"batchnorm": [False],
		}

		metric = Metric(metrics.mean_absolute_error)
		optimizer = HyperparamOpt(SingleTaskDNN, task_types, verbosity="high")
		best_dnn, best_hyperparams, all_results = optimizer.hyperparam_search(
		params_dict, train_dataset, valid_dataset, output_transformers, metric,
		use_max=False)
		```

		%% Output

		Starting epoch 1
		Starting epoch 2
		Starting epoch 3
		Starting epoch 4
		Starting epoch 5
		Starting epoch 6
		Starting epoch 7
		Starting epoch 8
		Starting epoch 9
		Starting epoch 10
		Starting epoch 11
		Starting epoch 12
		Starting epoch 13
		Starting epoch 14
		Starting epoch 15
		Starting epoch 16
		Starting epoch 17
		Starting epoch 18
		Starting epoch 19
		Starting epoch 20
		Starting epoch 21
		Starting epoch 22
		Starting epoch 23
		Starting epoch 24
		Starting epoch 25
		Starting epoch 26
		Starting epoch 27
		Starting epoch 28
		Starting epoch 29
		Starting epoch 30
		Starting epoch 31
		Starting epoch 32
		Starting epoch 33
		Starting epoch 34
		Starting epoch 35
		Starting epoch 36
		Starting epoch 37
		Starting epoch 38
		Starting epoch 39
		Starting epoch 40
		Starting epoch 41
		Starting epoch 42
		Starting epoch 43
		Starting epoch 44
		Starting epoch 45
		Starting epoch 46
		Starting epoch 47
		Starting epoch 48
		Starting epoch 49
		Starting epoch 50
		Saving predictions to <open file '<fdopen>', mode 'w+b' at 0x7fe8d34eba50>
		Saving model performance scores to <open file '<fdopen>', mode 'w+b' at 0x7fe8c929a9c0>
		Model 0/6, Metric mean_absolute_error, Validation set 0: 14.848437
		best_validation_score so far: 14.848437
		Starting epoch 1
		Starting epoch 2
		Starting epoch 3
		Starting epoch 4
		Starting epoch 5
		Starting epoch 6
		Starting epoch 7
		Starting epoch 8
		Starting epoch 9
		Starting epoch 10
		Starting epoch 11
		Starting epoch 12
		Starting epoch 13
		Starting epoch 14
		Starting epoch 15
		Starting epoch 16
		Starting epoch 17
		Starting epoch 18
		Starting epoch 19
		Starting epoch 20
		Starting epoch 21
		Starting epoch 22
		Starting epoch 23
		Starting epoch 24
		Starting epoch 25
		Starting epoch 26
		Starting epoch 27
		Starting epoch 28
		Starting epoch 29
		Starting epoch 30
		Starting epoch 31
		Starting epoch 32
		Starting epoch 33
		Starting epoch 34
		Starting epoch 35
		Starting epoch 36
		Starting epoch 37
		Starting epoch 38
		Starting epoch 39
		Starting epoch 40
		Starting epoch 41
		Starting epoch 42
		Starting epoch 43
		Starting epoch 44
		Starting epoch 45
		Starting epoch 46
		Starting epoch 47
		Starting epoch 48
		Starting epoch 49
		Starting epoch 50
		Saving predictions to <open file '<fdopen>', mode 'w+b' at 0x7fe8d34eba50>
		Saving model performance scores to <open file '<fdopen>', mode 'w+b' at 0x7fe8c929a9c0>
		Model 1/6, Metric mean_absolute_error, Validation set 1: 14.157977
		best_validation_score so far: 14.157977
		Starting epoch 1
		Starting epoch 2
		Starting epoch 3
		Starting epoch 4
		Starting epoch 5
		Starting epoch 6
		Starting epoch 7
		Starting epoch 8
		Starting epoch 9
		Starting epoch 10
		Starting epoch 11
		Starting epoch 12
		Starting epoch 13
		Starting epoch 14
		Starting epoch 15
		Starting epoch 16
		Starting epoch 17
		Starting epoch 18
		Starting epoch 19
		Starting epoch 20
		Starting epoch 21
		Starting epoch 22
		Starting epoch 23
		Starting epoch 24
		Starting epoch 25
		Starting epoch 26
		Starting epoch 27
		Starting epoch 28
		Starting epoch 29
		Starting epoch 30
		Starting epoch 31
		Starting epoch 32
		Starting epoch 33
		Starting epoch 34
		Starting epoch 35
		Starting epoch 36
		Starting epoch 37
		Starting epoch 38
		Starting epoch 39
		Starting epoch 40
		Starting epoch 41
		Starting epoch 42
		Starting epoch 43
		Starting epoch 44
		Starting epoch 45
		Starting epoch 46
		Starting epoch 47
		Starting epoch 48
		Starting epoch 49
		Starting epoch 50
		Saving predictions to <open file '<fdopen>', mode 'w+b' at 0x7fe8d34eba50>
		Saving model performance scores to <open file '<fdopen>', mode 'w+b' at 0x7fe8c929a9c0>
		Model 2/6, Metric mean_absolute_error, Validation set 2: 14.777830
		best_validation_score so far: 14.157977
		Starting epoch 1
		Starting epoch 2
		Starting epoch 3
		Starting epoch 4
		Starting epoch 5
		Starting epoch 6
		Starting epoch 7
		Starting epoch 8
		Starting epoch 9
		Starting epoch 10
		Starting epoch 11
		Starting epoch 12
		Starting epoch 13
		Starting epoch 14
		Starting epoch 15
		Starting epoch 16
		Starting epoch 17
		Starting epoch 18
		Starting epoch 19
		Starting epoch 20
		Starting epoch 21
		Starting epoch 22
		Starting epoch 23
		Starting epoch 24
		Starting epoch 25
		Starting epoch 26
		Starting epoch 27
		Starting epoch 28
		Starting epoch 29
		Starting epoch 30
		Starting epoch 31
		Starting epoch 32
		Starting epoch 33
		Starting epoch 34
		Starting epoch 35
		Starting epoch 36
		Starting epoch 37
		Starting epoch 38
		Starting epoch 39
		Starting epoch 40
		Starting epoch 41
		Starting epoch 42
		Starting epoch 43
		Starting epoch 44
		Starting epoch 45
		Starting epoch 46
		Starting epoch 47
		Starting epoch 48
		Starting epoch 49
		Starting epoch 50
		Saving predictions to <open file '<fdopen>', mode 'w+b' at 0x7fe8d34eba50>
		Saving model performance scores to <open file '<fdopen>', mode 'w+b' at 0x7fe8c929a9c0>
		Model 3/6, Metric mean_absolute_error, Validation set 3: 14.591309
		best_validation_score so far: 14.157977
		Starting epoch 1
		Starting epoch 2
		Starting epoch 3
		Starting epoch 4
		Starting epoch 5
		Starting epoch 6
		Starting epoch 7
		Starting epoch 8
		Starting epoch 9
		Starting epoch 10
		Starting epoch 11
		Starting epoch 12
		Starting epoch 13
		Starting epoch 14
		Starting epoch 15
		Starting epoch 16
		Starting epoch 17
		Starting epoch 18
		Starting epoch 19
		Starting epoch 20
		Starting epoch 21
		Starting epoch 22
		Starting epoch 23
		Starting epoch 24
		Starting epoch 25
		Starting epoch 26
		Starting epoch 27
		Starting epoch 28
		Starting epoch 29
		Starting epoch 30
		Starting epoch 31
		Starting epoch 32
		Starting epoch 33
		Starting epoch 34
		Starting epoch 35
		Starting epoch 36
		Starting epoch 37
		Starting epoch 38
		Starting epoch 39
		Starting epoch 40
		Starting epoch 41
		Starting epoch 42
		Starting epoch 43
		Starting epoch 44
		Starting epoch 45
		Starting epoch 46
		Starting epoch 47
		Starting epoch 48
		Starting epoch 49
		Starting epoch 50
		Saving predictions to <open file '<fdopen>', mode 'w+b' at 0x7fe8d34eba50>
		Saving model performance scores to <open file '<fdopen>', mode 'w+b' at 0x7fe8c929a9c0>
		Model 4/6, Metric mean_absolute_error, Validation set 4: 16.323611
		best_validation_score so far: 14.157977
		Starting epoch 1
		Starting epoch 2
		Starting epoch 3
		Starting epoch 4
		Starting epoch 5
		Starting epoch 6
		Starting epoch 7
		Starting epoch 8
		Starting epoch 9
		Starting epoch 10
		Starting epoch 11
		Starting epoch 12
		Starting epoch 13
		Starting epoch 14
		Starting epoch 15
		Starting epoch 16
		Starting epoch 17
		Starting epoch 18
		Starting epoch 19
		Starting epoch 20
		Starting epoch 21
		Starting epoch 22
		Starting epoch 23
		Starting epoch 24
		Starting epoch 25
		Starting epoch 26
		Starting epoch 27
		Starting epoch 28
		Starting epoch 29
		Starting epoch 30
		Starting epoch 31
		Starting epoch 32
		Starting epoch 33
		Starting epoch 34
		Starting epoch 35
		Starting epoch 36
		Starting epoch 37
		Starting epoch 38
		Starting epoch 39
		Starting epoch 40
		Starting epoch 41
		Starting epoch 42
		Starting epoch 43
		Starting epoch 44
		Starting epoch 45
		Starting epoch 46
		Starting epoch 47
		Starting epoch 48
		Starting epoch 49
		Starting epoch 50
		Saving predictions to <open file '<fdopen>', mode 'w+b' at 0x7fe8d34eba50>
		Saving model performance scores to <open file '<fdopen>', mode 'w+b' at 0x7fe8c929a9c0>
		Model 5/6, Metric mean_absolute_error, Validation set 5: 15.977724
		best_validation_score so far: 14.157977
		Saving predictions to <open file '<fdopen>', mode 'w+b' at 0x7fe8c582c150>
		Saving model performance scores to <open file '<fdopen>', mode 'w+b' at 0x7fe8c582c4b0>
		Best hyperparameters: [(u'data_shape', (23,)), (u'activation', u'relu'), (u'batch_size', 50), (u'nb_layers', 1), (u'decay', 8.4518635705285321e-05), (u'nesterov', True), (u'nb_epoch', 50), (u'learning_rate', 0.0003919210239049525), (u'init', u'glorot_uniform'), (u'nb_hidden', 1000), (u'batchnorm', False), (u'dropout', 0.001), (u'momentum', 0.9)]
		train_score: 16.801887
		validation_score: 14.157977

		%% Cell type:code id: tags:

		``` python
		def model_builder(task_types, params_dict, verbosity):
		n_estimators = params_dict["n_estimators"]
		max_features = params_dict["max_features"]
		return SklearnModel(
		task_types, params_dict,
		model_instance=RandomForestRegressor(n_estimators=n_estimators,
		max_features=max_features))
		params_dict = {
		"n_estimators": [10, 100],
		"data_shape": [train_dataset.get_data_shape()],
		"max_features": ["auto", "sqrt", "log2", None],
		}
		optimizer = HyperparamOpt(model_builder, task_types)
		best_rf, best_rf_hyperparams, all_rf_results = optimizer.hyperparam_search(
		params_dict, train_dataset, valid_dataset, output_transformers, metric="mae", use_max=False)
		```

		%% Output

		Model 0/8, Metric mae, Validation set 0: 14.402511
		best_validation_score so far: 14.402511
		Model 1/8, Metric mae, Validation set 1: 18.296759
		best_validation_score so far: 14.402511
		Model 2/8, Metric mae, Validation set 2: 18.351203
		best_validation_score so far: 14.402511
		Model 3/8, Metric mae, Validation set 3: 14.947294
		best_validation_score so far: 14.402511
		Model 4/8, Metric mae, Validation set 4: 14.215261
		best_validation_score so far: 14.215261
		Model 5/8, Metric mae, Validation set 5: 16.048053
		best_validation_score so far: 14.215261
		Model 6/8, Metric mae, Validation set 6: 16.323176
		best_validation_score so far: 14.215261
		Model 7/8, Metric mae, Validation set 7: 14.138251
		best_validation_score so far: 14.138251
		Best hyperparameters: [(u'n_estimators', 100), (u'data_shape', (23,)), (u'max_features', None)]
		train_score: 5.003712
		validation_score: 14.138251

		%% Cell type:markdown id: tags:

		Compute train/valid/test set mean absolute error for best RF hyperparameters

		%% Cell type:code id: tags:

		``` python
		rf_train_csv_out = "rf_train_regressor.csv"
		rf_train_stats_out = "rf_train_stats_regressor.txt"
		rf_train_evaluator = Evaluator(best_rf, train_dataset, output_transformers)
		rf_train_df, rf_train_mae = rf_train_evaluator.compute_model_performance(
		rf_train_csv_out, rf_train_stats_out)
		rf_train_mae = rf_train_mae.iloc[0]["mae"]
		print("RF Train set MAE %f" % (rf_train_mae))

		rf_valid_csv_out = "rf_valid_regressor.csv"
		rf_valid_stats_out = "rf_valid_stats_regressor.txt"
		rf_valid_evaluator = Evaluator(best_rf, valid_dataset, output_transformers)
		rf_valid_df, rf_valid_mae = rf_valid_evaluator.compute_model_performance(
		rf_valid_csv_out, rf_valid_stats_out)
		rf_valid_mae = rf_valid_mae.iloc[0]["mae"]
		print("RF Valid set MAE %f" % (rf_valid_mae))

		rf_test_csv_out = "rf_test_regressor.csv"
		rf_test_stats_out = "rf_test_stats_regressor.txt"
		rf_test_evaluator = Evaluator(best_rf, test_dataset, output_transformers)
		rf_test_df, rf_test_mae = rf_test_evaluator.compute_model_performance(
		rf_test_csv_out, rf_test_stats_out)
		rf_test_mae = rf_test_mae.iloc[0]["mae"]
		print("RF Test set MAE error %f" % (rf_test_mae))
		```

		%% Output

		RF Train set MAE 5.003712
		RF Valid set MAE 14.138251
		RF Test set MAE error 13.182236

		%% Cell type:markdown id: tags:

		Fit Kernal Ridge Regression with hyperparameter search

		%% Cell type:code id: tags:

		``` python
		def model_builder(task_types, params_dict, verbosity):
		kernel = params_dict["kernel"]
		alpha = params_dict["alpha"]
		gamma = params_dict["gamma"]
		return SklearnModel(
		task_types, params_dict,
		model_instance=KernelRidge(alpha=alpha,kernel=kernel,gamma=gamma))
		params_dict = {
		"kernel": ["rbf", "laplacian"],
		"alpha": [0.0001,0.001,0.01,0.1],
		"gamma": [0.0001,0.001,0.01,0.1]
		}
		optimizer = HyperparamOpt(model_builder, task_types)
		best_krr, best_krr_hyperparams, all_krr_results = optimizer.hyperparam_search(
		params_dict, train_dataset, valid_dataset, output_transformers, metric="mae", use_max=False)
		```

		%% Output

		Model 0/32, Metric mae, Validation set 0: 17.442589
		best_validation_score so far: 17.442589
		Model 1/32, Metric mae, Validation set 1: 20.738077
		best_validation_score so far: 17.442589
		Model 2/32, Metric mae, Validation set 2: 24.711658
		best_validation_score so far: 17.442589
		Model 3/32, Metric mae, Validation set 3: 28.887619
		best_validation_score so far: 17.442589
		Model 4/32, Metric mae, Validation set 4: 11.886870
		best_validation_score so far: 11.886870
		Model 5/32, Metric mae, Validation set 5: 13.795266
		best_validation_score so far: 11.886870
		Model 6/32, Metric mae, Validation set 6: 16.789905
		best_validation_score so far: 11.886870
		Model 7/32, Metric mae, Validation set 7: 20.518218
		best_validation_score so far: 11.886870
		Model 8/32, Metric mae, Validation set 8: 10.047228
		best_validation_score so far: 10.047228
		Model 9/32, Metric mae, Validation set 9: 10.179862
		best_validation_score so far: 10.047228
		Model 10/32, Metric mae, Validation set 10: 11.114865
		best_validation_score so far: 10.047228
		Model 11/32, Metric mae, Validation set 11: 13.676673
		best_validation_score so far: 10.047228
		Model 12/32, Metric mae, Validation set 12: 12.853101
		best_validation_score so far: 10.047228
		Model 13/32, Metric mae, Validation set 13: 11.392317
		best_validation_score so far: 10.047228
		Model 14/32, Metric mae, Validation set 14: 11.119340
		best_validation_score so far: 10.047228
		Model 15/32, Metric mae, Validation set 15: 12.525421
		best_validation_score so far: 10.047228
		Model 16/32, Metric mae, Validation set 16: 10.393173
		best_validation_score so far: 10.047228
		Model 17/32, Metric mae, Validation set 17: 10.541343
		best_validation_score so far: 10.047228
		Model 18/32, Metric mae, Validation set 18: 13.208075
		best_validation_score so far: 10.047228
		Model 19/32, Metric mae, Validation set 19: 22.621189
		best_validation_score so far: 10.047228
		Model 20/32, Metric mae, Validation set 20: 10.372002
		best_validation_score so far: 10.047228
		Model 21/32, Metric mae, Validation set 21: 10.068803
		best_validation_score so far: 10.047228
		Model 22/32, Metric mae, Validation set 22: 10.350311
		best_validation_score so far: 10.047228
		Model 23/32, Metric mae, Validation set 23: 13.182704
		best_validation_score so far: 10.047228
		Model 24/32, Metric mae, Validation set 24: 9.738578
		best_validation_score so far: 9.738578
		Model 25/32, Metric mae, Validation set 25: 9.414958
		best_validation_score so far: 9.414958
		Model 26/32, Metric mae, Validation set 26: 9.458250
		best_validation_score so far: 9.414958
		Model 27/32, Metric mae, Validation set 27: 10.039154
		best_validation_score so far: 9.414958
		Model 28/32, Metric mae, Validation set 28: 9.295678
		best_validation_score so far: 9.295678
		Model 29/32, Metric mae, Validation set 29: 9.296774
		best_validation_score so far: 9.295678
		Model 30/32, Metric mae, Validation set 30: 9.319664
		best_validation_score so far: 9.295678
		Model 31/32, Metric mae, Validation set 31: 9.702773
		best_validation_score so far: 9.295678
		Best hyperparameters: [(u'kernel', u'laplacian'), (u'gamma', 0.1), (u'alpha', 0.0001)]
		train_score: 0.011386
		validation_score: 9.295678

		%% Cell type:markdown id: tags:

		Compute train/valid/test set mean absolute error for best KRR hyperparameters

		%% Cell type:code id: tags:

		``` python
		krr_train_csv_out = "krr_train_regressor.csv"
		krr_train_stats_out = "krr_train_stats_regressor.txt"
		krr_train_evaluator = Evaluator(best_krr, train_dataset, output_transformers)
		krr_train_df, krr_train_mae = krr_train_evaluator.compute_model_performance(
		krr_train_csv_out, krr_train_stats_out)
		krr_train_mae = krr_train_mae.iloc[0]["mae"]
		print("KRR Train set MAE %f" % (krr_train_mae))

		krr_valid_csv_out = "krr_valid_regressor.csv"
		krr_valid_stats_out = "krr_valid_stats_regressor.txt"
		krr_valid_evaluator = Evaluator(best_krr, valid_dataset, output_transformers)
		krr_valid_df, krr_valid_mae = krr_valid_evaluator.compute_model_performance(
		krr_valid_csv_out, krr_valid_stats_out)
		krr_valid_mae = krr_valid_mae.iloc[0]["mae"]
		print("KRR Valid set MAE %f" % (krr_valid_mae))

		krr_test_csv_out = "krr_test_regressor.csv"
		krr_test_stats_out = "krr_test_stats_regressor.txt"
		krr_test_evaluator = Evaluator(best_krr, test_dataset, output_transformers)
		krr_test_df, krr_test_mae = krr_test_evaluator.compute_model_performance(
		krr_test_csv_out, krr_test_stats_out)
		krr_test_mae = krr_test_mae.iloc[0]["mae"]
		print("KRR Test set MAE error %f" % (krr_test_mae))
		```

		%% Output

		KRR Train set MAE 0.011386
		KRR Valid set MAE 9.295678
		KRR Test set MAE error 8.475528

		%% Cell type:markdown id: tags:

		Fit Single-task DNN with hyperparameter search

		%% Cell type:code id: tags:

		``` python
		np.random.seed()
		params_dict = {"activation": ["relu"],
		"momentum": [.9],
		"batch_size": [50],
		"init": ["glorot_uniform"],
		"data_shape": [train_dataset.get_data_shape()],
		"learning_rate": np.power(10., np.random.uniform(-5, -2, size=5)),
		"decay": np.power(10., np.random.uniform(-6, -4, size=5)),
		"nb_hidden": [1000],
		"nb_epoch": [50],
		"nesterov": [True],
		"dropout": [.1],
		"nb_layers": [1, 2],
		"batchnorm": [False],
		}

		optimizer = HyperparamOpt(SingleTaskDNN, task_types)
		best_dnn, best_hyperparams, all_results = optimizer.hyperparam_search(
		params_dict, train_dataset, valid_dataset, output_transformers, metric="mae",
		use_max=False, verbosity=None)
		```

		%% Output

		Model 0/50, Metric mae, Validation set 0: 16.893898
		best_validation_score so far: 16.893898
		Model 1/50, Metric mae, Validation set 1: 23.639633
		best_validation_score so far: 16.893898
		Model 2/50, Metric mae, Validation set 2: 16.960870
		best_validation_score so far: 16.893898
		Model 3/50, Metric mae, Validation set 3: 18.988264
		best_validation_score so far: 16.893898
		Model 4/50, Metric mae, Validation set 4: nan
		best_validation_score so far: 16.893898
		Model 5/50, Metric mae, Validation set 5: 16.960815
		best_validation_score so far: 16.893898
		Model 6/50, Metric mae, Validation set 6: 24.852288
		best_validation_score so far: 16.893898
		Model 7/50, Metric mae, Validation set 7: 16.984414
		best_validation_score so far: 16.893898
		Model 8/50, Metric mae, Validation set 8: 19.151421
		best_validation_score so far: 16.893898
		Model 9/50, Metric mae, Validation set 9: nan
		best_validation_score so far: 16.893898
		Model 10/50, Metric mae, Validation set 10: 16.896120
		best_validation_score so far: 16.893898
		Model 11/50, Metric mae, Validation set 11: 24.065200
		best_validation_score so far: 16.893898
		Model 12/50, Metric mae, Validation set 12: 16.554714
		best_validation_score so far: 16.554714
		Model 13/50, Metric mae, Validation set 13: 18.615458
		best_validation_score so far: 16.554714
		Model 14/50, Metric mae, Validation set 14: nan
		best_validation_score so far: 16.554714
		Model 15/50, Metric mae, Validation set 15: 16.691008
		best_validation_score so far: 16.554714
		Model 16/50, Metric mae, Validation set 16: 26.785887
		best_validation_score so far: 16.554714
		Model 17/50, Metric mae, Validation set 17: 16.968494
		best_validation_score so far: 16.554714
		Model 18/50, Metric mae, Validation set 18: 19.024821
		best_validation_score so far: 16.554714
		Model 19/50, Metric mae, Validation set 19: nan
		best_validation_score so far: 16.554714
		Model 20/50, Metric mae, Validation set 20: 16.655503
		best_validation_score so far: 16.554714
		Model 21/50, Metric mae, Validation set 21: 24.454552
		best_validation_score so far: 16.554714
		Model 22/50, Metric mae, Validation set 22: 16.901522
		best_validation_score so far: 16.554714
		Model 23/50, Metric mae, Validation set 23: 18.608702
		best_validation_score so far: 16.554714
		Model 24/50, Metric mae, Validation set 24: nan
		best_validation_score so far: 16.554714
		Model 25/50, Metric mae, Validation set 25: 16.109592
		best_validation_score so far: 16.109592
		Model 26/50, Metric mae, Validation set 26: 24.653540
		best_validation_score so far: 16.109592
		Model 27/50, Metric mae, Validation set 27: 16.385990
		best_validation_score so far: 16.109592
		Model 28/50, Metric mae, Validation set 28: 17.936302
		best_validation_score so far: 16.109592
		Model 29/50, Metric mae, Validation set 29: 19.840381
		best_validation_score so far: 16.109592
		Model 30/50, Metric mae, Validation set 30: 15.977344
		best_validation_score so far: 15.977344
		Model 31/50, Metric mae, Validation set 31: 24.147172
		best_validation_score so far: 15.977344
		Model 32/50, Metric mae, Validation set 32: 16.176972
		best_validation_score so far: 15.977344
		Model 33/50, Metric mae, Validation set 33: 18.216468
		best_validation_score so far: 15.977344
		Model 34/50, Metric mae, Validation set 34: 18.203847
		best_validation_score so far: 15.977344
		Model 35/50, Metric mae, Validation set 35: 16.190024
		best_validation_score so far: 15.977344
		Model 36/50, Metric mae, Validation set 36: 23.095421
		best_validation_score so far: 15.977344
		Model 37/50, Metric mae, Validation set 37: 16.383400
		best_validation_score so far: 15.977344
		Model 38/50, Metric mae, Validation set 38: 18.372743
		best_validation_score so far: 15.977344
		Model 39/50, Metric mae, Validation set 39: nan
		best_validation_score so far: 15.977344
		Model 40/50, Metric mae, Validation set 40: 16.185664
		best_validation_score so far: 15.977344
		Model 41/50, Metric mae, Validation set 41: 25.150945
		best_validation_score so far: 15.977344
		Model 42/50, Metric mae, Validation set 42: 16.227827
		best_validation_score so far: 15.977344
		Model 43/50, Metric mae, Validation set 43: 18.775972
		best_validation_score so far: 15.977344
		Model 44/50, Metric mae, Validation set 44: 20.465551
		best_validation_score so far: 15.977344
		Model 45/50, Metric mae, Validation set 45: 15.957421
		best_validation_score so far: 15.957421
		Model 46/50, Metric mae, Validation set 46: 23.816754
		best_validation_score so far: 15.957421
		Model 47/50, Metric mae, Validation set 47: 16.067526
		best_validation_score so far: 15.957421
		Model 48/50, Metric mae, Validation set 48: 18.219173
		best_validation_score so far: 15.957421
		Model 49/50, Metric mae, Validation set 49: 32.712455
		best_validation_score so far: 15.957421
		Best hyperparameters: [(u'data_shape', (23,)), (u'activation', u'relu'), (u'batch_size', 50), (u'nb_layers', 2), (u'decay', 2.1082531264299641e-05), (u'nesterov', True), (u'nb_epoch', 50), (u'learning_rate', 0.0010809218725085883), (u'init', u'glorot_uniform'), (u'nb_hidden', 1000), (u'batchnorm', False), (u'dropout', 0.1), (u'momentum', 0.9)]
		train_score: 15.519322
		validation_score: 15.957421

		%% Cell type:markdown id: tags:

		Compute train/valid/test set mean absolute error for best DNN hyperparameters

		%% Cell type:code id: tags:

		``` python
		dnn_train_csv_out = "dnn_train_regressor.csv"
		dnn_train_stats_out = "dnn_train_regressor_stats.txt"
		dnn_train_evaluator = Evaluator(best_dnn, train_dataset, output_transformers)
		dnn_train_df, dnn_train_mae = dnn_train_evaluator.compute_model_performance(
		dnn_train_csv_out, dnn_train_stats_out)
		dnn_train_mae = dnn_train_mae.iloc[0]["mae"]
		print("DNN Train set MAE error %f" % (dnn_train_mae))

		dnn_valid_csv_out = "dnn_valid_regressor.csv"
		dnn_valid_stats_out = "dnn_valid_regressor_stats.txt"
		dnn_valid_evaluator = Evaluator(best_dnn, valid_dataset, output_transformers)
		dnn_valid_df, dnn_valid_mae = dnn_valid_evaluator.compute_model_performance(
		dnn_valid_csv_out, dnn_valid_stats_out)
		dnn_valid_mae = dnn_valid_mae.iloc[0]["mae"]
		print("DNN Valid set MAE error %f" % (dnn_valid_mae))

		dnn_test_csv_out = "dnn_test_regressor.csv"
		dnn_test_stats_out = "dnn_test_regressor_stats.txt"
		dnn_test_evaluator = Evaluator(best_dnn, test_dataset, output_transformers)
		dnn_test_df, dnn_test_mae = dnn_test_evaluator.compute_model_performance(
		dnn_test_csv_out, dnn_test_stats_out)
		dnn_test_mae = dnn_test_mae.iloc[0]["mae"]
		print("DNN Test set MAE error %f" % (dnn_test_mae))
		```

		%% Output

		DNN Train set MAE error 15.519321
		DNN Valid set MAE error 15.957421
		DNN Test set MAE error 16.124620

		%% Cell type:code id: tags:

		``` python
		```

Admin message