Loading deepchem/hyper/tests/test_hyperparam_opt.py +34 −35 Original line number Diff line number Diff line Loading @@ -15,7 +15,6 @@ import tempfile import shutil import numpy as np import tensorflow as tf from keras import backend as K import deepchem as dc from sklearn.ensemble import RandomForestClassifier from sklearn.ensemble import RandomForestRegressor Loading Loading @@ -100,40 +99,40 @@ class TestHyperparamOptAPI(unittest.TestCase): params_dict, train_dataset, valid_dataset, transformers, classification_metric, logdir=None) def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Keras multitask deepchem classification API.""" task_type = "classification" current_dir = os.path.dirname(os.path.abspath(__file__)) input_file = os.path.join( current_dir, "../../models/tests/multitask_example.csv") tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] n_features = 1024 featurizer = dc.feat.CircularFingerprint(size=n_features) loader = dc.load.DataLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer, verbosity="low") dataset = loader.featurize(input_file) splitter = dc.splits.ScaffoldSplitter() train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset) transformers = [] metric = dc.metrics.Metric( dc.metrics.matthews_corrcoef, np.mean, mode="classification") params_dict= {"n_hidden": [5, 10]} def model_builder(model_params, model_dir): keras_model = dc.models.MultiTaskDNN( len(tasks), n_features, task_type, dropout=0., **model_params) return dc.models.KerasModel(keras_model, model_dir) optimizer = dc.hyper.HyperparamOpt(model_builder) best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, transformers, metric, logdir=None) #def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self): # """Straightforward test of Keras multitask deepchem classification API.""" # task_type = "classification" # current_dir = os.path.dirname(os.path.abspath(__file__)) # input_file = os.path.join( # current_dir, "../../models/tests/multitask_example.csv") # tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", # "task7", "task8", "task9", "task10", "task11", "task12", # "task13", "task14", "task15", "task16"] # n_features = 1024 # featurizer = dc.feat.CircularFingerprint(size=n_features) # loader = dc.load.DataLoader( # tasks=tasks, smiles_field="smiles", # featurizer=featurizer, verbosity="low") # dataset = loader.featurize(input_file) # splitter = dc.splits.ScaffoldSplitter() # train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( # dataset) # transformers = [] # metric = dc.metrics.Metric( # dc.metrics.matthews_corrcoef, np.mean, mode="classification") # params_dict= {"n_hidden": [5, 10]} # # def model_builder(model_params, model_dir): # keras_model = dc.models.MultiTaskDNN( # len(tasks), n_features, task_type, dropout=0., **model_params) # return dc.models.KerasModel(keras_model, model_dir) # optimizer = dc.hyper.HyperparamOpt(model_builder) # best_model, best_hyperparams, all_results = optimizer.hyperparam_search( # params_dict, train_dataset, valid_dataset, transformers, # metric, logdir=None) def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Tensorflow multitask deepchem classification API.""" Loading deepchem/models/tests/test_overfit.py +172 −172 Original line number Diff line number Diff line Loading @@ -116,40 +116,40 @@ class TestOverfit(test_util.TensorFlowTestCase): scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 def test_keras_regression_overfit(self): """Test that keras models can overfit simple regression datasets.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_samples = 10 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.rand(n_samples, n_tasks) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" regression_metric = dc.metrics.Metric( dc.metrics.r2_score, verbosity=verbosity) keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "regression", dropout=0., learning_rate=.15, decay=1e-4) model = dc.models.KerasModel(keras_model) # Fit trained model model.fit(dataset, nb_epoch=200) model.save() # Eval model on train scores = model.evaluate(dataset, [regression_metric]) assert scores[regression_metric.name] > .7 #def test_keras_regression_overfit(self): # """Test that keras models can overfit simple regression datasets.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_samples = 10 # n_features = 3 # n_tasks = 1 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.rand(n_samples, n_tasks) # w = np.ones((n_samples, n_tasks)) # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # regression_metric = dc.metrics.Metric( # dc.metrics.r2_score, verbosity=verbosity) # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "regression", # dropout=0., learning_rate=.15, decay=1e-4) # model = dc.models.KerasModel(keras_model) # # Fit trained model # model.fit(dataset, nb_epoch=200) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [regression_metric]) # assert scores[regression_metric.name] > .7 def test_tf_regression_overfit(self): """Test that TensorFlow models can overfit simple regression datasets.""" Loading Loading @@ -182,76 +182,76 @@ class TestOverfit(test_util.TensorFlowTestCase): scores = model.evaluate(dataset, [regression_metric]) assert scores[regression_metric.name] < .1 def test_keras_classification_overfit(self): """Test that keras models can overfit simple classification datasets.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_samples = 10 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = dc.metrics.Metric( dc.metrics.roc_auc_score, verbosity=verbosity) keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", learning_rate=.15, decay=1e-4, dropout=0.) model = dc.models.KerasModel(keras_model) # Fit trained model model.fit(dataset, nb_epoch=200) model.save() # Eval model on train scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 def test_keras_skewed_classification_overfit(self): """Test keras models can overfit 0/1 datasets with few actives.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_samples = 100 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) p = .05 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.binomial(1, p, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = dc.metrics.Metric( dc.metrics.roc_auc_score, verbosity=verbosity) keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", dropout=0., learning_rate=.15, decay=1e-4) model = dc.models.KerasModel(keras_model) # Fit trained model model.fit(dataset, batch_size=n_samples, nb_epoch=200) model.save() # Eval model on train scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 #def test_keras_classification_overfit(self): # """Test that keras models can overfit simple classification datasets.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_samples = 10 # n_features = 3 # n_tasks = 1 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.randint(2, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # classification_metric = dc.metrics.Metric( # dc.metrics.roc_auc_score, verbosity=verbosity) # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", # learning_rate=.15, decay=1e-4, dropout=0.) # model = dc.models.KerasModel(keras_model) # # Fit trained model # model.fit(dataset, nb_epoch=200) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [classification_metric]) # assert scores[classification_metric.name] > .9 #def test_keras_skewed_classification_overfit(self): # """Test keras models can overfit 0/1 datasets with few actives.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_samples = 100 # n_features = 3 # n_tasks = 1 # # # Generate dummy dataset # np.random.seed(123) # p = .05 # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.binomial(1, p, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # classification_metric = dc.metrics.Metric( # dc.metrics.roc_auc_score, verbosity=verbosity) # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", # dropout=0., learning_rate=.15, decay=1e-4) # model = dc.models.KerasModel(keras_model) # # Fit trained model # model.fit(dataset, batch_size=n_samples, nb_epoch=200) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [classification_metric]) # assert scores[classification_metric.name] > .9 def test_tf_classification_overfit(self): """Test that tensorflow models can overfit simple classification datasets.""" Loading Loading @@ -392,40 +392,40 @@ class TestOverfit(test_util.TensorFlowTestCase): scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 def test_keras_multitask_classification_overfit(self): """Test keras multitask overfits tiny data.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_tasks = 10 n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = dc.metrics.Metric( dc.metrics.roc_auc_score, verbosity=verbosity, task_averager=np.mean, mode="classification") keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", dropout=0., learning_rate=.15, decay=1e-4) model = dc.models.KerasModel(keras_model, verbosity=verbosity) # Fit trained model model.fit(dataset, nb_epoch=50) model.save() # Eval model on train scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 #def test_keras_multitask_classification_overfit(self): # """Test keras multitask overfits tiny data.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_tasks = 10 # n_samples = 10 # n_features = 3 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.randint(2, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # classification_metric = dc.metrics.Metric( # dc.metrics.roc_auc_score, verbosity=verbosity, # task_averager=np.mean, mode="classification") # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", dropout=0., learning_rate=.15, # decay=1e-4) # model = dc.models.KerasModel(keras_model, verbosity=verbosity) # # Fit trained model # model.fit(dataset, nb_epoch=50) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [classification_metric]) # assert scores[classification_metric.name] > .9 def test_tf_multitask_classification_overfit(self): """Test tf multitask overfits tiny data.""" Loading Loading @@ -551,40 +551,40 @@ class TestOverfit(test_util.TensorFlowTestCase): scores = model.evaluate(dataset, [regression_metric]) assert scores[regression_metric.name] > .7 def test_keras_multitask_regression_overfit(self): """Test keras multitask overfits tiny data.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_tasks = 10 n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" regression_metric = dc.metrics.Metric( dc.metrics.r2_score, verbosity=verbosity, task_averager=np.mean, mode="regression") keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "regression", dropout=0., learning_rate=.1, decay=1e-4) model = dc.models.KerasModel(keras_model, verbosity=verbosity) # Fit trained model model.fit(dataset, nb_epoch=100) model.save() # Eval model on train scores = model.evaluate(dataset, [regression_metric]) assert scores[regression_metric.name] > .75 #def test_keras_multitask_regression_overfit(self): # """Test keras multitask overfits tiny data.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_tasks = 10 # n_samples = 10 # n_features = 3 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.randint(2, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # regression_metric = dc.metrics.Metric( # dc.metrics.r2_score, verbosity=verbosity, task_averager=np.mean, # mode="regression") # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "regression", dropout=0., learning_rate=.1, # decay=1e-4) # model = dc.models.KerasModel(keras_model, verbosity=verbosity) # # Fit trained model # model.fit(dataset, nb_epoch=100) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [regression_metric]) # assert scores[regression_metric.name] > .75 def test_tf_multitask_regression_overfit(self): """Test tf multitask overfits tiny data.""" Loading deepchem/models/tests/test_reload.py +39 −39 Original line number Diff line number Diff line Loading @@ -51,45 +51,45 @@ class TestReload(unittest.TestCase): scores = reloaded_model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 def test_keras_reload(self): """Test that trained keras models can be reloaded correctly.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_samples = 10 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score) keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", dropout=0.) model_dir = tempfile.mkdtemp() model = dc.models.KerasModel(keras_model, model_dir) # Fit trained model model.fit(dataset) model.save() # Load trained model reloaded_keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", dropout=0.) reloaded_model = dc.models.KerasModel(reloaded_keras_model, model_dir) reloaded_model.reload( custom_objects={"MultiTaskDNN": dc.models.MultiTaskDNN}) # Eval model on train scores = reloaded_model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .6 #def test_keras_reload(self): # """Test that trained keras models can be reloaded correctly.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_samples = 10 # n_features = 3 # n_tasks = 1 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.randint(2, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # # dataset = dc.data.NumpyDataset(X, y, w, ids) # classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score) # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", dropout=0.) # model_dir = tempfile.mkdtemp() # model = dc.models.KerasModel(keras_model, model_dir) # # Fit trained model # model.fit(dataset) # model.save() # # Load trained model # reloaded_keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", dropout=0.) # reloaded_model = dc.models.KerasModel(reloaded_keras_model, model_dir) # reloaded_model.reload( # custom_objects={"MultiTaskDNN": dc.models.MultiTaskDNN}) # # Eval model on train # scores = reloaded_model.evaluate(dataset, [classification_metric]) # assert scores[classification_metric.name] > .6 def test_tf_reload(self): """Test that tensorflow models can overfit simple classification datasets.""" Loading Loading
deepchem/hyper/tests/test_hyperparam_opt.py +34 −35 Original line number Diff line number Diff line Loading @@ -15,7 +15,6 @@ import tempfile import shutil import numpy as np import tensorflow as tf from keras import backend as K import deepchem as dc from sklearn.ensemble import RandomForestClassifier from sklearn.ensemble import RandomForestRegressor Loading Loading @@ -100,40 +99,40 @@ class TestHyperparamOptAPI(unittest.TestCase): params_dict, train_dataset, valid_dataset, transformers, classification_metric, logdir=None) def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Keras multitask deepchem classification API.""" task_type = "classification" current_dir = os.path.dirname(os.path.abspath(__file__)) input_file = os.path.join( current_dir, "../../models/tests/multitask_example.csv") tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] n_features = 1024 featurizer = dc.feat.CircularFingerprint(size=n_features) loader = dc.load.DataLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer, verbosity="low") dataset = loader.featurize(input_file) splitter = dc.splits.ScaffoldSplitter() train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset) transformers = [] metric = dc.metrics.Metric( dc.metrics.matthews_corrcoef, np.mean, mode="classification") params_dict= {"n_hidden": [5, 10]} def model_builder(model_params, model_dir): keras_model = dc.models.MultiTaskDNN( len(tasks), n_features, task_type, dropout=0., **model_params) return dc.models.KerasModel(keras_model, model_dir) optimizer = dc.hyper.HyperparamOpt(model_builder) best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, transformers, metric, logdir=None) #def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self): # """Straightforward test of Keras multitask deepchem classification API.""" # task_type = "classification" # current_dir = os.path.dirname(os.path.abspath(__file__)) # input_file = os.path.join( # current_dir, "../../models/tests/multitask_example.csv") # tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", # "task7", "task8", "task9", "task10", "task11", "task12", # "task13", "task14", "task15", "task16"] # n_features = 1024 # featurizer = dc.feat.CircularFingerprint(size=n_features) # loader = dc.load.DataLoader( # tasks=tasks, smiles_field="smiles", # featurizer=featurizer, verbosity="low") # dataset = loader.featurize(input_file) # splitter = dc.splits.ScaffoldSplitter() # train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( # dataset) # transformers = [] # metric = dc.metrics.Metric( # dc.metrics.matthews_corrcoef, np.mean, mode="classification") # params_dict= {"n_hidden": [5, 10]} # # def model_builder(model_params, model_dir): # keras_model = dc.models.MultiTaskDNN( # len(tasks), n_features, task_type, dropout=0., **model_params) # return dc.models.KerasModel(keras_model, model_dir) # optimizer = dc.hyper.HyperparamOpt(model_builder) # best_model, best_hyperparams, all_results = optimizer.hyperparam_search( # params_dict, train_dataset, valid_dataset, transformers, # metric, logdir=None) def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Tensorflow multitask deepchem classification API.""" Loading
deepchem/models/tests/test_overfit.py +172 −172 Original line number Diff line number Diff line Loading @@ -116,40 +116,40 @@ class TestOverfit(test_util.TensorFlowTestCase): scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 def test_keras_regression_overfit(self): """Test that keras models can overfit simple regression datasets.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_samples = 10 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.rand(n_samples, n_tasks) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" regression_metric = dc.metrics.Metric( dc.metrics.r2_score, verbosity=verbosity) keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "regression", dropout=0., learning_rate=.15, decay=1e-4) model = dc.models.KerasModel(keras_model) # Fit trained model model.fit(dataset, nb_epoch=200) model.save() # Eval model on train scores = model.evaluate(dataset, [regression_metric]) assert scores[regression_metric.name] > .7 #def test_keras_regression_overfit(self): # """Test that keras models can overfit simple regression datasets.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_samples = 10 # n_features = 3 # n_tasks = 1 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.rand(n_samples, n_tasks) # w = np.ones((n_samples, n_tasks)) # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # regression_metric = dc.metrics.Metric( # dc.metrics.r2_score, verbosity=verbosity) # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "regression", # dropout=0., learning_rate=.15, decay=1e-4) # model = dc.models.KerasModel(keras_model) # # Fit trained model # model.fit(dataset, nb_epoch=200) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [regression_metric]) # assert scores[regression_metric.name] > .7 def test_tf_regression_overfit(self): """Test that TensorFlow models can overfit simple regression datasets.""" Loading Loading @@ -182,76 +182,76 @@ class TestOverfit(test_util.TensorFlowTestCase): scores = model.evaluate(dataset, [regression_metric]) assert scores[regression_metric.name] < .1 def test_keras_classification_overfit(self): """Test that keras models can overfit simple classification datasets.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_samples = 10 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = dc.metrics.Metric( dc.metrics.roc_auc_score, verbosity=verbosity) keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", learning_rate=.15, decay=1e-4, dropout=0.) model = dc.models.KerasModel(keras_model) # Fit trained model model.fit(dataset, nb_epoch=200) model.save() # Eval model on train scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 def test_keras_skewed_classification_overfit(self): """Test keras models can overfit 0/1 datasets with few actives.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_samples = 100 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) p = .05 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.binomial(1, p, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = dc.metrics.Metric( dc.metrics.roc_auc_score, verbosity=verbosity) keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", dropout=0., learning_rate=.15, decay=1e-4) model = dc.models.KerasModel(keras_model) # Fit trained model model.fit(dataset, batch_size=n_samples, nb_epoch=200) model.save() # Eval model on train scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 #def test_keras_classification_overfit(self): # """Test that keras models can overfit simple classification datasets.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_samples = 10 # n_features = 3 # n_tasks = 1 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.randint(2, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # classification_metric = dc.metrics.Metric( # dc.metrics.roc_auc_score, verbosity=verbosity) # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", # learning_rate=.15, decay=1e-4, dropout=0.) # model = dc.models.KerasModel(keras_model) # # Fit trained model # model.fit(dataset, nb_epoch=200) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [classification_metric]) # assert scores[classification_metric.name] > .9 #def test_keras_skewed_classification_overfit(self): # """Test keras models can overfit 0/1 datasets with few actives.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_samples = 100 # n_features = 3 # n_tasks = 1 # # # Generate dummy dataset # np.random.seed(123) # p = .05 # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.binomial(1, p, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # classification_metric = dc.metrics.Metric( # dc.metrics.roc_auc_score, verbosity=verbosity) # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", # dropout=0., learning_rate=.15, decay=1e-4) # model = dc.models.KerasModel(keras_model) # # Fit trained model # model.fit(dataset, batch_size=n_samples, nb_epoch=200) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [classification_metric]) # assert scores[classification_metric.name] > .9 def test_tf_classification_overfit(self): """Test that tensorflow models can overfit simple classification datasets.""" Loading Loading @@ -392,40 +392,40 @@ class TestOverfit(test_util.TensorFlowTestCase): scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 def test_keras_multitask_classification_overfit(self): """Test keras multitask overfits tiny data.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_tasks = 10 n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = dc.metrics.Metric( dc.metrics.roc_auc_score, verbosity=verbosity, task_averager=np.mean, mode="classification") keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", dropout=0., learning_rate=.15, decay=1e-4) model = dc.models.KerasModel(keras_model, verbosity=verbosity) # Fit trained model model.fit(dataset, nb_epoch=50) model.save() # Eval model on train scores = model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 #def test_keras_multitask_classification_overfit(self): # """Test keras multitask overfits tiny data.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_tasks = 10 # n_samples = 10 # n_features = 3 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.randint(2, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # classification_metric = dc.metrics.Metric( # dc.metrics.roc_auc_score, verbosity=verbosity, # task_averager=np.mean, mode="classification") # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", dropout=0., learning_rate=.15, # decay=1e-4) # model = dc.models.KerasModel(keras_model, verbosity=verbosity) # # Fit trained model # model.fit(dataset, nb_epoch=50) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [classification_metric]) # assert scores[classification_metric.name] > .9 def test_tf_multitask_classification_overfit(self): """Test tf multitask overfits tiny data.""" Loading Loading @@ -551,40 +551,40 @@ class TestOverfit(test_util.TensorFlowTestCase): scores = model.evaluate(dataset, [regression_metric]) assert scores[regression_metric.name] > .7 def test_keras_multitask_regression_overfit(self): """Test keras multitask overfits tiny data.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_tasks = 10 n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) verbosity = "high" regression_metric = dc.metrics.Metric( dc.metrics.r2_score, verbosity=verbosity, task_averager=np.mean, mode="regression") keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "regression", dropout=0., learning_rate=.1, decay=1e-4) model = dc.models.KerasModel(keras_model, verbosity=verbosity) # Fit trained model model.fit(dataset, nb_epoch=100) model.save() # Eval model on train scores = model.evaluate(dataset, [regression_metric]) assert scores[regression_metric.name] > .75 #def test_keras_multitask_regression_overfit(self): # """Test keras multitask overfits tiny data.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_tasks = 10 # n_samples = 10 # n_features = 3 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.randint(2, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # dataset = dc.data.NumpyDataset(X, y, w, ids) # verbosity = "high" # regression_metric = dc.metrics.Metric( # dc.metrics.r2_score, verbosity=verbosity, task_averager=np.mean, # mode="regression") # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "regression", dropout=0., learning_rate=.1, # decay=1e-4) # model = dc.models.KerasModel(keras_model, verbosity=verbosity) # # Fit trained model # model.fit(dataset, nb_epoch=100) # model.save() # # Eval model on train # scores = model.evaluate(dataset, [regression_metric]) # assert scores[regression_metric.name] > .75 def test_tf_multitask_regression_overfit(self): """Test tf multitask overfits tiny data.""" Loading
deepchem/models/tests/test_reload.py +39 −39 Original line number Diff line number Diff line Loading @@ -51,45 +51,45 @@ class TestReload(unittest.TestCase): scores = reloaded_model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .9 def test_keras_reload(self): """Test that trained keras models can be reloaded correctly.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_samples = 10 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score) keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", dropout=0.) model_dir = tempfile.mkdtemp() model = dc.models.KerasModel(keras_model, model_dir) # Fit trained model model.fit(dataset) model.save() # Load trained model reloaded_keras_model = dc.models.MultiTaskDNN( n_tasks, n_features, "classification", dropout=0.) reloaded_model = dc.models.KerasModel(reloaded_keras_model, model_dir) reloaded_model.reload( custom_objects={"MultiTaskDNN": dc.models.MultiTaskDNN}) # Eval model on train scores = reloaded_model.evaluate(dataset, [classification_metric]) assert scores[classification_metric.name] > .6 #def test_keras_reload(self): # """Test that trained keras models can be reloaded correctly.""" # g = tf.Graph() # sess = tf.Session(graph=g) # K.set_session(sess) # with g.as_default(): # n_samples = 10 # n_features = 3 # n_tasks = 1 # # # Generate dummy dataset # np.random.seed(123) # ids = np.arange(n_samples) # X = np.random.rand(n_samples, n_features) # y = np.random.randint(2, size=(n_samples, n_tasks)) # w = np.ones((n_samples, n_tasks)) # # dataset = dc.data.NumpyDataset(X, y, w, ids) # classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score) # keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", dropout=0.) # model_dir = tempfile.mkdtemp() # model = dc.models.KerasModel(keras_model, model_dir) # # Fit trained model # model.fit(dataset) # model.save() # # Load trained model # reloaded_keras_model = dc.models.MultiTaskDNN( # n_tasks, n_features, "classification", dropout=0.) # reloaded_model = dc.models.KerasModel(reloaded_keras_model, model_dir) # reloaded_model.reload( # custom_objects={"MultiTaskDNN": dc.models.MultiTaskDNN}) # # Eval model on train # scores = reloaded_model.evaluate(dataset, [classification_metric]) # assert scores[classification_metric.name] > .6 def test_tf_reload(self): """Test that tensorflow models can overfit simple classification datasets.""" Loading
