Commit 3389acea authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Removing commented out keras tests

parent ed78f8e7
Loading
Loading
Loading
Loading
+0 −35
Original line number Diff line number Diff line
@@ -99,41 +99,6 @@ class TestHyperparamOptAPI(unittest.TestCase):
      params_dict, train_dataset, valid_dataset, transformers,
      classification_metric, logdir=None)

  #def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self):
  #  """Straightforward test of Keras multitask deepchem classification API."""
  #  task_type = "classification"
  #  current_dir = os.path.dirname(os.path.abspath(__file__))
  #  input_file = os.path.join(
  #      current_dir, "../../models/tests/multitask_example.csv")
  #  tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",
  #           "task7", "task8", "task9", "task10", "task11", "task12",
  #           "task13", "task14", "task15", "task16"]

  #  n_features = 1024
  #  featurizer = dc.feat.CircularFingerprint(size=n_features)
  #  loader = dc.load.DataLoader(
  #      tasks=tasks, smiles_field="smiles",
  #      featurizer=featurizer, verbosity="low")
  #  dataset = loader.featurize(input_file)

  #  splitter = dc.splits.ScaffoldSplitter()
  #  train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(
  #      dataset)

  #  transformers = []
  #  metric = dc.metrics.Metric(
  #      dc.metrics.matthews_corrcoef, np.mean, mode="classification")
  #  params_dict= {"n_hidden": [5, 10]}
  #    
  #  def model_builder(model_params, model_dir):
  #    keras_model = dc.models.MultiTaskDNN(
  #        len(tasks), n_features, task_type, dropout=0., **model_params)
  #    return dc.models.KerasModel(keras_model, model_dir)
  #  optimizer = dc.hyper.HyperparamOpt(model_builder)
  #  best_model, best_hyperparams, all_results = optimizer.hyperparam_search(
  #    params_dict, train_dataset, valid_dataset, transformers,
  #    metric, logdir=None)

  def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self):
    """Straightforward test of Tensorflow multitask deepchem classification API."""
    task_type = "classification"
+0 −40
Original line number Diff line number Diff line
@@ -133,46 +133,6 @@ class TestAPI(unittest.TestCase):
    _ = model.evaluate(train_dataset, regression_metrics, transformers)
    _ = model.evaluate(test_dataset, regression_metrics, transformers)

  #def test_multitask_keras_mlp_ECFP_classification_API(self):
  #  """Test of Keras multitask deepchem classification API."""
  #  g = tf.Graph()
  #  sess = tf.Session(graph=g)
  #  K.set_session(sess)
  #  with g.as_default():
  #    task_type = "classification"
  #    current_dir = os.path.dirname(os.path.abspath(__file__))
  #    input_file = os.path.join(current_dir, "multitask_example.csv")
  #    tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",
  #             "task7", "task8", "task9", "task10", "task11", "task12",
  #             "task13", "task14", "task15", "task16"]

  #    n_features = 1024
  #    featurizer = dc.feat.CircularFingerprint(size=n_features)
  #    loader = dc.load.DataLoader(
  #        tasks=tasks, smiles_field="smiles",
  #        featurizer=featurizer, verbosity="low")
  #    dataset = loader.featurize(input_file)

  #    splitter = dc.splits.ScaffoldSplitter()
  #    train_dataset, test_dataset = splitter.train_test_split(dataset)

  #    metrics = [dc.metrics.Metric(dc.metrics.roc_auc_score),
  #               dc.metrics.Metric(dc.metrics.matthews_corrcoef),
  #               dc.metrics.Metric(dc.metrics.recall_score),
  #               dc.metrics.Metric(dc.metrics.accuracy_score)]
  #    
  #    keras_model = dc.models.MultiTaskDNN(
  #        len(tasks), n_features, "classification", dropout=0.)
  #    model = dc.models.KerasModel(keras_model)

  #    # Fit trained model
  #    model.fit(train_dataset)
  #    model.save()

  #    # Eval model on train/test
  #    _ = model.evaluate(train_dataset, metrics)
  #    _ = model.evaluate(test_dataset, metrics)

  def test_singletask_tf_mlp_ECFP_classification_API(self):
    """Test of Tensorflow singletask deepchem classification API."""
    n_features = 1024
+0 −176
Original line number Diff line number Diff line
@@ -116,41 +116,6 @@ class TestOverfit(test_util.TensorFlowTestCase):
    scores = model.evaluate(dataset, [classification_metric])
    assert scores[classification_metric.name] > .9

  #def test_keras_regression_overfit(self):
  #  """Test that keras models can overfit simple regression datasets."""
  #  g = tf.Graph()
  #  sess = tf.Session(graph=g)
  #  K.set_session(sess)
  #  with g.as_default():
  #    n_samples = 10
  #    n_features = 3
  #    n_tasks = 1 
  #    
  #    # Generate dummy dataset
  #    np.random.seed(123)
  #    ids = np.arange(n_samples)
  #    X = np.random.rand(n_samples, n_features)
  #    y = np.random.rand(n_samples, n_tasks)
  #    w = np.ones((n_samples, n_tasks))

  #    dataset = dc.data.NumpyDataset(X, y, w, ids)

  #    verbosity = "high"
  #    regression_metric = dc.metrics.Metric(
  #        dc.metrics.r2_score, verbosity=verbosity)
  #    keras_model = dc.models.MultiTaskDNN(
  #        n_tasks, n_features, "regression",
  #        dropout=0., learning_rate=.15, decay=1e-4)
  #    model = dc.models.KerasModel(keras_model)

  #    # Fit trained model
  #    model.fit(dataset, nb_epoch=200)
  #    model.save()

  #    # Eval model on train
  #    scores = model.evaluate(dataset, [regression_metric])
  #    assert scores[regression_metric.name] > .7

  def test_tf_regression_overfit(self):
    """Test that TensorFlow models can overfit simple regression datasets."""
    n_samples = 10
@@ -182,77 +147,6 @@ class TestOverfit(test_util.TensorFlowTestCase):
    scores = model.evaluate(dataset, [regression_metric])
    assert scores[regression_metric.name] < .1

  #def test_keras_classification_overfit(self):
  #  """Test that keras models can overfit simple classification datasets."""
  #  g = tf.Graph()
  #  sess = tf.Session(graph=g)
  #  K.set_session(sess)
  #  with g.as_default():
  #    n_samples = 10
  #    n_features = 3
  #    n_tasks = 1
  #    
  #    # Generate dummy dataset
  #    np.random.seed(123)
  #    ids = np.arange(n_samples)
  #    X = np.random.rand(n_samples, n_features)
  #    y = np.random.randint(2, size=(n_samples, n_tasks))
  #    w = np.ones((n_samples, n_tasks))
  #  
  #    dataset = dc.data.NumpyDataset(X, y, w, ids)

  #    verbosity = "high"
  #    classification_metric = dc.metrics.Metric(
  #        dc.metrics.roc_auc_score, verbosity=verbosity)
  #    keras_model = dc.models.MultiTaskDNN(
  #        n_tasks, n_features, "classification",
  #        learning_rate=.15, decay=1e-4, dropout=0.)
  #    model = dc.models.KerasModel(keras_model)

  #    # Fit trained model
  #    model.fit(dataset, nb_epoch=200)
  #    model.save()

  #    # Eval model on train
  #    scores = model.evaluate(dataset, [classification_metric])
  #    assert scores[classification_metric.name] > .9

  #def test_keras_skewed_classification_overfit(self):
  #  """Test keras models can overfit 0/1 datasets with few actives."""
  #  g = tf.Graph()
  #  sess = tf.Session(graph=g)
  #  K.set_session(sess)
  #  with g.as_default():
  #    n_samples = 100
  #    n_features = 3
  #    n_tasks = 1
  #    
  #    # Generate dummy dataset
  #    np.random.seed(123)
  #    p = .05
  #    ids = np.arange(n_samples)
  #    X = np.random.rand(n_samples, n_features)
  #    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
  #    w = np.ones((n_samples, n_tasks))
  #  
  #    dataset = dc.data.NumpyDataset(X, y, w, ids)

  #    verbosity = "high"
  #    classification_metric = dc.metrics.Metric(
  #        dc.metrics.roc_auc_score, verbosity=verbosity)
  #    keras_model = dc.models.MultiTaskDNN(
  #        n_tasks, n_features, "classification",
  #        dropout=0., learning_rate=.15, decay=1e-4)
  #    model = dc.models.KerasModel(keras_model)

  #    # Fit trained model
  #    model.fit(dataset, batch_size=n_samples, nb_epoch=200)
  #    model.save()

  #    # Eval model on train
  #    scores = model.evaluate(dataset, [classification_metric])
  #    assert scores[classification_metric.name] > .9

  def test_tf_classification_overfit(self):
    """Test that tensorflow models can overfit simple classification datasets."""
    n_samples = 10
@@ -392,41 +286,6 @@ class TestOverfit(test_util.TensorFlowTestCase):
    scores = model.evaluate(dataset, [classification_metric])
    assert scores[classification_metric.name] > .9

  #def test_keras_multitask_classification_overfit(self):
  #  """Test keras multitask overfits tiny data."""
  #  g = tf.Graph()
  #  sess = tf.Session(graph=g)
  #  K.set_session(sess)
  #  with g.as_default():
  #    n_tasks = 10
  #    n_samples = 10
  #    n_features = 3
  #    
  #    # Generate dummy dataset
  #    np.random.seed(123)
  #    ids = np.arange(n_samples)
  #    X = np.random.rand(n_samples, n_features)
  #    y = np.random.randint(2, size=(n_samples, n_tasks))
  #    w = np.ones((n_samples, n_tasks))
  #    dataset = dc.data.NumpyDataset(X, y, w, ids)

  #    verbosity = "high"
  #    classification_metric = dc.metrics.Metric(
  #        dc.metrics.roc_auc_score, verbosity=verbosity,
  #       task_averager=np.mean, mode="classification")
  #    keras_model = dc.models.MultiTaskDNN(
  #        n_tasks, n_features, "classification", dropout=0., learning_rate=.15,
  #        decay=1e-4)
  #    model = dc.models.KerasModel(keras_model, verbosity=verbosity)

  #    # Fit trained model
  #    model.fit(dataset, nb_epoch=50)
  #    model.save()

  #    # Eval model on train
  #    scores = model.evaluate(dataset, [classification_metric])
  #    assert scores[classification_metric.name] > .9

  def test_tf_multitask_classification_overfit(self):
    """Test tf multitask overfits tiny data."""
    n_tasks = 10
@@ -551,41 +410,6 @@ class TestOverfit(test_util.TensorFlowTestCase):
    scores = model.evaluate(dataset, [regression_metric])
    assert scores[regression_metric.name] > .7

  #def test_keras_multitask_regression_overfit(self):
  #  """Test keras multitask overfits tiny data."""
  #  g = tf.Graph()
  #  sess = tf.Session(graph=g)
  #  K.set_session(sess)
  #  with g.as_default():
  #    n_tasks = 10
  #    n_samples = 10
  #    n_features = 3
  #    
  #    # Generate dummy dataset
  #    np.random.seed(123)
  #    ids = np.arange(n_samples)
  #    X = np.random.rand(n_samples, n_features)
  #    y = np.random.randint(2, size=(n_samples, n_tasks))
  #    w = np.ones((n_samples, n_tasks))
  #    dataset = dc.data.NumpyDataset(X, y, w, ids)

  #    verbosity = "high"
  #    regression_metric = dc.metrics.Metric(
  #        dc.metrics.r2_score, verbosity=verbosity, task_averager=np.mean,
  #        mode="regression")
  #    keras_model = dc.models.MultiTaskDNN(
  #        n_tasks, n_features, "regression", dropout=0., learning_rate=.1,
  #        decay=1e-4)
  #    model = dc.models.KerasModel(keras_model, verbosity=verbosity)

  #    # Fit trained model
  #    model.fit(dataset, nb_epoch=100)
  #    model.save()

  #    # Eval model on train
  #    scores = model.evaluate(dataset, [regression_metric])
  #    assert scores[regression_metric.name] > .75

  def test_tf_multitask_regression_overfit(self):
    """Test tf multitask overfits tiny data."""
    n_tasks = 10
+0 −40
Original line number Diff line number Diff line
@@ -51,46 +51,6 @@ class TestReload(unittest.TestCase):
    scores = reloaded_model.evaluate(dataset, [classification_metric])
    assert scores[classification_metric.name] > .9

  #def test_keras_reload(self):
  #  """Test that trained keras models can be reloaded correctly."""
  #  g = tf.Graph()
  #  sess = tf.Session(graph=g)
  #  K.set_session(sess)
  #  with g.as_default():
  #    n_samples = 10
  #    n_features = 3
  #    n_tasks = 1
  #    
  #    # Generate dummy dataset
  #    np.random.seed(123)
  #    ids = np.arange(n_samples)
  #    X = np.random.rand(n_samples, n_features)
  #    y = np.random.randint(2, size=(n_samples, n_tasks))
  #    w = np.ones((n_samples, n_tasks))
  #  
  #    dataset = dc.data.NumpyDataset(X, y, w, ids)

  #    classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score)
  #    keras_model = dc.models.MultiTaskDNN(
  #        n_tasks, n_features, "classification", dropout=0.)
  #    model_dir = tempfile.mkdtemp()
  #    model = dc.models.KerasModel(keras_model, model_dir)

  #    # Fit trained model
  #    model.fit(dataset)
  #    model.save()

  #    # Load trained model
  #    reloaded_keras_model = dc.models.MultiTaskDNN(
  #        n_tasks, n_features, "classification", dropout=0.)
  #    reloaded_model = dc.models.KerasModel(reloaded_keras_model, model_dir)
  #    reloaded_model.reload(
  #        custom_objects={"MultiTaskDNN": dc.models.MultiTaskDNN})

  #    # Eval model on train
  #    scores = reloaded_model.evaluate(dataset, [classification_metric])
  #    assert scores[classification_metric.name] > .6

  def test_tf_reload(self):
    """Test that tensorflow models can overfit simple classification datasets."""
    n_samples = 10