Commit 933a4f1c authored by Peter Eastman's avatar Peter Eastman
Browse files

Added test cases

parent 10471ebf

deepchem/models/tensorgraph/layers.py

+2 −1
Original line number Diff line number Diff line
@@ -386,7 +386,8 @@ class L2Loss(Layer): 
    in_layers = convert_to_layers(in_layers)

    guess, label = in_layers[0], in_layers[1]

    self.out_tensor = tf.reduce_mean(

        tf.square(guess.out_tensor - label.out_tensor))

        tf.square(guess.out_tensor - label.out_tensor),

        axis=list(range(1, len(label.out_tensor.shape))))

    return self.out_tensor

deepchem/models/tensorgraph/tests/test_layers.py

+1 −1
Original line number Diff line number Diff line
@@ -187,7 +187,7 @@ class TestLayers(test_util.TensorFlowTestCase): 
      label_tensor = tf.convert_to_tensor(label_tensor, dtype=tf.float32)

      out_tensor = L2Loss()(guess_tensor, label_tensor)

      out_tensor = out_tensor.eval()

      assert isinstance(out_tensor, np.float32)

      assert out_tensor.shape == (batch_size,)



  def test_softmax(self):

    """Test that Softmax can be invoked."""

deepchem/models/tests/test_api.py

+42 −0
Original line number Diff line number Diff line
@@ -182,3 +182,45 @@ class TestAPI(unittest.TestCase): 
    # Eval model on train/test

    _ = model.evaluate(train_dataset, classification_metrics, transformers)

    _ = model.evaluate(test_dataset, classification_metrics, transformers)



  def test_singletask_tg_mlp_ECFP_classification_API(self):

    """Test of TensorGraph singletask deepchem classification API."""

    n_features = 1024

    featurizer = dc.feat.CircularFingerprint(size=n_features)



    tasks = ["outcome"]

    current_dir = os.path.dirname(os.path.abspath(__file__))

    input_file = os.path.join(current_dir, "example_classification.csv")



    loader = dc.data.CSVLoader(

        tasks=tasks, smiles_field="smiles", featurizer=featurizer)

    dataset = loader.featurize(input_file)



    splitter = dc.splits.ScaffoldSplitter()

    train_dataset, test_dataset = splitter.train_test_split(dataset)



    transformers = [

        dc.trans.NormalizationTransformer(

            transform_y=True, dataset=train_dataset)

    ]



    for dataset in [train_dataset, test_dataset]:

      for transformer in transformers:

        dataset = transformer.transform(dataset)



    classification_metrics = [

        dc.metrics.Metric(dc.metrics.roc_auc_score),

        dc.metrics.Metric(dc.metrics.matthews_corrcoef),

        dc.metrics.Metric(dc.metrics.recall_score),

        dc.metrics.Metric(dc.metrics.accuracy_score)

    ]



    model = dc.models.TensorGraphMultiTaskClassifier(len(tasks), n_features)



    # Fit trained model

    model.fit(train_dataset)

    model.save()



    # Eval model on train/test

    _ = model.evaluate(train_dataset, classification_metrics, transformers)

    _ = model.evaluate(test_dataset, classification_metrics, transformers)

deepchem/models/tests/test_overfit.py

+271 −0
Original line number Diff line number Diff line
@@ -144,6 +144,41 @@ class TestOverfit(test_util.TensorFlowTestCase): 
    scores = model.evaluate(dataset, [regression_metric])

    assert scores[regression_metric.name] < .1



  def test_tg_regression_overfit(self):

    """Test that TensorGraph models can overfit simple regression datasets."""

    n_samples = 10

    n_features = 3

    n_tasks = 1



    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)



    regression_metric = dc.metrics.Metric(dc.metrics.mean_squared_error)

    # TODO(rbharath): This breaks with optimizer="momentum". Why?

    model = dc.models.TensorGraphMultiTaskRegressor(

        n_tasks,

        n_features,

        dropouts=[0.],

        weight_init_stddevs=[np.sqrt(6) / np.sqrt(1000)],

        batch_size=n_samples)

    model.set_optimizer(

        dc.models.tensorgraph.tensor_graph.TFWrapper(

            tf.train.AdamOptimizer, learning_rate=0.003, beta1=0.9,

            beta2=0.999))



    # Fit trained model

    model.fit(dataset, nb_epoch=100)

    model.save()



    # Eval model on train

    scores = model.evaluate(dataset, [regression_metric])

    assert scores[regression_metric.name] < .1



  def test_tf_classification_overfit(self):

    """Test that tensorflow models can overfit simple classification datasets."""

    n_samples = 10
@@ -176,6 +211,43 @@ class TestOverfit(test_util.TensorFlowTestCase): 
    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .9



  def test_tg_classification_overfit(self):

    """Test that TensorGraph models can overfit simple classification datasets."""

    n_samples = 10

    n_features = 3

    n_tasks = 1

    n_classes = 2



    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)



    classification_metric = dc.metrics.Metric(dc.metrics.accuracy_score)

    model = dc.models.TensorGraphMultiTaskClassifier(

        n_tasks,

        n_features,

        dropouts=[0.],

        weight_init_stddevs=[.1],

        batch_size=n_samples)

    model.set_optimizer(

        dc.models.tensorgraph.tensor_graph.TFWrapper(

            tf.train.AdamOptimizer,

            learning_rate=0.0003,

            beta1=0.9,

            beta2=0.999))



    # Fit trained model

    model.fit(dataset, nb_epoch=100)

    model.save()



    # Eval model on train

    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .9



  def test_tf_fittransform_regression_overfit(self):

    """Test that TensorFlow FitTransform models can overfit simple regression datasets."""

    n_samples = 10
@@ -209,6 +281,42 @@ class TestOverfit(test_util.TensorFlowTestCase): 
    scores = model.evaluate(dataset, [regression_metric])

    assert scores[regression_metric.name] < .1



  def test_tg_fittransform_regression_overfit(self):

    """Test that TensorGraph FitTransform models can overfit simple regression datasets."""

    n_samples = 10

    n_features = 3

    n_tasks = 1



    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features, n_features)

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)



    fit_transformers = [dc.trans.CoulombFitTransformer(dataset)]

    regression_metric = dc.metrics.Metric(dc.metrics.mean_squared_error)

    model = dc.models.TensorGraphMultiTaskFitTransformRegressor(

        n_tasks, [n_features, n_features],

        dropouts=[0.],

        weight_init_stddevs=[np.sqrt(6) / np.sqrt(1000)],

        batch_size=n_samples,

        fit_transformers=fit_transformers,

        n_evals=1)

    model.set_optimizer(

        dc.models.tensorgraph.tensor_graph.TFWrapper(

            tf.train.AdamOptimizer, learning_rate=0.003, beta1=0.9,

            beta2=0.999))



    # Fit trained model

    model.fit(dataset, nb_epoch=100)

    model.save()



    # Eval model on train

    scores = model.evaluate(dataset, [regression_metric])

    assert scores[regression_metric.name] < .1



  def test_tf_skewed_classification_overfit(self):

    """Test tensorflow models can overfit 0/1 datasets with few actives."""

    #n_samples = 100
@@ -244,6 +352,44 @@ class TestOverfit(test_util.TensorFlowTestCase): 
    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .75



  def test_tg_skewed_classification_overfit(self):

    """Test TensorGraph models can overfit 0/1 datasets with few actives."""

    #n_samples = 100

    n_samples = 100

    n_features = 3

    n_tasks = 1

    n_classes = 2



    # Generate dummy dataset

    np.random.seed(123)

    p = .05

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))



    dataset = dc.data.NumpyDataset(X, y, w, ids)



    classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score)

    model = dc.models.TensorGraphMultiTaskClassifier(

        n_tasks,

        n_features,

        dropouts=[0.],

        weight_init_stddevs=[.1],

        batch_size=n_samples)

    model.set_optimizer(

        dc.models.tensorgraph.tensor_graph.TFWrapper(

            tf.train.AdamOptimizer, learning_rate=0.003, beta1=0.9,

            beta2=0.999))



    # Fit trained model

    model.fit(dataset, nb_epoch=100)

    model.save()



    # Eval model on train

    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .75



  def test_tf_skewed_missing_classification_overfit(self):

    """TF, skewed data, few actives
@@ -289,6 +435,54 @@ class TestOverfit(test_util.TensorFlowTestCase): 
    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .8



  def test_tg_skewed_missing_classification_overfit(self):

    """TG, skewed data, few actives



    Test TensorGraph models overfit 0/1 datasets with missing data and few

    actives. This is intended to be as close to singletask MUV datasets as

    possible.

    """

    n_samples = 5120

    n_features = 6

    n_tasks = 1

    n_classes = 2



    # Generate dummy dataset

    np.random.seed(123)

    p = .002

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    y_flat, w_flat = np.squeeze(y), np.squeeze(w)

    y_nonzero = y_flat[w_flat != 0]

    num_nonzero = np.count_nonzero(y_nonzero)

    weight_nonzero = len(y_nonzero) / num_nonzero

    w_flat[y_flat != 0] = weight_nonzero

    w = np.reshape(w_flat, (n_samples, n_tasks))



    dataset = dc.data.DiskDataset.from_numpy(X, y, w, ids)



    classification_metric = dc.metrics.Metric(dc.metrics.roc_auc_score)

    model = dc.models.TensorGraphMultiTaskClassifier(

        n_tasks,

        n_features,

        dropouts=[0.],

        weight_init_stddevs=[1.],

        batch_size=n_samples)

    model.set_optimizer(

        dc.models.tensorgraph.tensor_graph.TFWrapper(

            tf.train.AdamOptimizer, learning_rate=0.003, beta1=0.9,

            beta2=0.999))



    # Fit trained model

    model.fit(dataset, nb_epoch=100)

    model.save()



    # Eval model on train

    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .8



  def test_sklearn_multitask_classification_overfit(self):

    """Test SKLearn singletask-to-multitask overfits tiny data."""

    n_tasks = 10
@@ -354,6 +548,44 @@ class TestOverfit(test_util.TensorFlowTestCase): 
    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .9



  def test_tg_multitask_classification_overfit(self):

    """Test TensorGraph multitask overfits tiny data."""

    n_tasks = 10

    n_samples = 10

    n_features = 3

    n_classes = 2



    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)



    classification_metric = dc.metrics.Metric(

        dc.metrics.accuracy_score, task_averager=np.mean)

    model = dc.models.TensorGraphMultiTaskClassifier(

        n_tasks,

        n_features,

        dropouts=[0.],

        weight_init_stddevs=[.1],

        batch_size=n_samples)

    model.set_optimizer(

        dc.models.tensorgraph.tensor_graph.TFWrapper(

            tf.train.AdamOptimizer,

            learning_rate=0.0003,

            beta1=0.9,

            beta2=0.999))



    # Fit trained model

    model.fit(dataset)

    model.save()



    # Eval model on train

    scores = model.evaluate(dataset, [classification_metric])

    assert scores[classification_metric.name] > .9



  def test_tf_robust_multitask_classification_overfit(self):

    """Test tf robust multitask overfits tiny data."""

    n_tasks = 10
@@ -517,6 +749,45 @@ class TestOverfit(test_util.TensorFlowTestCase): 
    scores = model.evaluate(dataset, [regression_metric])

    assert scores[regression_metric.name] < .1



  def test_tg_multitask_regression_overfit(self):

    """Test TensorGraph multitask overfits tiny data."""

    n_tasks = 10

    n_samples = 10

    n_features = 3

    n_classes = 2



    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))



    dataset = dc.data.NumpyDataset(X, y, w, ids)



    regression_metric = dc.metrics.Metric(

        dc.metrics.mean_squared_error, task_averager=np.mean, mode="regression")

    model = dc.models.TensorGraphMultiTaskRegressor(

        n_tasks,

        n_features,

        dropouts=[0.],

        weight_init_stddevs=[.1],

        batch_size=n_samples)

    model.set_optimizer(

        dc.models.tensorgraph.tensor_graph.TFWrapper(

            tf.train.AdamOptimizer,

            learning_rate=0.0003,

            beta1=0.9,

            beta2=0.999))



    # Fit trained model

    model.fit(dataset, nb_epoch=50)

    model.save()



    # Eval model on train

    scores = model.evaluate(dataset, [regression_metric])

    assert scores[regression_metric.name] < .1



  def test_tf_robust_multitask_regression_overfit(self):

    """Test tf robust multitask overfits tiny data."""

    np.random.seed(123)

CRA Git | Maintained and supported by SUSTech CRA and CCSE