Add simple overfit test for sklearn. Still broken

    Raises:

      NotImplementedError: If metric_str is not in METRICS.

    """

    print("compute_singletask_metric")

    print("y_true")

    print(y_true)

    print("y_pred")

    print(y_pred)

    print("w")

    print(w)

    y_true = y_true[w != 0]

    y_pred = y_pred[w != 0]

    # If there are no nonzero examples, metric is ill-defined.

    raise NotImplementedError(

        "Each model is responsible for its own predict_on_batch method.")

  def predict_proba_on_batch(self, X):

    """

    Makes predictions of class probabilities on given batch of new data.

    """

    raise NotImplementedError(

        "Each model is responsible for its own predict_on_batch method.")

  def set_raw_model(self, raw_model):

    """

    Set underlying raw model. Useful when loading from disk.

    """

    Uses self to make predictions on provided Dataset object.

    """

    X, y, w, ids = dataset.to_numpy()

    batch_size = self.model_params["batch_size"]

    y_preds = []

    batch_size = self.model_params["batch_size"]

    for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):

      y_pred_batch = np.reshape(self.predict_on_batch(X_batch), y_batch.shape)

      y_pred_batch = undo_transforms(y_pred_batch, transformers)

    # The iterbatches does padding with zero-weight examples on the last batch.

    # Remove padded examples.

    y_pred = y_pred[:len(dataset)]

    return y_pred

  def predict_proba(self, dataset, transformers):

    y_preds = []

    batch_size = self.model_params["batch_size"]

    for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):

      y_pred_batch = np.reshape(self.predict_proba_on_batch(X_batch), y_batch.shape)

      y_pred_batch = undo_transforms(y_pred_batch, transformers)

      y_preds.append(y_pred_batch)

    y_pred = np.vstack(y_preds)

    # The iterbatches does padding with zero-weight examples on the last batch.

    # Remove padded examples.

    y_pred = y_pred[:len(dataset)]

    return y_pred

  def get_task_type(self):

    """

    Fits SKLearn model to data.

    """

    Xs, ys, ws = [], [], []

    for (X_batch, y_batch, w_batch, _) in dataset.iterbatches(batch_size=32):

      Xs.append(X_batch)

      ys.append(y_batch)

      ws.append(w_batch)

    X = np.concatenate(Xs)

    y = np.concatenate(ys).ravel()

    w = np.concatenate(ws).ravel()

    X, y, w, _ = dataset.to_numpy()

    y, w = y.flatten(), w.flatten()

    print("fit")

    print("X")

    print(X)

    print("y")

    print(y)

    self.raw_model.fit(X, y, w)

    y_pred_raw = self.raw_model.predict(X)

  def predict_on_batch(self, X):

    """

    Makes predictions on batch of data.

    """

    if self.mode == "classification":

      return self.raw_model.predict_proba(X)

    else:

    return self.raw_model.predict(X)

  def predict_proba_on_batch(self, X):

    return self.raw_model.predict_proba(X)

  def predict(self, X, transformers):

    """

    Makes predictions on dataset.

"""

Tests to make sure TF models can overfit on tiny datasets.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals

__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "LGPL"

import tempfile

import numpy as np

import unittest

import sklearn

from deepchem import metrics

from deepchem.datasets import Dataset

from deepchem.metrics import Metric

from deepchem.models.test import TestAPI

from deepchem.utils.evaluate import Evaluator

from deepchem.models.sklearn_models import SklearnModel

from sklearn.ensemble import RandomForestClassifier

class TestOverfitAPI(TestAPI):

  """

  Test that sklearn and keras models can overfit simple datasets.

  """

  def test_classification_overfit(self):

    """Test that data associated with a tasks stays associated with it."""

    tasks = ["task0"]

    task_types = {task: "classification" for task in tasks}

    n_samples = 10

    n_features = 3

    n_tasks = len(tasks)

    # Generate dummy dataset

    ids = np.arange(n_samples)

    #X = np.random.rand(n_samples, n_features)

    X = np.ones((n_samples, n_features))

    #y = np.random.randint(2, size=(n_samples, n_tasks))

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)

    model_params = {

      "batch_size": None,

      "data_shape": dataset.get_data_shape()

    }

    np.set_printoptions(precision=5)

    verbosity = "high"

    classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity)

    model = SklearnModel(tasks, task_types, model_params, self.model_dir,

                         mode="classification",

                         model_instance=RandomForestClassifier())

    cl = RandomForestClassifier()

    y, w = y.flatten(), w.flatten()

    cl.fit(X, y, w)

    y_pred = cl.predict(X)

    np.set_printoptions(precision=5)

    y, y_pred = y.flatten(), y_pred.flatten()

    np.testing.assert_array_almost_equal(y, y_pred)

    # Fit trained model

    model.fit(dataset)

    model.save()

    X_dataset, y_dataset, _, _ = dataset.to_numpy()

    np.testing.assert_array_almost_equal(X, X_dataset)

    np.testing.assert_array_almost_equal(y.flatten(), y_dataset.flatten())

    y_pred_model = model.predict(dataset, transformers=[])

    print("y_pred_model")

    print(y_pred_model)

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])

    print("y_pred_proba_model")

    print(y_pred_proba_model)

    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)

    print("sklearn.metrics.accuracy_score(y, y_pred)")

    print(sklearn.metrics.accuracy_score(y, y_pred))

    print("metrics.compute_roc_auc_scores(y, y_pred_proba_model)")

    print(metrics.compute_roc_auc_scores(y, y_pred_proba_model))

    print("scores")

    print(scores)

    assert scores[classification_metric.name] > .9

from deepchem.models.tensorflow_models import TensorflowModel

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

class TestTensorflowAPI(TestAPI):

class TestTensorflowOverfitAPI(TestAPI):

  """

  Test top-level API for ML models."

  Test tensorflow models can overfit simple datasets.

  """

  def test_classification_overfit(self):