Merge pull request #1965 from deepchem/transformer_docs

"""

General API for testing dataset objects

"""

__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "MIT"

import unittest

import tempfile

import os

import shutil

import numpy as np

import deepchem as dc

def load_solubility_data():

  """Loads solubility dataset"""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = ["log-solubility"]

  task_type = "regression"

  input_file = os.path.join(current_dir, "../../models/tests/example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

def load_butina_data():

  """Loads solubility dataset"""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = ["task"]

  # task_type = "regression"

  input_file = os.path.join(current_dir,

                            "../../models/tests/butina_example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

def load_multitask_data():

  """Load example multitask data."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = [

      "task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7",

      "task8", "task9", "task10", "task11", "task12", "task13", "task14",

      "task15", "task16"

  ]

  input_file = os.path.join(current_dir,

                            "../../models/tests/multitask_example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

def load_classification_data():

  """Loads classification data from example.csv"""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = ["outcome"]

  task_type = "classification"

  input_file = os.path.join(current_dir,

                            "../../models/tests/example_classification.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

def load_sparse_multitask_dataset():

  """Load sparse tox multitask data, sample dataset."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = [

      "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8",

      "task9"

  ]

  input_file = os.path.join(current_dir,

                            "../../models/tests/sparse_multitask_example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

def load_feat_multitask_data():

  """Load example with numerical features, tasks."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  features = ["feat0", "feat1", "feat2", "feat3", "feat4", "feat5"]

  featurizer = dc.feat.UserDefinedFeaturizer(features)

  tasks = ["task0", "task1", "task2", "task3", "task4", "task5"]

  input_file = os.path.join(current_dir,

                            "../../models/tests/feat_multitask_example.csv")

  loader = dc.data.UserCSVLoader(

      tasks=tasks, featurizer=featurizer, id_field="id")

  return loader.featurize(input_file)

def load_gaussian_cdf_data():

  """Load example with numbers sampled from Gaussian normal distribution.

     Each feature and task is a column of values that is sampled

     from a normal distribution of mean 0, stdev 1."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  features = ["feat0", "feat1"]

  featurizer = dc.feat.UserDefinedFeaturizer(features)

  tasks = ["task0", "task1"]

  input_file = os.path.join(current_dir,

                            "../../models/tests/gaussian_cdf_example.csv")

  loader = dc.data.UserCSVLoader(

      tasks=tasks, featurizer=featurizer, id_field="id")

  return loader.featurize(input_file)

def load_unlabelled_data():

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = []

  input_file = os.path.join(current_dir, "../../data/tests/no_labels.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

  PYTORCH_IMPORT_FAILED = True

def load_solubility_data():

  """Loads solubility dataset"""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = ["log-solubility"]

  task_type = "regression"

  input_file = os.path.join(current_dir, "../../models/tests/example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.create_dataset(input_file)

def load_multitask_data():

  """Load example multitask data."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = [

      "task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7",

      "task8", "task9", "task10", "task11", "task12", "task13", "task14",

      "task15", "task16"

  ]

  input_file = os.path.join(current_dir,

                            "../../models/tests/multitask_example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

class TestDatasets(test_util.TensorFlowTestCase):

  """

  Test basic top-level API for dataset objects.

  def test_get_task_names(self):

    """Test that get_task_names returns correct task_names"""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    assert solubility_dataset.get_task_names() == ["log-solubility"]

    multitask_dataset = dc.data.tests.load_multitask_data()

    multitask_dataset = load_multitask_data()

    assert sorted(multitask_dataset.get_task_names()) == sorted([

        "task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7",

        "task8", "task9", "task10", "task11", "task12", "task13", "task14",

  def test_get_data_shape(self):

    """Test that get_data_shape returns currect data shape"""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    assert solubility_dataset.get_data_shape() == (1024,)

    multitask_dataset = dc.data.tests.load_multitask_data()

    multitask_dataset = load_multitask_data()

    assert multitask_dataset.get_data_shape() == (1024,)

  def test_len(self):

    """Test that len(dataset) works."""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    assert len(solubility_dataset) == 10

  def test_reshard(self):

    """Test that resharding the dataset works."""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    X, y, w, ids = (solubility_dataset.X, solubility_dataset.y,

                    solubility_dataset.w, solubility_dataset.ids)

    assert solubility_dataset.get_number_shards() == 1

  def test_iterbatches(self):

    """Test that iterating over batches of data works."""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    batch_size = 2

    data_shape = solubility_dataset.get_data_shape()

    tasks = solubility_dataset.get_task_names()

  def test_itersamples_disk(self):

    """Test that iterating over samples in a DiskDataset works."""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    X = solubility_dataset.X

    y = solubility_dataset.y

    w = solubility_dataset.w

  def test_transform_disk(self):

    """Test that the transform() method works for DiskDatasets."""

    dataset = dc.data.tests.load_solubility_data()

    dataset = load_solubility_data()

    X = dataset.X

    y = dataset.y

    w = dataset.w

  def test_to_numpy(self):

    """Test that transformation to numpy arrays is sensible."""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    data_shape = solubility_dataset.get_data_shape()

    tasks = solubility_dataset.get_task_names()

    X, y, w, ids = (solubility_dataset.X, solubility_dataset.y,

  def test_consistent_ordering(self):

    """Test that ordering of labels is consistent over time."""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    ids1 = solubility_dataset.ids

    ids2 = solubility_dataset.ids

  def test_get_statistics(self):

    """Test statistics computation of this dataset."""

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,

                  solubility_dataset.w, solubility_dataset.ids)

    X_means, y_means = np.mean(X, axis=0), np.mean(y, axis=0)

    np.testing.assert_allclose(comp_y_stds, y_stds)

  def test_disk_iterate_batch_size(self):

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,

                  solubility_dataset.w, solubility_dataset.ids)

    batch_sizes = []

          np.sort(all_ids, axis=0), np.sort(test_ids, axis=0))

  def test_numpy_iterate_batch_size(self):

    solubility_dataset = dc.data.tests.load_solubility_data()

    solubility_dataset = load_solubility_data()

    X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,

                  solubility_dataset.w, solubility_dataset.ids)

    solubility_dataset = dc.data.NumpyDataset.from_DiskDataset(

  @unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')

  def test_make_pytorch_dataset_from_disk(self):

    """Test creating a PyTorch Dataset from a DiskDataset."""

    dataset = dc.data.tests.load_solubility_data()

    dataset = load_solubility_data()

    self._validate_pytorch_dataset(dataset)

  def test_dataframe(self):

    """Test converting between Datasets and DataFrames."""

    dataset = dc.data.tests.load_solubility_data()

    dataset = load_solubility_data()

    # A round trip from Dataset to DataFrame to Dataset should produce identical arrays.

import itertools

import os

import deepchem as dc

import logging

from deepchem.data import DiskDataset

from deepchem.utils import ScaffoldGenerator

from deepchem.utils.save import log

from deepchem.data import NumpyDataset

from deepchem.utils.save import load_data

logger = logging.getLogger(__name__)

def generate_scaffold(smiles, include_chirality=False):

  """Compute the Bemis-Murcko scaffold for a SMILES string."""

    Abstract base class for chemically aware splits..

    """

  def __init__(self, verbose=False):

    """Creates splitter object."""

    self.verbose = verbose

  def k_fold_split(self, dataset, k, directories=None, **kwargs):

    """

    Parameters

    :param kwargs:

    :return: list of length k tuples of (train, cv)

    """

    log("Computing K-fold split", self.verbose)

    logger.info("Computing K-fold split")

    if directories is None:

      directories = [tempfile.mkdtemp() for _ in range(2 * k)]

    else:

                             frac_test=.1,

                             seed=None,

                             log_every_n=1000,

                             verbose=True,

                             **kwargs):

    """

        Splits self into train/validation/test sets.

    """ Splits self into train/validation/test sets.

    Returns Dataset objects.

    """

    log("Computing train/valid/test indices", self.verbose)

    logger.info("Computing train/valid/test indices")

    train_inds, valid_inds, test_inds = self.split(

        dataset,

        seed=seed,

                       test_dir=None,

                       seed=None,

                       frac_train=.8,

                       verbose=True,

                       **kwargs):

    """

        Splits self into train/test sets.

    """Splits self into train/test sets.

    Returns Dataset objects.

    """

    valid_dir = tempfile.mkdtemp()

        frac_test=1 - frac_train,

        frac_valid=0.,

        seed=seed,

        verbose=verbose,

        **kwargs)

    return train_dataset, test_dataset

            frac_valid=None,

            frac_test=None,

            log_every_n=None,

            verbose=False,

            **kwargs):

    """

    Stub to be filled in by child classes.

  def k_fold_split(self, dataset, k, directories=None, **kwargs):

    """Needs custom implementation due to ragged splits for stratification."""

    log("Computing K-fold split", self.verbose)

    logger.info("Computing K-fold split")

    if directories is None:

      directories = [tempfile.mkdtemp() for _ in range(k)]

    else:

    >>> y = np.random.rand(n_samples, n_tasks)

    >>> w = np.ones_like(y)

    >>> dataset = DiskDataset.from_numpy(np.ones((100,n_tasks)), np.ones((100,n_tasks)))

    >>> splitter = SingletaskStratifiedSplitter(task_number=5, verbose=False)

    >>> splitter = SingletaskStratifiedSplitter(task_number=5)

    >>> train_dataset, test_dataset = splitter.train_test_split(dataset)

    """

  def __init__(self, task_number=0, verbose=False):

  def __init__(self, task_number=0):

    """

    Creates splitter object.

    ----------

    task_number: int (Optional, Default 0)

      Task number for stratification.

        verbose: bool (Optional, Default False)

          Controls logging frequency.

    """

    self.task_number = task_number

    self.verbose = verbose

  def k_fold_split(self,

                   dataset,

    fold_datasets: List

      List containing dc.data.Dataset objects

    """

    log("Computing K-fold split", self.verbose)

    logger.info("Computing K-fold split")

    if directories is None:

      directories = [tempfile.mkdtemp() for _ in range(k)]

    else:

    Class for splits based on input order.

    """

  def __init__(self, verbose=False, valid_indices=None, test_indices=None):

  def __init__(self, valid_indices=None, test_indices=None):

    """

    Parameters

    -----------

    test_indices: list of int

        indices of samples in the test set

    """

    self.verbose = verbose

    self.valid_indices = valid_indices

    self.test_indices = test_indices

    valid_cutoff = (frac_train + frac_valid) * len(dataset)

    train_inds, valid_inds, test_inds = [], [], []

    log("About to sort in scaffold sets", self.verbose)

    logger.info("About to sort in scaffold sets")

    for scaffold_set in scaffold_sets:

      if len(train_inds) + len(scaffold_set) > train_cutoff:

        if len(train_inds) + len(valid_inds) + len(scaffold_set) > valid_cutoff:

    scaffolds = {}

    data_len = len(dataset)

    log("About to generate scaffolds", self.verbose)

    logger.info("About to generate scaffolds")

    for ind, smiles in enumerate(dataset.ids):

      if ind % log_every_n == 0:

        log("Generating scaffold %d/%d" % (ind, data_len), self.verbose)

        logger.info("Generating scaffold %d/%d" % (ind, data_len))

      scaffold = generate_scaffold(smiles)

      if scaffold not in scaffolds:

        scaffolds[scaffold] = [ind]

  Class that splits data according to user specification.

  """

  def __init__(self, input_file, split_field, verbose=False):

  def __init__(self, input_file, split_field):

    """Provide input information for splits."""

    raw_df = next(load_data([input_file], shard_size=None))

    self.splits = raw_df[split_field].values

    self.verbose = verbose

  def split(self,

            dataset,

  Class that splits data according to user index specification

  """

  def __init__(self, train_inds, valid_inds, test_inds, verbose=False):

  def __init__(self, train_inds, valid_inds, test_inds):

    """Provide input information for splits."""

    self.train_inds = train_inds

    self.valid_inds = valid_inds

    self.test_inds = test_inds

    self.verbose = verbose

    super(SpecifiedIndexSplitter, self).__init__(verbose)

  def split(self,

            dataset,

            frac_train=.8,

            frac_valid=.1,

            frac_test=.1,

            log_every_n=1000,

            verbose=False):

            log_every_n=1000):

    """

    Splits internal compounds into train/validation/test by user-specification.

    """

class TimeSplitterPDBbind(Splitter):

  def __init__(self, ids, year_file=None, verbose=False):

  def __init__(self, ids, year_file=None):

    self.ids = ids

    self.year_file = year_file

    self.verbose = verbose

  def split(self,

            dataset,

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "MIT"

import os

import tempfile

import unittest

import numpy as np

from deepchem.splits import IndexSplitter

def load_sparse_multitask_dataset():

  """Load sparse tox multitask data, sample dataset."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = [

      "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8",

      "task9"

  ]

  input_file = os.path.join(current_dir,

                            "../../models/tests/sparse_multitask_example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

def load_multitask_data():

  """Load example multitask data."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = [

      "task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7",

      "task8", "task9", "task10", "task11", "task12", "task13", "task14",

      "task15", "task16"

  ]

  input_file = os.path.join(current_dir,

                            "../../models/tests/multitask_example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

def load_solubility_data():

  """Loads solubility dataset"""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = ["log-solubility"]

  task_type = "regression"

  input_file = os.path.join(current_dir, "../../models/tests/example.csv")

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

  return loader.featurize(input_file)

def load_butina_data():

  """Loads solubility dataset"""

  current_dir = os.path.dirname(os.path.abspath(__file__))