Merge pull request #2044 from deepchem/duplicate_balancing

    -------

    a newly constructed Dataset object

    """

    newx, newy, neww = transformer.transform_array(self._X, self._y, self._w)

    return NumpyDataset(newx, newy, neww, self._ids[:])

    newx, newy, neww, newids = transformer.transform_array(

        self._X, self._y, self._w, self._ids)

    return NumpyDataset(newx, newy, neww, newids)

  def select(self, indices: Sequence[int],

             select_dir: str = None) -> "NumpyDataset":

        for shard_num, row in self.metadata_df.iterrows():

          logger.info("Transforming shard %d/%d" % (shard_num, n_shards))

          X, y, w, ids = self.get_shard(shard_num)

          newx, newy, neww = transformer.transform_array(X, y, w)

          yield (newx, newy, neww, ids)

          newx, newy, neww, newids = transformer.transform_array(X, y, w, ids)

          yield (newx, newy, neww, newids)

      dataset = DiskDataset.create_dataset(

          generator(), data_dir=out_dir, tasks=tasks)

    y = None if y_file is None else np.array(load_from_disk(y_file))

    w = None if w_file is None else np.array(load_from_disk(w_file))

    ids = np.array(load_from_disk(ids_file))

    X, y, w = transformer.transform_array(X, y, w)

    X, y, w, ids = transformer.transform_array(X, y, w, ids)

    basename = "shard-%d" % shard_num

    return DiskDataset.write_data_to_disk(out_dir, basename, tasks, X, y, w,

                                          ids)

    -------

    a newly constructed Dataset object

    """

    newx, newy, neww = transformer.transform_array(self.X, self.y, self.w)

    return NumpyDataset(newx, newy, neww, self.ids[:])

    newx, newy, neww, newids = transformer.transform_array(

        self.X, self.y, self.w, self.ids)

    return NumpyDataset(newx, newy, neww, newids)

  def select(self, indices: Sequence[int],

             select_dir: str = None) -> "ImageDataset":

class TestTransformer(dc.trans.Transformer):

  def transform_array(self, X, y, w):

    return (2 * X, 1.5 * y, w)

  def transform_array(self, X, y, w, ids):

    return (2 * X, 1.5 * y, w, ids)

class TestDatasets(test_util.TensorFlowTestCase):

  """

  Test basic top-level API for dataset objects.

  """

def test_transform_disk():

  """Test that the transform() method works for DiskDatasets."""

  dataset = load_solubility_data()

  X = dataset.X

  y = dataset.y

  w = dataset.w

  ids = dataset.ids

  # Transform it

  def test_sparsify_and_densify(self):

  transformer = TestTransformer(transform_X=True, transform_y=True)

  for parallel in (True, False):

    transformed = dataset.transform(transformer, parallel=parallel)

    np.testing.assert_array_equal(X, dataset.X)

    np.testing.assert_array_equal(y, dataset.y)

    np.testing.assert_array_equal(w, dataset.w)

    np.testing.assert_array_equal(ids, dataset.ids)

    np.testing.assert_array_equal(2 * X, transformed.X)

    np.testing.assert_array_equal(1.5 * y, transformed.y)

    np.testing.assert_array_equal(w, transformed.w)

    np.testing.assert_array_equal(ids, transformed.ids)

def test_sparsify_and_densify():

  """Test that sparsify and densify work as inverses."""

  # Test on identity matrix

  num_samples = 10

  X_reconstructed = dc.data.densify_features(X_sparse, num_features)

  np.testing.assert_array_equal(X, X_reconstructed)

  def test_pad_features(self):

def test_pad_features():

  """Test that pad_features pads features correctly."""

  batch_size = 100

  num_features = 10

  X_out = dc.data.pad_features(batch_size, X_b)

  assert len(X_out) == batch_size

  def test_pad_batches(self):

def test_pad_batches():

  """Test that pad_batch pads batches correctly."""

  batch_size = 100

  num_features = 10

                                                   ids_b)

  assert len(X_out) == len(y_out) == len(w_out) == len(ids_out) == batch_size

  def test_get_task_names(self):

def test_get_task_names():

  """Test that get_task_names returns correct task_names"""

  solubility_dataset = load_solubility_data()

  assert solubility_dataset.get_task_names() == ["log-solubility"]

      "task15", "task16"

  ])

  def test_get_data_shape(self):

def test_get_data_shape():

  """Test that get_data_shape returns currect data shape"""

  solubility_dataset = load_solubility_data()

  assert solubility_dataset.get_data_shape() == (1024,)

  multitask_dataset = load_multitask_data()

  assert multitask_dataset.get_data_shape() == (1024,)

  def test_len(self):

def test_len():

  """Test that len(dataset) works."""

  solubility_dataset = load_solubility_data()

  assert len(solubility_dataset) == 10

  def test_reshard(self):

def test_reshard():

  """Test that resharding the dataset works."""

  solubility_dataset = load_solubility_data()

  X, y, w, ids = (solubility_dataset.X, solubility_dataset.y,

  np.testing.assert_array_equal(w, w_rr)

  np.testing.assert_array_equal(ids, ids_rr)

  def test_select(self):

def test_select():

  """Test that dataset select works."""

  num_datapoints = 10

  num_features = 10

  np.testing.assert_array_equal(w[indices], w_sel)

  np.testing.assert_array_equal(ids[indices], ids_sel)

  def test_complete_shuffle(self):

def test_complete_shuffle():

  shard_sizes = [1, 2, 3, 4, 5]

  batch_size = 10

      np.sort(dataset.w, axis=0), np.sort(res.w, axis=0))

  np.testing.assert_array_equal(np.sort(dataset.ids), np.sort(res.ids))

  def test_get_shape(self):

def test_get_shape():

  """Test that get_shape works."""

  num_datapoints = 100

  num_features = 10

  assert w_shape == w.shape

  assert ids_shape == ids.shape

  def test_iterbatches(self):

def test_iterbatches():

  """Test that iterating over batches of data works."""

  solubility_dataset = load_solubility_data()

  batch_size = 2

    assert w_b.shape == (batch_size,) + (len(tasks),)

    assert ids_b.shape == (batch_size,)

  def test_itersamples_numpy(self):

def test_itersamples_numpy():

  """Test that iterating over samples in a NumpyDataset works."""

  num_datapoints = 100

  num_features = 10

    np.testing.assert_array_equal(sw, w[i])

    np.testing.assert_array_equal(sid, ids[i])

  def test_itersamples_disk(self):

def test_itersamples_disk():

  """Test that iterating over samples in a DiskDataset works."""

  solubility_dataset = load_solubility_data()

  X = solubility_dataset.X

    np.testing.assert_array_equal(sw, w[i])

    np.testing.assert_array_equal(sid, ids[i])

  def test_transform_numpy(self):

def test_transform_numpy():

  """Test that the transform() method works for NumpyDatasets."""

  num_datapoints = 100

  num_features = 10

  np.testing.assert_array_equal(w, transformed.w)

  np.testing.assert_array_equal(ids, transformed.ids)

  def test_transform_disk(self):

    """Test that the transform() method works for DiskDatasets."""

    dataset = load_solubility_data()

    X = dataset.X

    y = dataset.y

    w = dataset.w

    ids = dataset.ids

    # Transform it

    transformer = TestTransformer(transform_X=True, transform_y=True)

    for parallel in (True, False):

      transformed = dataset.transform(transformer, parallel=parallel)

      np.testing.assert_array_equal(X, dataset.X)

      np.testing.assert_array_equal(y, dataset.y)

      np.testing.assert_array_equal(w, dataset.w)

      np.testing.assert_array_equal(ids, dataset.ids)

      np.testing.assert_array_equal(2 * X, transformed.X)

      np.testing.assert_array_equal(1.5 * y, transformed.y)

      np.testing.assert_array_equal(w, transformed.w)

      np.testing.assert_array_equal(ids, transformed.ids)

  def test_to_numpy(self):

def test_to_numpy():

  """Test that transformation to numpy arrays is sensible."""

  solubility_dataset = load_solubility_data()

  data_shape = solubility_dataset.get_data_shape()

  assert w.shape == (N_samples, N_tasks)

  assert ids.shape == (N_samples,)

  def test_consistent_ordering(self):

def test_consistent_ordering():

  """Test that ordering of labels is consistent over time."""

  solubility_dataset = load_solubility_data()

  assert np.array_equal(ids1, ids2)

  def test_get_statistics(self):

def test_get_statistics():

  """Test statistics computation of this dataset."""

  solubility_dataset = load_solubility_data()

  X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,

  np.testing.assert_allclose(comp_X_stds, X_stds)

  np.testing.assert_allclose(comp_y_stds, y_stds)

  def test_disk_iterate_batch_size(self):

def test_disk_iterate_batch_size():

  solubility_dataset = load_solubility_data()

  X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,

                solubility_dataset.w, solubility_dataset.ids)

  for X, y, _, _ in solubility_dataset.iterbatches(

      3, epochs=2, pad_batches=False, deterministic=True):

    batch_sizes.append(len(X))

    self.assertEqual([3, 3, 3, 1, 3, 3, 3, 1], batch_sizes)

  assert [3, 3, 3, 1, 3, 3, 3, 1] == batch_sizes

  def test_disk_pad_batches(self):

def test_disk_pad_batches():

  shard_sizes = [21, 11, 41, 21, 51]

  batch_size = 10

  np.testing.assert_array_equal(all_ws, test_ws[:total_size, :])

  np.testing.assert_array_equal(all_ids, test_ids[:total_size])

  def test_disk_iterate_y_w_None(self):

def test_disk_iterate_y_w_None():

  shard_sizes = [21, 11, 41, 21, 51]

  batch_size = 10

  np.testing.assert_array_equal(all_Xs, test_Xs[:total_size, :])

  np.testing.assert_array_equal(all_ids, test_ids[:total_size])

  def test_disk_iterate_batch(self):

def test_disk_iterate_batch():

  all_batch_sizes = [None, 32, 17, 11]

  all_shard_sizes = [[7, 3, 12, 4, 5], [1, 1, 1, 1, 1], [31, 31, 31, 31, 31],

    np.testing.assert_array_equal(

        np.sort(all_ids, axis=0), np.sort(test_ids, axis=0))

  def test_numpy_iterate_batch_size(self):

    solubility_dataset = load_solubility_data()

    X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,

                  solubility_dataset.w, solubility_dataset.ids)

    solubility_dataset = dc.data.NumpyDataset.from_DiskDataset(

        solubility_dataset)

    batch_sizes = []

    for X, y, _, _ in solubility_dataset.iterbatches(

        3, epochs=2, pad_batches=False, deterministic=True):

      batch_sizes.append(len(X))

    self.assertEqual([3, 3, 3, 1, 3, 3, 3, 1], batch_sizes)

  def test_merge(self):

def test_merge():

  """Test that dataset merge works."""

  num_datapoints = 10

  num_features = 10

  assert new_data.y.shape == (num_datapoints * num_datasets, num_tasks)

  assert len(new_data.tasks) == len(datasets[0].tasks)

  def test_make_tf_dataset(self):

def test_make_tf_dataset():

  """Test creating a Tensorflow Iterator from a Dataset."""

  X = np.random.random((100, 5))

  y = np.random.random((100, 1))

    np.testing.assert_array_equal(np.ones((10, 1)), batch_w)

  assert i == 19

  def _validate_pytorch_dataset(self, dataset):

def _validate_pytorch_dataset(dataset):

  X = dataset.X

  y = dataset.y

  w = dataset.w

    id_count[iter_id[0]] += 1

  assert all(id_count[id] == 2 for id in ids)

  @unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')

  def test_make_pytorch_dataset_from_numpy(self):

    """Test creating a PyTorch Dataset from a NumpyDataset."""

    X = np.random.random((100, 5))

    y = np.random.random((100, 1))

    ids = [str(i) for i in range(100)]

    dataset = dc.data.NumpyDataset(X, y, ids=ids)

    self._validate_pytorch_dataset(dataset)

  @unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')

  def test_make_pytorch_dataset_from_images(self):

    """Test creating a PyTorch Dataset from an ImageDataset."""

    path = os.path.join(os.path.dirname(__file__), 'images')

    files = [os.path.join(path, f) for f in os.listdir(path)]

    y = np.random.random((10, 1))

    ids = [str(i) for i in range(len(files))]

    dataset = dc.data.ImageDataset(files, y, ids=ids)

    self._validate_pytorch_dataset(dataset)

  @unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')

  def test_make_pytorch_dataset_from_disk(self):

    """Test creating a PyTorch Dataset from a DiskDataset."""

    dataset = load_solubility_data()

    self._validate_pytorch_dataset(dataset)

  def test_dataframe(self):

def test_dataframe():

  """Test converting between Datasets and DataFrames."""

  dataset = load_solubility_data()

  np.testing.assert_array_equal(

      np.stack([dataset.y[:, 0], dataset.X[:, 0]], axis=1), dataset3.w)

  def test_to_str(self):

def test_to_str():

  """Tests to string representation of Dataset."""

  dataset = dc.data.NumpyDataset(

      X=np.random.rand(5, 3), y=np.random.rand(5,), ids=np.arange(5))

      X=np.random.rand(50, 3), y=np.random.rand(50,), ids=np.arange(50))

  ref_str = '<NumpyDataset X.shape: (50, 3), y.shape: (50,), w.shape: (50,), task_names: [0]>'

  assert str(dataset) == ref_str

class TestDatasets(test_util.TensorFlowTestCase):

  """

  Test basic top-level API for dataset objects.

  """

  def test_numpy_iterate_batch_size(self):

    solubility_dataset = load_solubility_data()

    X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,

                  solubility_dataset.w, solubility_dataset.ids)

    solubility_dataset = dc.data.NumpyDataset.from_DiskDataset(

        solubility_dataset)

    batch_sizes = []

    for X, y, _, _ in solubility_dataset.iterbatches(

        3, epochs=2, pad_batches=False, deterministic=True):

      batch_sizes.append(len(X))

    self.assertEqual([3, 3, 3, 1, 3, 3, 3, 1], batch_sizes)

  @unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')

  def test_make_pytorch_dataset_from_numpy(self):

    """Test creating a PyTorch Dataset from a NumpyDataset."""

    X = np.random.random((100, 5))

    y = np.random.random((100, 1))

    ids = [str(i) for i in range(100)]

    dataset = dc.data.NumpyDataset(X, y, ids=ids)

    _validate_pytorch_dataset(dataset)

  @unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')

  def test_make_pytorch_dataset_from_images(self):

    """Test creating a PyTorch Dataset from an ImageDataset."""

    path = os.path.join(os.path.dirname(__file__), 'images')

    files = [os.path.join(path, f) for f in os.listdir(path)]

    y = np.random.random((10, 1))

    ids = [str(i) for i in range(len(files))]

    dataset = dc.data.ImageDataset(files, y, ids=ids)

    _validate_pytorch_dataset(dataset)

  @unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')

  def test_make_pytorch_dataset_from_disk(self):

    """Test creating a PyTorch Dataset from a DiskDataset."""

    dataset = load_solubility_data()

    _validate_pytorch_dataset(dataset)

    for transformer in fit_transformers:

      assert transformer.transform_X and not (transformer.transform_y or

                                              transformer.transform_w)

      X_b, _, _ = transformer.transform_array(X_b, None, None)

      X_b, _, _, _ = transformer.transform_array(X_b, None, None, None)

    n_features = X_b.shape[1]

    logger.info("n_features after fit_transform: %d", int(n_features))

    super(MultitaskFitTransformRegressor, self).__init__(

        if X_b is not None:

          if mode == 'fit':

            for transformer in self.fit_transformers:

              X_b, _, _ = transformer.transform_array(X_b, None, None)

              X_b, _, _, _ = transformer.transform_array(X_b, None, None, None)

        if mode == 'predict':

          dropout = np.array(0.0)

        else:

from deepchem.trans.transformers import ImageTransformer

from deepchem.trans.transformers import DataTransforms

from deepchem.trans.transformers import Transformer

from deepchem.trans.duplicate import DuplicateBalancingTransformer

import logging

import numpy as np

from typing import Tuple

from deepchem.data import Dataset

from deepchem.trans.transformers import Transformer

logger = logging.getLogger(__name__)

class DuplicateBalancingTransformer(Transformer):

  """Balance binary or multiclass datasets by duplicating rarer class samples.

  This class balances a dataset by duplicating samples of the rarer class so

  that the sum of all example weights from all classes is the same. (Up to

  integer rounding of course). This can be useful when you're working on an

  imabalanced dataset where there are far fewer examples of some classes than

  others. 

  This class differs from `BalancingTransformer` in that it actually

  duplicates rarer class samples rather than just increasing their sample

  weights. This may be more friendly for models that are numerically fragile

  and can't handle imbalanced example weights.

  Examples

  --------

  Here's an example for a binary dataset.

  >>> n_samples = 10

  >>> n_features = 3

  >>> n_tasks = 1

  >>> n_classes = 2

  >>> import deepchem as dc

  >>> import numpy as np

  >>> ids = np.arange(n_samples)

  >>> X = np.random.rand(n_samples, n_features)

  >>> y = np.random.randint(n_classes, size=(n_samples, n_tasks))

  >>> w = np.ones((n_samples, n_tasks))

  >>> dataset = dc.data.NumpyDataset(X, y, w, ids)

  >>> transformer = dc.trans.DuplicateBalancingTransformer(dataset=dataset)

  >>> dataset = transformer.transform(dataset)

  And here's a multiclass dataset example.

  >>> n_samples = 50

  >>> n_features = 3

  >>> n_tasks = 1

  >>> n_classes = 5

  >>> ids = np.arange(n_samples)

  >>> X = np.random.rand(n_samples, n_features)

  >>> y = np.random.randint(n_classes, size=(n_samples, n_tasks))

  >>> w = np.ones((n_samples, n_tasks))

  >>> dataset = dc.data.NumpyDataset(X, y, w, ids)

  >>> transformer = dc.trans.DuplicateBalancingTransformer(dataset=dataset)

  >>> dataset = transformer.transform(dataset)

  See Also

  --------

  deepchem.trans.BalancingTransformer: Balance by changing sample weights.

  Note

  ----

  This transformer is only well-defined for singletask datasets. (Since

  examples are actually duplicated, there's no meaningful way to duplicate

  across multiple tasks in a way that preserves the balance.) 

  This transformer is only meaningful for classification datasets where `y`

  takes on a limited set of values. This class transforms all of `X`, `y`,

  `w`, `ids`.

  Raises

  ------

  `ValueError` if the provided dataset is multitask.

  """

  def __init__(self, dataset: Dataset):

    super(DuplicateBalancingTransformer, self).__init__(

        transform_X=True,

        transform_y=True,

        transform_w=True,

        transform_ids=True,

        dataset=dataset)

    if len(dataset.get_task_names()) > 1:

      raise ValueError(

          "This transformation is only defined for singletask datsets.")

    # Get the labels/weights

    y = dataset.y

    w = dataset.w

    # Normalize shapes

    if len(y.shape) == 1:

      y = np.reshape(y, (len(y), 1))

    if len(w.shape) == 1:

      w = np.reshape(w, (len(w), 1))

    if len(y.shape) != 2: