Adding in changes to Transformers/Datasets for id transforms

    -------

    a newly constructed Dataset object

    """

    newx, newy, neww = transformer.transform_array(self._X, self._y, self._w)

    return NumpyDataset(newx, newy, neww, self._ids[:])

    newx, newy, neww, newids = transformer.transform_array(

        self._X, self._y, self._w, self._ids)

    return NumpyDataset(newx, newy, neww, newids)

  def select(self, indices: Sequence[int],

             select_dir: str = None) -> "NumpyDataset":

        for shard_num, row in self.metadata_df.iterrows():

          logger.info("Transforming shard %d/%d" % (shard_num, n_shards))

          X, y, w, ids = self.get_shard(shard_num)

          newx, newy, neww = transformer.transform_array(X, y, w)

          yield (newx, newy, neww, ids)

          newx, newy, neww, newids = transformer.transform_array(X, y, w, ids)

          yield (newx, newy, neww, newids)

      dataset = DiskDataset.create_dataset(

          generator(), data_dir=out_dir, tasks=tasks)

from deepchem.trans.transformers import ImageTransformer

from deepchem.trans.transformers import DataTransforms

from deepchem.trans.transformers import Transformer

from deepchem.trans.duplicate import DuplicateBalancingTransformer

import logging

from deepchem.trans.transformers import Transformer

import numpy as np

from typing import Tuple

from deepchem.data import Dataset

from deepchem.trans.transformers import Transformer

logger = logging.getLogger(__name__)

  """

  def __init__(self, dataset: Dataset):

    # BalancingTransformer can only transform weights.

    super(BalancingTransformer, self).__init__(

    super(DuplicateBalancingTransformer, self).__init__(

        transform_X=True,

        transform_y=True,

        transform_w=True,

      # this works because y is 1D

      num_c = len(np.where(y == c)[0])

      class_counts.append(num_c)

    N_largest = max(class_counts)

    # This is the right ratio since int(N/num_c) * num_c \approx N

    # for all classes

    duplication_ratio = [

        int(N_task / float(num_c)) if num_c > 0 else 0 for num_c in class_counts

        int(N_largest / float(num_c)) if num_c > 0 else 0

        for num_c in class_counts

    ]

    self.duplication_ratio = duplication_ratio

    """

    if not (len(y.shape) == 1 or (len(y.shape) == 2 and y[1] == 1)):

      raise ValueError("y must be of shape (N,) or (N, 1)")

    if not (len(w.shape) == 1 or (len(w.shape) == 2 and w[1] == 1)):

      raise ValueError("w must be of shape (N,) or (N, 1)")

    # Flattening is safe because of shape check above

    y = y.flatten()

    w = w.flatten()

    X_dups, y_dups, w_dups, ids_dups = [], [], [], []

    for i, c in enumerate(self.classes):

      duplication_ratio = self.duplication_ratio[i]

import os

import unittest

import numpy as np

import pandas as pd

import deepchem as dc

import tensorflow as tf

import scipy.ndimage

    data = np.reshape(data, (28, 28))

    self.d = data

  def test_clipping_X_transformer(self):

    """Test clipping transformer on X of singletask dataset."""

    n_samples = 10

    n_features = 3

    n_tasks = 1

    ids = np.arange(n_samples)

    X = np.ones((n_samples, n_features))

    target = 5. * X

    X *= 6.

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)

    transformer = dc.trans.ClippingTransformer(transform_X=True, x_max=5.)

    clipped_dataset = transformer.transform(dataset)

    X_t, y_t, w_t, ids_t = (clipped_dataset.X, clipped_dataset.y,

                            clipped_dataset.w, clipped_dataset.ids)

    # Check ids are unchanged.

    for id_elt, id_t_elt in zip(ids, ids_t):

      assert id_elt == id_t_elt

    # Check y is unchanged since this is an X transformer

    np.testing.assert_allclose(y, y_t)

    # Check w is unchanged since this is an X transformer

    np.testing.assert_allclose(w, w_t)

    # Check X is now holding the proper values when sorted.

    np.testing.assert_allclose(X_t, target)

  def test_clipping_y_transformer(self):

    """Test clipping transformer on y of singletask dataset."""

    n_samples = 10

    n_features = 3

    n_tasks = 1

    ids = np.arange(n_samples)

    X = np.zeros((n_samples, n_features))

    y = np.ones((n_samples, n_tasks))

    target = 5. * y

    y *= 6.

    w = np.ones((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)

    transformer = dc.trans.ClippingTransformer(transform_y=True, y_max=5.)

    clipped_dataset = transformer.transform(dataset)

    X_t, y_t, w_t, ids_t = (clipped_dataset.X, clipped_dataset.y,

                            clipped_dataset.w, clipped_dataset.ids)

    # Check ids are unchanged.

    for id_elt, id_t_elt in zip(ids, ids_t):

      assert id_elt == id_t_elt

    # Check X is unchanged since this is a y transformer

    np.testing.assert_allclose(X, X_t)

    # Check w is unchanged since this is a y transformer

    np.testing.assert_allclose(w, w_t)

    # Check y is now holding the proper values when sorted.

    np.testing.assert_allclose(y_t, target)

  def test_coulomb_fit_transformer(self):

    """Test coulomb fit transformer on singletask dataset."""

    n_samples = 10