Loading deepchem/trans/duplicate.py 0 → 100644 +163 −0 Original line number Diff line number Diff line import logging from deepchem.trans.transformers import Transformer from typing import Tuple logger = logging.getLogger(__name__) class DuplicateBalancingTransformer(Transformer): """Balance binary or multiclass datasets by duplicating rarer class samples. This class balances a dataset by duplicating samples of the rarer class so that the sum of all example weights from all classes is the same. (Up to integer rounding of course). This can be useful when you're working on an imabalanced dataset where there are far fewer examples of some classes than others. This class differs from `BalancingTransformer` in that it actually duplicates rarer class samples rather than just increasing their sample weights. This may be more friendly for models that are numerically fragile and can't handle imbalanced example weights. Examples -------- Here's an example for a binary dataset. >>> n_samples = 10 >>> n_features = 3 >>> n_tasks = 1 >>> n_classes = 2 >>> ids = np.arange(n_samples) >>> X = np.random.rand(n_samples, n_features) >>> y = np.random.randint(n_classes, size=(n_samples, n_tasks)) >>> w = np.ones((n_samples, n_tasks)) >>> dataset = dc.data.NumpyDataset(X, y, w, ids) >>> transformer = dc.trans.DuplicateBalancingTransformer(dataset=dataset) >>> dataset = transformer.transform(dataset) And here's a multiclass dataset example. >>> n_samples = 50 >>> n_features = 3 >>> n_tasks = 1 >>> n_classes = 5 >>> ids = np.arange(n_samples) >>> X = np.random.rand(n_samples, n_features) >>> y = np.random.randint(n_classes, size=(n_samples, n_tasks)) >>> w = np.ones((n_samples, n_tasks)) >>> dataset = dc.data.NumpyDataset(X, y, w, ids) >>> transformer = dc.trans.DuplicateBalancingTransformer(dataset=dataset) >>> dataset = transformer.transform(dataset) See Also -------- deepchem.trans.BalancingTransformer: Balance by changing sample weights. Note ---- This transformer is only well-defined for singletask datasets. (Since examples are actually duplicated, there's no meaningful way to duplicate across multiple tasks in a way that preserves the balance.) This transformer is only meaningful for classification datasets where `y` takes on a limited set of values. This class transforms all of `X`, `y`, `w`, `ids`. Raises ------ `ValueError` if the provided dataset is multitask. """ def __init__(self, dataset: Dataset): # BalancingTransformer can only transform weights. super(BalancingTransformer, self).__init__( transform_X=True, transform_y=True, transform_w=True, transform_ids=True, dataset=dataset) if len(dataset.get_task_names()) > 1: raise ValueError( "This transformation is only defined for singletask datsets.") # Get the labels/weights y = dataset.y w = dataset.w # Normalize shapes if len(y.shape) == 1: y = np.reshape(y, (len(y), 1)) if len(w.shape) == 1: w = np.reshape(w, (len(w), 1)) if len(y.shape) != 2: raise ValueError("y must be of shape (N,) or (N, n_tasks)") if len(w.shape) != 2: raise ValueError("w must be of shape (N,) or (N, n_tasks)") self.classes = sorted(np.unique(y)) # Remove labels with zero weights y = y[w != 0] N = len(y) class_counts = [] # Note that we may have 0 elements of a given class since we remove those # labels with zero weight. for c in self.classes: # this works because y is 1D num_c = len(np.where(y == c)[0]) class_counts.append(num_c) # This is the right ratio since int(N/num_c) * num_c \approx N # for all classes duplication_ratio = [ int(N_task / float(num_c)) if num_c > 0 else 0 for num_c in class_counts ] self.duplication_ratio = duplication_ratio def transform_array( self, X: np.ndarray, y: np.ndarray, w: np.ndarray, ids: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Transform the data in a set of (X, y, w, id) arrays. Parameters ---------- X: np.ndarray Array of features y: np.ndarray Array of labels w: np.ndarray Array of weights. ids: np.ndarray Array of identifiers Returns ------- Xtrans: np.ndarray Transformed array of features ytrans: np.ndarray Transformed array of labels wtrans: np.ndarray Transformed array of weights idtrans: np.ndarray Transformed array of identifiers """ if not (len(y.shape) == 1 or (len(y.shape) == 2 and y[1] == 1)): raise ValueError("y must be of shape (N,) or (N, 1)") X_dups, y_dups, w_dups, ids_dups = [], [], [], [] for i, c in enumerate(self.classes): duplication_ratio = self.duplication_ratio[i] c_inds = (y == c) X_c = X[c_inds] y_c = y[c_inds] w_c = w[c_inds] ids_c = ids[c_inds] X_c_dup = np.repeat(X_c, duplication_ratio, axis=0) y_c_dup = np.repeat(y_c, duplication_ratio, axis=0) w_c_dup = np.repeat(w_c, duplication_ratio, axis=0) ids_c_dup = np.repeat(ids_c, duplication_ratio, axis=0) X_dups.append(X_c_dup) y_dups.append(y_c_dup) w_dups.append(w_c_dup) ids_dups.append(ids_c_dup) Xtrans = np.concatenate(X_dups, axis=0) ytrans = np.concatenate(y_dups, axis=0) wtrans = np.concatenate(w_dups, axis=0) idstrans = np.concatenate(ids_dups, axis=0) return (Xtrans, ytrans, wtrans, idstrans) deepchem/trans/tests/test_balancing.py +0 −3 Original line number Diff line number Diff line import os import numpy as np import unittest import deepchem as dc import itertools import tempfile Loading Loading @@ -85,8 +84,6 @@ def test_binary_multitask(): multitask_dataset = dc.data.NumpyDataset(X, y, w) balancing_transformer = dc.trans.BalancingTransformer( dataset=multitask_dataset) #X, y, w, ids = (multitask_dataset.X, multitask_dataset.y, # multitask_dataset.w, multitask_dataset.ids) multitask_dataset = balancing_transformer.transform(multitask_dataset) X_t, y_t, w_t, ids_t = (multitask_dataset.X, multitask_dataset.y, multitask_dataset.w, multitask_dataset.ids) Loading deepchem/trans/tests/test_duplicate_balancing.py 0 → 100644 +129 −0 Original line number Diff line number Diff line import deepchem as dc def test_binary_1d(): """Test balancing transformer on single-task dataset without explicit task dimension.""" n_samples = 6 n_features = 3 n_classes = 2 np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.array([1, 1, 0, 0, 0, 0]) w = np.ones((n_samples,)) dataset = dc.data.NumpyDataset(X, y, w) duplicator = dc.trans.DuplicateBalancingTransformer(dataset=dataset) dataset = duplicator.transform(dataset) # Check that we have length 8 now with duplication assert len(dataset) == 8 X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check shapes assert X_t.shape == (8, n_features) assert y_t.shape == (8,) assert w_t.shape == (8,) assert ids_t.shape == (8,) # Check that we have 4 positives and 4 negatives assert np.sum(y_t == 0) == 4 assert np.sum(y_t == 1) == 4 # Check that sum of 0s equals sum of 1s in transformed for each task assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 1])) def test_binary_singletask(): """Test duplicate balancing transformer on single-task dataset.""" n_samples = 20 n_features = 3 n_tasks = 1 n_classes = 2 np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.reshape(np.array([1, 1, 0, 0, 0, 0]), (n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w) duplicator = dc.trans.DuplicateBalancingTransformer(dataset=dataset) dataset = duplicator.transform(dataset) X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check that we have length 8 now with duplication assert len(dataset) == 8 X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check shapes assert X_t.shape == (8, n_features) assert y_t.shape == (8,) assert w_t.shape == (8,) assert ids_t.shape == (8,) # Check that we have 4 positives and 4 negatives assert np.sum(y_t == 0) == 4 assert np.sum(y_t == 1) == 4 # Check that sum of 0s equals sum of 1s in transformed for each task assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 1])) def test_multiclass_singletask(): """Test balancing transformer on single-task dataset.""" n_samples = 10 n_features = 3 n_classes = 5 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) # 6-1 imbalance in favor of class 0 y = np.array([0, 0, 0, 0, 0, 0, 1, 2, 3, 4]) w = np.ones((n_samples,)) dataset = dc.data.NumpyDataset(X, y, w) duplicator = dc.trans.DuplicateBalancingTransformer(dataset=dataset) dataset = duplicator.transform(dataset) X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check that we have length 30 now with duplication assert len(dataset) == 30 X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check shapes assert X_t.shape == (30, n_features) assert y_t.shape == (30,) assert w_t.shape == (30,) assert ids_t.shape == (30,) # Check that we have 6 of each class assert np.sum(y_t == 0) == 6 assert np.sum(y_t == 1) == 6 assert np.sum(y_t == 2) == 6 assert np.sum(y_t == 3) == 6 assert np.sum(y_t == 4) == 6 # Check that sum of all class weights is equal by comparing to 0 weight assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 1])) assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 2])) assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 3])) assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 4])) def test_transform_to_directory(): """Test that output can be written to a directory.""" n_samples = 10 n_features = 3 n_classes = 2 np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) # Note class imbalance. This will round to 2x duplication for 1 y = np.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) w = np.ones((n_samples,)) dataset = dc.data.NumpyDataset(X, y, w) duplicator = dc.trans.DuplicateBalancingTransformer(dataset=dataset) with tempfile.TemporaryDirectory() as tmpdirname: dataset = duplicator.transform(dataset, out_dir=tmpdirname) balanced_dataset = dc.data.DiskDataset(tmpdirname) X_t, y_t, w_t, ids_t = (balanced_dataset.X, balanced_dataset.y, balanced_dataset.w, balanced_dataset.ids) # Check that we have length 13 now with duplication assert len(balanced_dataset) == 13 # Check shapes assert X_t.shape == (13, n_features) assert y_t.shape == (13,) assert w_t.shape == (13,) assert ids_t.shape == (13,) # Check that we have 6 positives and 7 negatives assert np.sum(y_t == 0) == 6 assert np.sum(y_t == 1) == 7 deepchem/trans/transformers.py +15 −14 Original line number Diff line number Diff line # coding=utf-8 """ Contains an abstract base class that supports data transformations. """ Loading Loading @@ -254,8 +253,8 @@ class MinMaxTransformer(Transformer): >>> A_max = np.max(A, axis=0) >>> A_t = np.nan_to_num((A - A_min)/(A_max - A_min)) Example ------- Examples -------- >>> n_samples = 10 >>> n_features = 3 Loading Loading @@ -398,8 +397,8 @@ class NormalizationTransformer(Transformer): This transformer transforms datasets to have zero mean and unit standard deviation. Example ------- Examples -------- >>> n_samples = 10 >>> n_features = 3 Loading Loading @@ -556,8 +555,8 @@ class NormalizationTransformer(Transformer): class ClippingTransformer(Transformer): """Clip large values in datasets. Example ------- Examples -------- >>> n_samples = 10 >>> n_features = 3 >>> n_tasks = 1 Loading Loading @@ -652,8 +651,8 @@ class LogTransformer(Transformer): Assuming that tasks/features are not specified. If specified, then transformations are only performed on specified tasks/features. Example ------- Examples -------- >>> n_samples = 10 >>> n_features = 3 >>> n_tasks = 1 Loading Loading @@ -781,15 +780,15 @@ class LogTransformer(Transformer): class BalancingTransformer(Transformer): """Balance positive and negative examples for weights. """Balance positive and negative (or multiclass) example weights. This class balances the sample weights so that the sum of all example weights from all classes is the same. This can be useful when you're working on an imbalanced dataset where there are far fewer examples of some classes than others. Example ------- Examples -------- Here's an example for a binary dataset. Loading Loading @@ -819,6 +818,9 @@ class BalancingTransformer(Transformer): >>> transformer = dc.trans.BalancingTransformer(dataset=dataset) >>> dataset = transformer.transform(dataset) See Also -------- deepchem.trans.DuplicateBalancingTransformer: Balance by duplicating samples. Note ---- This transformer is only meaningful for classification datasets where `y` Loading Loading @@ -848,7 +850,6 @@ class BalancingTransformer(Transformer): raise ValueError("y must be of shape (N,) or (N, n_tasks)") if len(w.shape) != 2: raise ValueError("w must be of shape (N,) or (N, n_tasks)") # Ensure dataset is binary self.classes = sorted(np.unique(y)) weights = [] for ind, task in enumerate(dataset.get_task_names()): Loading @@ -858,7 +859,7 @@ class BalancingTransformer(Transformer): task_y = task_y[task_w != 0] N_task = len(task_y) class_counts = [] # Note that we may 0 elements of a given class since we remove those # Note that we may have 0 elements of a given class since we remove those # labels with zero weight. This typically happens in multitask datasets # where some datapoints only have labels for some tasks. for c in self.classes: Loading Loading
deepchem/trans/duplicate.py 0 → 100644 +163 −0 Original line number Diff line number Diff line import logging from deepchem.trans.transformers import Transformer from typing import Tuple logger = logging.getLogger(__name__) class DuplicateBalancingTransformer(Transformer): """Balance binary or multiclass datasets by duplicating rarer class samples. This class balances a dataset by duplicating samples of the rarer class so that the sum of all example weights from all classes is the same. (Up to integer rounding of course). This can be useful when you're working on an imabalanced dataset where there are far fewer examples of some classes than others. This class differs from `BalancingTransformer` in that it actually duplicates rarer class samples rather than just increasing their sample weights. This may be more friendly for models that are numerically fragile and can't handle imbalanced example weights. Examples -------- Here's an example for a binary dataset. >>> n_samples = 10 >>> n_features = 3 >>> n_tasks = 1 >>> n_classes = 2 >>> ids = np.arange(n_samples) >>> X = np.random.rand(n_samples, n_features) >>> y = np.random.randint(n_classes, size=(n_samples, n_tasks)) >>> w = np.ones((n_samples, n_tasks)) >>> dataset = dc.data.NumpyDataset(X, y, w, ids) >>> transformer = dc.trans.DuplicateBalancingTransformer(dataset=dataset) >>> dataset = transformer.transform(dataset) And here's a multiclass dataset example. >>> n_samples = 50 >>> n_features = 3 >>> n_tasks = 1 >>> n_classes = 5 >>> ids = np.arange(n_samples) >>> X = np.random.rand(n_samples, n_features) >>> y = np.random.randint(n_classes, size=(n_samples, n_tasks)) >>> w = np.ones((n_samples, n_tasks)) >>> dataset = dc.data.NumpyDataset(X, y, w, ids) >>> transformer = dc.trans.DuplicateBalancingTransformer(dataset=dataset) >>> dataset = transformer.transform(dataset) See Also -------- deepchem.trans.BalancingTransformer: Balance by changing sample weights. Note ---- This transformer is only well-defined for singletask datasets. (Since examples are actually duplicated, there's no meaningful way to duplicate across multiple tasks in a way that preserves the balance.) This transformer is only meaningful for classification datasets where `y` takes on a limited set of values. This class transforms all of `X`, `y`, `w`, `ids`. Raises ------ `ValueError` if the provided dataset is multitask. """ def __init__(self, dataset: Dataset): # BalancingTransformer can only transform weights. super(BalancingTransformer, self).__init__( transform_X=True, transform_y=True, transform_w=True, transform_ids=True, dataset=dataset) if len(dataset.get_task_names()) > 1: raise ValueError( "This transformation is only defined for singletask datsets.") # Get the labels/weights y = dataset.y w = dataset.w # Normalize shapes if len(y.shape) == 1: y = np.reshape(y, (len(y), 1)) if len(w.shape) == 1: w = np.reshape(w, (len(w), 1)) if len(y.shape) != 2: raise ValueError("y must be of shape (N,) or (N, n_tasks)") if len(w.shape) != 2: raise ValueError("w must be of shape (N,) or (N, n_tasks)") self.classes = sorted(np.unique(y)) # Remove labels with zero weights y = y[w != 0] N = len(y) class_counts = [] # Note that we may have 0 elements of a given class since we remove those # labels with zero weight. for c in self.classes: # this works because y is 1D num_c = len(np.where(y == c)[0]) class_counts.append(num_c) # This is the right ratio since int(N/num_c) * num_c \approx N # for all classes duplication_ratio = [ int(N_task / float(num_c)) if num_c > 0 else 0 for num_c in class_counts ] self.duplication_ratio = duplication_ratio def transform_array( self, X: np.ndarray, y: np.ndarray, w: np.ndarray, ids: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Transform the data in a set of (X, y, w, id) arrays. Parameters ---------- X: np.ndarray Array of features y: np.ndarray Array of labels w: np.ndarray Array of weights. ids: np.ndarray Array of identifiers Returns ------- Xtrans: np.ndarray Transformed array of features ytrans: np.ndarray Transformed array of labels wtrans: np.ndarray Transformed array of weights idtrans: np.ndarray Transformed array of identifiers """ if not (len(y.shape) == 1 or (len(y.shape) == 2 and y[1] == 1)): raise ValueError("y must be of shape (N,) or (N, 1)") X_dups, y_dups, w_dups, ids_dups = [], [], [], [] for i, c in enumerate(self.classes): duplication_ratio = self.duplication_ratio[i] c_inds = (y == c) X_c = X[c_inds] y_c = y[c_inds] w_c = w[c_inds] ids_c = ids[c_inds] X_c_dup = np.repeat(X_c, duplication_ratio, axis=0) y_c_dup = np.repeat(y_c, duplication_ratio, axis=0) w_c_dup = np.repeat(w_c, duplication_ratio, axis=0) ids_c_dup = np.repeat(ids_c, duplication_ratio, axis=0) X_dups.append(X_c_dup) y_dups.append(y_c_dup) w_dups.append(w_c_dup) ids_dups.append(ids_c_dup) Xtrans = np.concatenate(X_dups, axis=0) ytrans = np.concatenate(y_dups, axis=0) wtrans = np.concatenate(w_dups, axis=0) idstrans = np.concatenate(ids_dups, axis=0) return (Xtrans, ytrans, wtrans, idstrans)
deepchem/trans/tests/test_balancing.py +0 −3 Original line number Diff line number Diff line import os import numpy as np import unittest import deepchem as dc import itertools import tempfile Loading Loading @@ -85,8 +84,6 @@ def test_binary_multitask(): multitask_dataset = dc.data.NumpyDataset(X, y, w) balancing_transformer = dc.trans.BalancingTransformer( dataset=multitask_dataset) #X, y, w, ids = (multitask_dataset.X, multitask_dataset.y, # multitask_dataset.w, multitask_dataset.ids) multitask_dataset = balancing_transformer.transform(multitask_dataset) X_t, y_t, w_t, ids_t = (multitask_dataset.X, multitask_dataset.y, multitask_dataset.w, multitask_dataset.ids) Loading
deepchem/trans/tests/test_duplicate_balancing.py 0 → 100644 +129 −0 Original line number Diff line number Diff line import deepchem as dc def test_binary_1d(): """Test balancing transformer on single-task dataset without explicit task dimension.""" n_samples = 6 n_features = 3 n_classes = 2 np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.array([1, 1, 0, 0, 0, 0]) w = np.ones((n_samples,)) dataset = dc.data.NumpyDataset(X, y, w) duplicator = dc.trans.DuplicateBalancingTransformer(dataset=dataset) dataset = duplicator.transform(dataset) # Check that we have length 8 now with duplication assert len(dataset) == 8 X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check shapes assert X_t.shape == (8, n_features) assert y_t.shape == (8,) assert w_t.shape == (8,) assert ids_t.shape == (8,) # Check that we have 4 positives and 4 negatives assert np.sum(y_t == 0) == 4 assert np.sum(y_t == 1) == 4 # Check that sum of 0s equals sum of 1s in transformed for each task assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 1])) def test_binary_singletask(): """Test duplicate balancing transformer on single-task dataset.""" n_samples = 20 n_features = 3 n_tasks = 1 n_classes = 2 np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.reshape(np.array([1, 1, 0, 0, 0, 0]), (n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w) duplicator = dc.trans.DuplicateBalancingTransformer(dataset=dataset) dataset = duplicator.transform(dataset) X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check that we have length 8 now with duplication assert len(dataset) == 8 X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check shapes assert X_t.shape == (8, n_features) assert y_t.shape == (8,) assert w_t.shape == (8,) assert ids_t.shape == (8,) # Check that we have 4 positives and 4 negatives assert np.sum(y_t == 0) == 4 assert np.sum(y_t == 1) == 4 # Check that sum of 0s equals sum of 1s in transformed for each task assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 1])) def test_multiclass_singletask(): """Test balancing transformer on single-task dataset.""" n_samples = 10 n_features = 3 n_classes = 5 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) # 6-1 imbalance in favor of class 0 y = np.array([0, 0, 0, 0, 0, 0, 1, 2, 3, 4]) w = np.ones((n_samples,)) dataset = dc.data.NumpyDataset(X, y, w) duplicator = dc.trans.DuplicateBalancingTransformer(dataset=dataset) dataset = duplicator.transform(dataset) X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check that we have length 30 now with duplication assert len(dataset) == 30 X_t, y_t, w_t, ids_t = (dataset.X, dataset.y, dataset.w, dataset.ids) # Check shapes assert X_t.shape == (30, n_features) assert y_t.shape == (30,) assert w_t.shape == (30,) assert ids_t.shape == (30,) # Check that we have 6 of each class assert np.sum(y_t == 0) == 6 assert np.sum(y_t == 1) == 6 assert np.sum(y_t == 2) == 6 assert np.sum(y_t == 3) == 6 assert np.sum(y_t == 4) == 6 # Check that sum of all class weights is equal by comparing to 0 weight assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 1])) assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 2])) assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 3])) assert np.isclose(np.sum(w_task[y_task == 0]), np.sum(w_task[y_task == 4])) def test_transform_to_directory(): """Test that output can be written to a directory.""" n_samples = 10 n_features = 3 n_classes = 2 np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) # Note class imbalance. This will round to 2x duplication for 1 y = np.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) w = np.ones((n_samples,)) dataset = dc.data.NumpyDataset(X, y, w) duplicator = dc.trans.DuplicateBalancingTransformer(dataset=dataset) with tempfile.TemporaryDirectory() as tmpdirname: dataset = duplicator.transform(dataset, out_dir=tmpdirname) balanced_dataset = dc.data.DiskDataset(tmpdirname) X_t, y_t, w_t, ids_t = (balanced_dataset.X, balanced_dataset.y, balanced_dataset.w, balanced_dataset.ids) # Check that we have length 13 now with duplication assert len(balanced_dataset) == 13 # Check shapes assert X_t.shape == (13, n_features) assert y_t.shape == (13,) assert w_t.shape == (13,) assert ids_t.shape == (13,) # Check that we have 6 positives and 7 negatives assert np.sum(y_t == 0) == 6 assert np.sum(y_t == 1) == 7
deepchem/trans/transformers.py +15 −14 Original line number Diff line number Diff line # coding=utf-8 """ Contains an abstract base class that supports data transformations. """ Loading Loading @@ -254,8 +253,8 @@ class MinMaxTransformer(Transformer): >>> A_max = np.max(A, axis=0) >>> A_t = np.nan_to_num((A - A_min)/(A_max - A_min)) Example ------- Examples -------- >>> n_samples = 10 >>> n_features = 3 Loading Loading @@ -398,8 +397,8 @@ class NormalizationTransformer(Transformer): This transformer transforms datasets to have zero mean and unit standard deviation. Example ------- Examples -------- >>> n_samples = 10 >>> n_features = 3 Loading Loading @@ -556,8 +555,8 @@ class NormalizationTransformer(Transformer): class ClippingTransformer(Transformer): """Clip large values in datasets. Example ------- Examples -------- >>> n_samples = 10 >>> n_features = 3 >>> n_tasks = 1 Loading Loading @@ -652,8 +651,8 @@ class LogTransformer(Transformer): Assuming that tasks/features are not specified. If specified, then transformations are only performed on specified tasks/features. Example ------- Examples -------- >>> n_samples = 10 >>> n_features = 3 >>> n_tasks = 1 Loading Loading @@ -781,15 +780,15 @@ class LogTransformer(Transformer): class BalancingTransformer(Transformer): """Balance positive and negative examples for weights. """Balance positive and negative (or multiclass) example weights. This class balances the sample weights so that the sum of all example weights from all classes is the same. This can be useful when you're working on an imbalanced dataset where there are far fewer examples of some classes than others. Example ------- Examples -------- Here's an example for a binary dataset. Loading Loading @@ -819,6 +818,9 @@ class BalancingTransformer(Transformer): >>> transformer = dc.trans.BalancingTransformer(dataset=dataset) >>> dataset = transformer.transform(dataset) See Also -------- deepchem.trans.DuplicateBalancingTransformer: Balance by duplicating samples. Note ---- This transformer is only meaningful for classification datasets where `y` Loading Loading @@ -848,7 +850,6 @@ class BalancingTransformer(Transformer): raise ValueError("y must be of shape (N,) or (N, n_tasks)") if len(w.shape) != 2: raise ValueError("w must be of shape (N,) or (N, n_tasks)") # Ensure dataset is binary self.classes = sorted(np.unique(y)) weights = [] for ind, task in enumerate(dataset.get_task_names()): Loading @@ -858,7 +859,7 @@ class BalancingTransformer(Transformer): task_y = task_y[task_w != 0] N_task = len(task_y) class_counts = [] # Note that we may 0 elements of a given class since we remove those # Note that we may have 0 elements of a given class since we remove those # labels with zero weight. This typically happens in multitask datasets # where some datapoints only have labels for some tasks. for c in self.classes: Loading
