Loading deepchem/datasets/__init__.py +34 −2 Original line number Diff line number Diff line Loading @@ -148,9 +148,13 @@ class Dataset(object): """ return self.metadata_df.shape[0] def _itershards(self): def itershards(self): """ Iterates over all shards in dataset. Datasets are stored in sharded fashion on disk. Each call to next() for the generator defined by this function returns the data from a particular shard. The order of shards returned is guaranteed to remain fixed. """ for _, row in self.metadata_df.iterrows(): X = load_from_disk(row['X-transformed']) Loading @@ -165,7 +169,7 @@ class Dataset(object): """ if batch_size == None: batch_size = len(self) for i, (X, y, w, ids) in enumerate(self._itershards()): for i, (X, y, w, ids) in enumerate(self.itershards()): log("Iterating on shard-%s/epoch-%s" % (str(i+1), str(epoch+1)), self.verbosity) nb_sample = np.shape(X)[0] Loading Loading @@ -200,6 +204,15 @@ class Dataset(object): return (np.vstack(Xs), np.vstack(ys), np.vstack(ws), np.squeeze(np.vstack(ids))) def get_labels(self): """ Returns all labels for this dataset. """ ys = [] for (_, y_b, _, _) in self.itershards(): ys.append(y_b) return np.vstack(ys) def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size): """Fix batch to have exactly batch_size elements. Loading Loading @@ -251,6 +264,25 @@ class Dataset(object): df = self.metadata_df update_mean_and_std(df) def balance_positives_and_negatives(self): """For binary datasets, balance pos and neg examples.""" labels = self.get_labels() # Ensure dataset is binary np.testing.assert_allclose(sorted(np.unique(labels)), np.array([0., 1.])) weights = [] # TODO(rbharath): This doesn't deal with zeroed out labels. for ind, task in enumerate(self.get_task_names()): task_labels = labels[:, ind] num_positives = np.count_nonzero(task_labels) num_negatives = len(task_labels) - num_positives if num_positives > 0: pos_weight = float(num_negatives)/num_positives else: pos_weight = 1 neg_weight = 1 weights.append((pos_weight, neg_weight)) return weights def compute_sums_and_nb_sample(tensor, W=None): """ Loading deepchem/datasets/tests/__init__.py +98 −0 Original line number Diff line number Diff line """ General API for testing dataset objects """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals __author__ = "Bharath Ramsundar" __copyright__ = "Copyright 2016, Stanford University" __license__ = "LGPL" import unittest import tempfile import os import shutil import numpy as np from deepchem.datasets import Dataset from deepchem.featurizers.featurize import DataFeaturizer from deepchem.featurizers.fingerprints import CircularFingerprint from deepchem.transformers import NormalizationTransformer class TestDatasetAPI(unittest.TestCase): """ Shared API for testing with dataset objects. """ def setUp(self): self.current_dir = os.path.dirname(os.path.abspath(__file__)) self.test_data_dir = os.path.join(self.current_dir, "../../models/test") self.smiles_field = "smiles" self.feature_dir = tempfile.mkdtemp() self.samples_dir = tempfile.mkdtemp() self.data_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.feature_dir) shutil.rmtree(self.samples_dir) shutil.rmtree(self.data_dir) # TODO(rbharath): There should be a more natural way to create a dataset # object, perhaps just starting from (Xs, ys, ws) def _create_dataset(self, compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, tasks, protein_pdb_field=None, ligand_pdb_field=None, user_specified_features=None, split_field=None, shard_size=100): # Featurize input featurizers = compound_featurizers + complex_featurizers input_file = os.path.join(self.test_data_dir, input_file) featurizer = DataFeaturizer(tasks=tasks, smiles_field=self.smiles_field, protein_pdb_field=protein_pdb_field, ligand_pdb_field=ligand_pdb_field, compound_featurizers=compound_featurizers, complex_featurizers=complex_featurizers, user_specified_features=user_specified_features, split_field=split_field, verbosity="low") samples = featurizer.featurize(input_file, self.feature_dir, self.samples_dir, shard_size=shard_size) use_user_specified_features = (user_specified_features is not None) dataset = Dataset(data_dir=self.data_dir, samples=samples, featurizers=featurizers, tasks=tasks, use_user_specified_features=use_user_specified_features) return dataset def _load_solubility_data(self): """Loads solubility data from example.csv""" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] input_transformer_classes = [] output_transformer_classes = [] task_types = {"log-solubility": "regression"} input_file = "example.csv" return self._create_dataset( compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, task_types.keys()) def _load_multitask_data(self): """Load example multitask data.""" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] output_transformer_classes = [] input_transformer_classes = [] tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] task_types = {task: "classification" for task in tasks} input_file = "multitask_example.csv" return self._create_dataset( compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, task_types.keys()) deepchem/datasets/tests/test_datasets.py +25 −59 Original line number Diff line number Diff line Loading @@ -18,78 +18,32 @@ from deepchem.datasets import Dataset from deepchem.featurizers.featurize import DataFeaturizer from deepchem.featurizers.fingerprints import CircularFingerprint from deepchem.transformers import NormalizationTransformer from deepchem.datasets.tests import TestDatasetAPI class TestAPI(unittest.TestCase): class TestBasicDatasetAPI(TestDatasetAPI): """ Test top-level API for ML models." Test basic top-level API for dataset objects. """ def setUp(self): self.current_dir = os.path.dirname(os.path.abspath(__file__)) self.test_data_dir = os.path.join(self.current_dir, "../../models/test") self.smiles_field = "smiles" self.feature_dir = tempfile.mkdtemp() self.samples_dir = tempfile.mkdtemp() self.data_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.feature_dir) shutil.rmtree(self.samples_dir) shutil.rmtree(self.data_dir) # TODO(rbharath): There should be a more natural way to create a dataset # object, perhaps just starting from (Xs, ys, ws) def _create_dataset(self, compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, tasks, protein_pdb_field=None, ligand_pdb_field=None, user_specified_features=None, split_field=None, shard_size=100): # Featurize input featurizers = compound_featurizers + complex_featurizers input_file = os.path.join(self.test_data_dir, input_file) featurizer = DataFeaturizer(tasks=tasks, smiles_field=self.smiles_field, protein_pdb_field=protein_pdb_field, ligand_pdb_field=ligand_pdb_field, compound_featurizers=compound_featurizers, complex_featurizers=complex_featurizers, user_specified_features=user_specified_features, split_field=split_field, verbosity="low") samples = featurizer.featurize(input_file, self.feature_dir, self.samples_dir, shard_size=shard_size) use_user_specified_features = (user_specified_features is not None) dataset = Dataset(data_dir=self.data_dir, samples=samples, featurizers=featurizers, tasks=tasks, use_user_specified_features=use_user_specified_features) return dataset def _load_solubility_data(self): """Loads solubility data from example.csv""" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] input_transformer_classes = [] output_transformer_classes = [NormalizationTransformer] task_types = {"log-solubility": "regression"} input_file = "example.csv" return self._create_dataset( compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, task_types.keys()) def test_get_task_names(self): """Test that get_task_names returns correct task_names""" solubility_dataset = self._load_solubility_data() assert solubility_dataset.get_task_names() == ["log-solubility"] multitask_dataset = self._load_multitask_data() assert sorted(multitask_dataset.get_task_names()) == sorted(["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"]) def test_get_data_shape(self): """Test that get_data_shape returns currect data shape""" solubility_dataset = self._load_solubility_data() assert solubility_dataset.get_data_shape() == (1024,) multitask_dataset = self._load_multitask_data() assert multitask_dataset.get_data_shape() == (1024,) def test_len(self): """Test that len(dataset) works.""" solubility_dataset = self._load_solubility_data() Loading Loading @@ -133,3 +87,15 @@ class TestAPI(unittest.TestCase): np.testing.assert_allclose(comp_y_means, y_means) np.testing.assert_allclose(comp_X_stds, X_stds) np.testing.assert_allclose(comp_y_stds, y_stds) def test_balance_positives_and_negatives(self): """Test balancing of positive and negative examples.""" multitask_dataset = self._load_multitask_data() weights = multitask_dataset.balance_positives_and_negatives() X, y, w, ids = multitask_dataset.to_numpy() for ind, task in enumerate(multitask_dataset.get_task_names()): task_labels = y[:, ind] num_positives = np.count_nonzero(task_labels) num_negatives = len(task_labels) - num_positives pos_weight, neg_weight = weights[ind] assert np.isclose(num_positives * pos_weight, num_negatives * neg_weight) deepchem/models/test/example.csv +10 −10 Original line number Diff line number Diff line Compound ID,log-solubility,smiles Amigdalin,-0.9740000000000001,OCC3OC(OCC2OC(OC(C#N)c1ccccc1)C(O)C(O)C2O)C(O)C(O)C3O Fenfuram,-2.885,Cc1occc1C(=O)Nc2ccccc2 citral,-2.5789999999999997,CC(C)=CCCC(C)=CC(=O) Picene,-6.617999999999999,c1ccc2c(c1)ccc3c2ccc4c5ccccc5ccc43 Thiophene,-2.2319999999999998,c1ccsc1 benzothiazole,-2.733,c2ccc1scnc1c2 "2,2,4,6,6'-PCB",-6.545,Clc1cc(Cl)c(c(Cl)c1)c2c(Cl)cccc2Cl Estradiol,-4.138,CC12CCC3C(CCc4cc(O)ccc34)C2CCC1O Dieldrin,-4.533,ClC4=C(Cl)C5(Cl)C3C1CC(C2OC12)C3C4(Cl)C5(Cl)Cl Rotenone,-5.246,COc5cc4OCC3Oc2c1CC(Oc1ccc2C(=O)C3c4cc5OC)C(C)=C Amigdalin,0.9740000000000001,OCC3OC(OCC2OC(OC(C#N)c1ccccc1)C(O)C(O)C2O)C(O)C(O)C3O Fenfuram,2.885,Cc1occc1C(=O)Nc2ccccc2 citral,2.5789999999999997,CC(C)=CCCC(C)=CC(=O) Picene,6.617999999999999,c1ccc2c(c1)ccc3c2ccc4c5ccccc5ccc43 Thiophene,2.2319999999999998,c1ccsc1 benzothiazole,2.733,c2ccc1scnc1c2 "2,2,4,6,6'-PCB",6.545,Clc1cc(Cl)c(c(Cl)c1)c2c(Cl)cccc2Cl Estradiol,4.138,CC12CCC3C(CCc4cc(O)ccc34)C2CCC1O Dieldrin,4.533,ClC4=C(Cl)C5(Cl)C3C1CC(C2OC12)C3C4(Cl)C5(Cl)Cl Rotenone,5.246,COc5cc4OCC3Oc2c1CC(Oc1ccc2C(=O)C3c4cc5OC)C(C)=C deepchem/transformers/__init__.py +18 −17 Original line number Diff line number Diff line Loading @@ -52,7 +52,8 @@ class Transformer(object): """ df = dataset.metadata_df indices = range(0, df.shape[0]) transform_row_partial = partial(_transform_row, df=df, transformer=self) transform_row_partial = partial( _transform_row, df=df, transformer=self) if parallel: pool = mp.Pool(int(mp.cpu_count()/4)) pool.map(transform_row_partial, indices) Loading @@ -74,9 +75,8 @@ class NormalizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None): """Initialize normalization transformation.""" super(NormalizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) super(NormalizationTransformer, self).__init__( transform_X=transform_X, transform_y=transform_y, dataset=dataset) X_means, X_stds, y_means, y_stds = dataset.get_statistics() self.X_means = X_means self.X_stds = X_stds Loading @@ -89,7 +89,8 @@ class NormalizationTransformer(Transformer): self.X_stds = X_stds self.y_means = y_means self.y_stds = y_stds super(NormalizationTransformer, self).transform(dataset, parallel=parallel) super(NormalizationTransformer, self).transform( dataset, parallel=parallel) def transform_row(self, i, df): Loading Loading @@ -150,7 +151,7 @@ class ClippingTransformer(Transformer): class LogTransformer(Transformer): def transform_row(i, df): def transform_row(self, i, df): """Logarithmically transforms data in dataset.""" row = df.iloc[i] if self.transform_X: Loading @@ -172,14 +173,13 @@ class CoulombRandomizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None, seed=None): """Iniitialize coulomb matrix randomization transformation. """ super(CoulombRandomizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) super(CoulombRandomizationTransformer, self).__init__( transform_X=transform_X, transform_y=transform_y, dataset=dataset) self.seed = seed def construct_cm_from_triu(self, x): """ Constructs the unpadded coulomb matrix from the upper triangular portion. Constructs unpadded coulomb matrix from upper triangular portion. """ d = int((np.sqrt(8*len(x)+1)-1)/2) cm = np.zeros([d,d]) Loading @@ -206,7 +206,8 @@ class CoulombRandomizationTransformer(Transformer): upcm = cm[0:atom_number,0:atom_number] row_norms = np.asarray([np.linalg.norm(row) for row in upcm], dtype=float) row_norms = np.asarray( [np.linalg.norm(row) for row in upcm], dtype=float) rng = np.random.RandomState(self.seed) e = rng.normal(size=row_norms.size) p = np.argsort(row_norms+e) Loading @@ -229,7 +230,8 @@ class CoulombRandomizationTransformer(Transformer): save_to_disk(X, row['X-transformed']) if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") print("y will not be transformed by " "CoulombRandomizationTransformer.") def untransform(self, z): print("Cannot undo CoulombRandomizationTransformer.") Loading @@ -239,9 +241,8 @@ class CoulombBinarizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None, theta=1, update_state=True): """Initialize binarization transformation.""" super(CoulombBinarizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) super(CoulombBinarizationTransformer, self).__init__( transform_X=transform_X, transform_y=transform_y, dataset=dataset) self.theta = theta self.feature_max = np.zeros(dataset.get_data_shape()) self.update_state = update_state Loading Loading @@ -292,8 +293,8 @@ class CoulombBinarizationTransformer(Transformer): save_to_disk(X_bin, row['X-transformed']) if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") print("y will not be transformed by " "CoulombBinarizationTransformer.") def untranform(self, z): print("Cannot undo CoulombBinarizationTransformer.") Loading
deepchem/datasets/__init__.py +34 −2 Original line number Diff line number Diff line Loading @@ -148,9 +148,13 @@ class Dataset(object): """ return self.metadata_df.shape[0] def _itershards(self): def itershards(self): """ Iterates over all shards in dataset. Datasets are stored in sharded fashion on disk. Each call to next() for the generator defined by this function returns the data from a particular shard. The order of shards returned is guaranteed to remain fixed. """ for _, row in self.metadata_df.iterrows(): X = load_from_disk(row['X-transformed']) Loading @@ -165,7 +169,7 @@ class Dataset(object): """ if batch_size == None: batch_size = len(self) for i, (X, y, w, ids) in enumerate(self._itershards()): for i, (X, y, w, ids) in enumerate(self.itershards()): log("Iterating on shard-%s/epoch-%s" % (str(i+1), str(epoch+1)), self.verbosity) nb_sample = np.shape(X)[0] Loading Loading @@ -200,6 +204,15 @@ class Dataset(object): return (np.vstack(Xs), np.vstack(ys), np.vstack(ws), np.squeeze(np.vstack(ids))) def get_labels(self): """ Returns all labels for this dataset. """ ys = [] for (_, y_b, _, _) in self.itershards(): ys.append(y_b) return np.vstack(ys) def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size): """Fix batch to have exactly batch_size elements. Loading Loading @@ -251,6 +264,25 @@ class Dataset(object): df = self.metadata_df update_mean_and_std(df) def balance_positives_and_negatives(self): """For binary datasets, balance pos and neg examples.""" labels = self.get_labels() # Ensure dataset is binary np.testing.assert_allclose(sorted(np.unique(labels)), np.array([0., 1.])) weights = [] # TODO(rbharath): This doesn't deal with zeroed out labels. for ind, task in enumerate(self.get_task_names()): task_labels = labels[:, ind] num_positives = np.count_nonzero(task_labels) num_negatives = len(task_labels) - num_positives if num_positives > 0: pos_weight = float(num_negatives)/num_positives else: pos_weight = 1 neg_weight = 1 weights.append((pos_weight, neg_weight)) return weights def compute_sums_and_nb_sample(tensor, W=None): """ Loading
deepchem/datasets/tests/__init__.py +98 −0 Original line number Diff line number Diff line """ General API for testing dataset objects """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals __author__ = "Bharath Ramsundar" __copyright__ = "Copyright 2016, Stanford University" __license__ = "LGPL" import unittest import tempfile import os import shutil import numpy as np from deepchem.datasets import Dataset from deepchem.featurizers.featurize import DataFeaturizer from deepchem.featurizers.fingerprints import CircularFingerprint from deepchem.transformers import NormalizationTransformer class TestDatasetAPI(unittest.TestCase): """ Shared API for testing with dataset objects. """ def setUp(self): self.current_dir = os.path.dirname(os.path.abspath(__file__)) self.test_data_dir = os.path.join(self.current_dir, "../../models/test") self.smiles_field = "smiles" self.feature_dir = tempfile.mkdtemp() self.samples_dir = tempfile.mkdtemp() self.data_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.feature_dir) shutil.rmtree(self.samples_dir) shutil.rmtree(self.data_dir) # TODO(rbharath): There should be a more natural way to create a dataset # object, perhaps just starting from (Xs, ys, ws) def _create_dataset(self, compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, tasks, protein_pdb_field=None, ligand_pdb_field=None, user_specified_features=None, split_field=None, shard_size=100): # Featurize input featurizers = compound_featurizers + complex_featurizers input_file = os.path.join(self.test_data_dir, input_file) featurizer = DataFeaturizer(tasks=tasks, smiles_field=self.smiles_field, protein_pdb_field=protein_pdb_field, ligand_pdb_field=ligand_pdb_field, compound_featurizers=compound_featurizers, complex_featurizers=complex_featurizers, user_specified_features=user_specified_features, split_field=split_field, verbosity="low") samples = featurizer.featurize(input_file, self.feature_dir, self.samples_dir, shard_size=shard_size) use_user_specified_features = (user_specified_features is not None) dataset = Dataset(data_dir=self.data_dir, samples=samples, featurizers=featurizers, tasks=tasks, use_user_specified_features=use_user_specified_features) return dataset def _load_solubility_data(self): """Loads solubility data from example.csv""" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] input_transformer_classes = [] output_transformer_classes = [] task_types = {"log-solubility": "regression"} input_file = "example.csv" return self._create_dataset( compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, task_types.keys()) def _load_multitask_data(self): """Load example multitask data.""" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] output_transformer_classes = [] input_transformer_classes = [] tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] task_types = {task: "classification" for task in tasks} input_file = "multitask_example.csv" return self._create_dataset( compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, task_types.keys())
deepchem/datasets/tests/test_datasets.py +25 −59 Original line number Diff line number Diff line Loading @@ -18,78 +18,32 @@ from deepchem.datasets import Dataset from deepchem.featurizers.featurize import DataFeaturizer from deepchem.featurizers.fingerprints import CircularFingerprint from deepchem.transformers import NormalizationTransformer from deepchem.datasets.tests import TestDatasetAPI class TestAPI(unittest.TestCase): class TestBasicDatasetAPI(TestDatasetAPI): """ Test top-level API for ML models." Test basic top-level API for dataset objects. """ def setUp(self): self.current_dir = os.path.dirname(os.path.abspath(__file__)) self.test_data_dir = os.path.join(self.current_dir, "../../models/test") self.smiles_field = "smiles" self.feature_dir = tempfile.mkdtemp() self.samples_dir = tempfile.mkdtemp() self.data_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.feature_dir) shutil.rmtree(self.samples_dir) shutil.rmtree(self.data_dir) # TODO(rbharath): There should be a more natural way to create a dataset # object, perhaps just starting from (Xs, ys, ws) def _create_dataset(self, compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, tasks, protein_pdb_field=None, ligand_pdb_field=None, user_specified_features=None, split_field=None, shard_size=100): # Featurize input featurizers = compound_featurizers + complex_featurizers input_file = os.path.join(self.test_data_dir, input_file) featurizer = DataFeaturizer(tasks=tasks, smiles_field=self.smiles_field, protein_pdb_field=protein_pdb_field, ligand_pdb_field=ligand_pdb_field, compound_featurizers=compound_featurizers, complex_featurizers=complex_featurizers, user_specified_features=user_specified_features, split_field=split_field, verbosity="low") samples = featurizer.featurize(input_file, self.feature_dir, self.samples_dir, shard_size=shard_size) use_user_specified_features = (user_specified_features is not None) dataset = Dataset(data_dir=self.data_dir, samples=samples, featurizers=featurizers, tasks=tasks, use_user_specified_features=use_user_specified_features) return dataset def _load_solubility_data(self): """Loads solubility data from example.csv""" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] input_transformer_classes = [] output_transformer_classes = [NormalizationTransformer] task_types = {"log-solubility": "regression"} input_file = "example.csv" return self._create_dataset( compound_featurizers, complex_featurizers, input_transformer_classes, output_transformer_classes, input_file, task_types.keys()) def test_get_task_names(self): """Test that get_task_names returns correct task_names""" solubility_dataset = self._load_solubility_data() assert solubility_dataset.get_task_names() == ["log-solubility"] multitask_dataset = self._load_multitask_data() assert sorted(multitask_dataset.get_task_names()) == sorted(["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"]) def test_get_data_shape(self): """Test that get_data_shape returns currect data shape""" solubility_dataset = self._load_solubility_data() assert solubility_dataset.get_data_shape() == (1024,) multitask_dataset = self._load_multitask_data() assert multitask_dataset.get_data_shape() == (1024,) def test_len(self): """Test that len(dataset) works.""" solubility_dataset = self._load_solubility_data() Loading Loading @@ -133,3 +87,15 @@ class TestAPI(unittest.TestCase): np.testing.assert_allclose(comp_y_means, y_means) np.testing.assert_allclose(comp_X_stds, X_stds) np.testing.assert_allclose(comp_y_stds, y_stds) def test_balance_positives_and_negatives(self): """Test balancing of positive and negative examples.""" multitask_dataset = self._load_multitask_data() weights = multitask_dataset.balance_positives_and_negatives() X, y, w, ids = multitask_dataset.to_numpy() for ind, task in enumerate(multitask_dataset.get_task_names()): task_labels = y[:, ind] num_positives = np.count_nonzero(task_labels) num_negatives = len(task_labels) - num_positives pos_weight, neg_weight = weights[ind] assert np.isclose(num_positives * pos_weight, num_negatives * neg_weight)
deepchem/models/test/example.csv +10 −10 Original line number Diff line number Diff line Compound ID,log-solubility,smiles Amigdalin,-0.9740000000000001,OCC3OC(OCC2OC(OC(C#N)c1ccccc1)C(O)C(O)C2O)C(O)C(O)C3O Fenfuram,-2.885,Cc1occc1C(=O)Nc2ccccc2 citral,-2.5789999999999997,CC(C)=CCCC(C)=CC(=O) Picene,-6.617999999999999,c1ccc2c(c1)ccc3c2ccc4c5ccccc5ccc43 Thiophene,-2.2319999999999998,c1ccsc1 benzothiazole,-2.733,c2ccc1scnc1c2 "2,2,4,6,6'-PCB",-6.545,Clc1cc(Cl)c(c(Cl)c1)c2c(Cl)cccc2Cl Estradiol,-4.138,CC12CCC3C(CCc4cc(O)ccc34)C2CCC1O Dieldrin,-4.533,ClC4=C(Cl)C5(Cl)C3C1CC(C2OC12)C3C4(Cl)C5(Cl)Cl Rotenone,-5.246,COc5cc4OCC3Oc2c1CC(Oc1ccc2C(=O)C3c4cc5OC)C(C)=C Amigdalin,0.9740000000000001,OCC3OC(OCC2OC(OC(C#N)c1ccccc1)C(O)C(O)C2O)C(O)C(O)C3O Fenfuram,2.885,Cc1occc1C(=O)Nc2ccccc2 citral,2.5789999999999997,CC(C)=CCCC(C)=CC(=O) Picene,6.617999999999999,c1ccc2c(c1)ccc3c2ccc4c5ccccc5ccc43 Thiophene,2.2319999999999998,c1ccsc1 benzothiazole,2.733,c2ccc1scnc1c2 "2,2,4,6,6'-PCB",6.545,Clc1cc(Cl)c(c(Cl)c1)c2c(Cl)cccc2Cl Estradiol,4.138,CC12CCC3C(CCc4cc(O)ccc34)C2CCC1O Dieldrin,4.533,ClC4=C(Cl)C5(Cl)C3C1CC(C2OC12)C3C4(Cl)C5(Cl)Cl Rotenone,5.246,COc5cc4OCC3Oc2c1CC(Oc1ccc2C(=O)C3c4cc5OC)C(C)=C
deepchem/transformers/__init__.py +18 −17 Original line number Diff line number Diff line Loading @@ -52,7 +52,8 @@ class Transformer(object): """ df = dataset.metadata_df indices = range(0, df.shape[0]) transform_row_partial = partial(_transform_row, df=df, transformer=self) transform_row_partial = partial( _transform_row, df=df, transformer=self) if parallel: pool = mp.Pool(int(mp.cpu_count()/4)) pool.map(transform_row_partial, indices) Loading @@ -74,9 +75,8 @@ class NormalizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None): """Initialize normalization transformation.""" super(NormalizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) super(NormalizationTransformer, self).__init__( transform_X=transform_X, transform_y=transform_y, dataset=dataset) X_means, X_stds, y_means, y_stds = dataset.get_statistics() self.X_means = X_means self.X_stds = X_stds Loading @@ -89,7 +89,8 @@ class NormalizationTransformer(Transformer): self.X_stds = X_stds self.y_means = y_means self.y_stds = y_stds super(NormalizationTransformer, self).transform(dataset, parallel=parallel) super(NormalizationTransformer, self).transform( dataset, parallel=parallel) def transform_row(self, i, df): Loading Loading @@ -150,7 +151,7 @@ class ClippingTransformer(Transformer): class LogTransformer(Transformer): def transform_row(i, df): def transform_row(self, i, df): """Logarithmically transforms data in dataset.""" row = df.iloc[i] if self.transform_X: Loading @@ -172,14 +173,13 @@ class CoulombRandomizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None, seed=None): """Iniitialize coulomb matrix randomization transformation. """ super(CoulombRandomizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) super(CoulombRandomizationTransformer, self).__init__( transform_X=transform_X, transform_y=transform_y, dataset=dataset) self.seed = seed def construct_cm_from_triu(self, x): """ Constructs the unpadded coulomb matrix from the upper triangular portion. Constructs unpadded coulomb matrix from upper triangular portion. """ d = int((np.sqrt(8*len(x)+1)-1)/2) cm = np.zeros([d,d]) Loading @@ -206,7 +206,8 @@ class CoulombRandomizationTransformer(Transformer): upcm = cm[0:atom_number,0:atom_number] row_norms = np.asarray([np.linalg.norm(row) for row in upcm], dtype=float) row_norms = np.asarray( [np.linalg.norm(row) for row in upcm], dtype=float) rng = np.random.RandomState(self.seed) e = rng.normal(size=row_norms.size) p = np.argsort(row_norms+e) Loading @@ -229,7 +230,8 @@ class CoulombRandomizationTransformer(Transformer): save_to_disk(X, row['X-transformed']) if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") print("y will not be transformed by " "CoulombRandomizationTransformer.") def untransform(self, z): print("Cannot undo CoulombRandomizationTransformer.") Loading @@ -239,9 +241,8 @@ class CoulombBinarizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None, theta=1, update_state=True): """Initialize binarization transformation.""" super(CoulombBinarizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) super(CoulombBinarizationTransformer, self).__init__( transform_X=transform_X, transform_y=transform_y, dataset=dataset) self.theta = theta self.feature_max = np.zeros(dataset.get_data_shape()) self.update_state = update_state Loading Loading @@ -292,8 +293,8 @@ class CoulombBinarizationTransformer(Transformer): save_to_disk(X_bin, row['X-transformed']) if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") print("y will not be transformed by " "CoulombBinarizationTransformer.") def untranform(self, z): print("Cannot undo CoulombBinarizationTransformer.")
