Loading deepchem/data/__init__.py +1 −0 Original line number Diff line number Diff line Loading @@ -8,3 +8,4 @@ from __future__ import unicode_literals # TODO(rbharath): Get rid of * import from deepchem.data.datasets import * from deepchem.data.supports import * import deepchem.data.tests deepchem/data/datasets.py +7 −0 Original line number Diff line number Diff line Loading @@ -567,6 +567,10 @@ class DiskDataset(Dataset): for i in range(num_shards): X, y, w, ids = dataset.get_shard(shard_perm[i]) n_samples = X.shape[0] # TODO(rbharath): This happens in tests sometimes, but don't understand why? # Handle edge case. if n_samples == 0: continue if not deterministic: sample_perm = np.random.permutation(n_samples) else: Loading Loading @@ -954,6 +958,9 @@ class DiskDataset(Dataset): # Updating counts indices_count += num_shard_elts count += shard_len # Break when all indices have been used up already if indices_count >= len(indices): break return DiskDataset(data_dir=select_dir, metadata_rows=metadata_rows, verbosity=self.verbosity) Loading deepchem/data/supports.py +179 −40 Original line number Diff line number Diff line Loading @@ -5,6 +5,7 @@ from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time import numpy as np from deepchem.data import NumpyDataset Loading Loading @@ -56,7 +57,44 @@ def get_task_dataset(dataset, task): return NumpyDataset(X_task, y_task, w_task, ids_task) def get_task_test(dataset, batch_size, task, replace=True): def get_task_test(dataset, n_episodes, n_test, task, log_every_n=50): """Gets test set from specified task. Parameters ---------- dataset: dc.data.Dataset Dataset from which to sample. n_episodes: int Number of episodes to sample test sets for. n_test: int Number of compounds per test set. log_every_n: int, optional Prints every log_every_n supports sampled. """ w_task = dataset.w[:, task] X_task = dataset.X[w_task != 0] y_task = dataset.y[w_task != 0] ids_task = dataset.ids[w_task != 0] # Now just get weights for this task w_task = dataset.w[w_task != 0] n_samples = len(X_task) ids = np.random.choice(np.arange(n_samples), (n_episodes, n_test)) tests = [] for episode in range(n_episodes): if episode % log_every_n == 0: print("Sampling test %d" % episode) inds = ids[episode] X_batch = X_task[inds] y_batch = np.squeeze(y_task[inds, task]) w_batch = np.squeeze(w_task[inds, task]) ids_batch = ids_task[inds] tests.append(NumpyDataset(X_batch, y_batch, w_batch, ids_batch)) return tests def get_single_task_test(dataset, batch_size, task, replace=True): """Gets test set from specified task. Samples random subset of size batch_size from specified task of dataset. Loading @@ -76,8 +114,10 @@ def get_task_test(dataset, batch_size, task, replace=True): ids_batch = ids_task[inds] return NumpyDataset(X_batch, y_batch, w_batch, ids_batch) def get_task_support(dataset, n_pos, n_neg, task, replace=True): """Generates a support set purely for specified task. def get_single_task_support(dataset, n_pos, n_neg, task, replace=True): """Generates one support set purely for specified task. Parameters ---------- Loading @@ -92,36 +132,143 @@ def get_task_support(dataset, n_pos, n_neg, task, replace=True): replace: bool, optional Whether or not to use replacement when sampling supports. Returns ------- list List of NumpyDatasets, each of which is a support set. """ return get_task_support(dataset, 1, n_pos, n_neg, task)[0] def get_task_support(dataset, n_episodes, n_pos, n_neg, task, log_every_n=50): """Generates one support set purely for specified task. Parameters ---------- datasets: dc.data.Dataset Dataset from which supports are sampled. n_episodes: int Number of episodes for which supports have to be sampled from this task. n_pos: int Number of positive samples in support. n_neg: int Number of negative samples in support. task: int Index of current task. log_every_n: int, optional Prints every log_every_n supports sampled. Returns ------- list List of NumpyDatasets, each of which is a support set. """ y_task = dataset.y[:, task] w_task = dataset.w[:, task] # Split data into pos and neg lists. pos_mols = np.where(y_task == 1)[0] neg_mols = np.where(y_task == 0)[0] # Get randomly sampled pos/neg indices (with replacement) pos_inds = pos_mols[np.random.choice(len(pos_mols), (n_pos), replace=replace)] neg_inds = neg_mols[np.random.choice(len(neg_mols), (n_neg), replace=replace)] pos_mols = np.where(np.logical_and(y_task == 1, w_task != 0))[0] neg_mols = np.where(np.logical_and(y_task == 0, w_task != 0))[0] supports = [] for episode in range(n_episodes): if episode % log_every_n == 0: print("Sampling support %d" % episode) # No replacement allowed for supports pos_ids = np.random.choice(len(pos_mols), (n_pos,), replace=False) neg_ids = np.random.choice(len(neg_mols), (n_neg,), replace=False) pos_inds, neg_inds = pos_mols[pos_ids], neg_mols[neg_ids] # Handle one-d vs. non one-d feature matrices one_dimensional_features = (len(dataset.X.shape) == 1) if not one_dimensional_features: X_trial = np.vstack( X = np.vstack( [dataset.X[pos_inds], dataset.X[neg_inds]]) else: X_trial = np.concatenate( X = np.concatenate( [dataset.X[pos_inds], dataset.X[neg_inds]]) y_trial = np.concatenate( y = np.concatenate( [dataset.y[pos_inds, task], dataset.y[neg_inds, task]]) w_trial = np.concatenate( w = np.concatenate( [dataset.w[pos_inds, task], dataset.w[neg_inds, task]]) ids_trial = np.concatenate( ids = np.concatenate( [dataset.ids[pos_inds], dataset.ids[neg_inds]]) return NumpyDataset(X_trial, y_trial, w_trial, ids_trial) supports.append(NumpyDataset(X, y, w, ids)) return supports class EpisodeGenerator(object): """Generates (support, test) pairs for episodic training. Precomputes all (support, test) pairs at construction. Allows to reduce overhead from computation. """ def __init__(self, dataset, n_pos, n_neg, n_test, n_episodes_per_task): """ Parameters ---------- dataset: dc.data.Dataset Holds dataset from which support sets will be sampled. n_pos: int Number of positive samples n_neg: int Number of negative samples. n_test: int Number of samples in test set. n_episodes_per_task: int Number of (support, task) pairs to sample per task. replace: bool Whether to use sampling with or without replacement. """ time_start = time.time() self.tasks = range(len(dataset.get_task_names()) ) self.n_tasks = len(self.tasks) self.n_episodes_per_task = n_episodes_per_task self.dataset = dataset self.n_pos = n_pos self.n_neg = n_neg self.task_episodes = {} for task in range(self.n_tasks): task_supports = get_task_support( self.dataset, n_episodes_per_task, n_pos, n_neg, task) task_tests = get_task_test( self.dataset, n_episodes_per_task, n_test, task) self.task_episodes[task] = (task_supports, task_tests) # Init the iterator self.perm_tasks = np.random.permutation(self.tasks) # Set initial iterator state self.task_num = 0 self.trial_num = 0 time_end = time.time() print("Constructing EpisodeGenerator took %s seconds" % str(time_end-time_start)) def __iter__(self): return self def next(self): """Sample next (support, test) pair. Return from internal storage. """ if self.trial_num == self.n_episodes_per_task: raise StopIteration else: task = self.perm_tasks[self.task_num] # Get id from permutation #support = self.supports[task][self.trial_num] task_supports, task_tests = self.task_episodes[task] support, test = (task_supports[self.trial_num], task_tests[self.trial_num]) # Increment and update logic self.task_num += 1 if self.task_num == self.n_tasks: self.task_num = 0 # Reset self.perm_tasks = np.random.permutation(self.tasks) # Permute again self.trial_num += 1 # Upgrade trial index return (task, support, test) __next__ = next # Python 3.X compatibility class SupportGenerator(object): """Generate support sets from a dataset. Loading @@ -129,33 +276,27 @@ class SupportGenerator(object): Iterates over tasks and trials. For each trial, picks one support from each task, and returns in a randomized order """ def __init__(self, dataset, tasks, n_pos, n_neg, n_trials, replace): def __init__(self, dataset, n_pos, n_neg, n_trials): """ Parameters ---------- dataset: dc.data.Dataset Holds dataset from which support sets will be sampled. tasks: list Indices of tasks from which supports are sampled. TODO(rbharath): Can this be removed. n_pos: int Number of positive samples n_neg: int Number of negative samples. n_trials: int Number of passes over tasks to make. In total, n_tasks*n_trials Number of passes over dataset to make. In total, n_tasks*n_trials support sets will be sampled by algorithm. replace: bool Whether to use sampling with or without replacement. """ self.tasks = tasks self.n_tasks = len(tasks) self.tasks = range(len(dataset.get_task_names()) ) self.n_tasks = len(self.tasks) self.n_trials = n_trials self.dataset = dataset self.n_pos = n_pos self.n_neg = n_neg self.replace = replace # Init the iterator self.perm_tasks = np.random.permutation(self.tasks) Loading @@ -166,8 +307,6 @@ class SupportGenerator(object): def __iter__(self): return self # TODO(rbharath): This is generating data from one task at a time. Is it # wrong to have batches that mix information from multiple tasks? def next(self): """Sample next support. Loading @@ -179,9 +318,9 @@ class SupportGenerator(object): else: task = self.perm_tasks[self.task_num] # Get id from permutation #support = self.supports[task][self.trial_num] support = get_task_support( support = get_single_task_support( self.dataset, n_pos=self.n_pos, n_neg=self.n_neg, task=task, replace=self.replace) replace=False) # Increment and update logic self.task_num += 1 if self.task_num == self.n_tasks: Loading deepchem/data/tests/test_support_generator.py 0 → 100644 +305 −0 Original line number Diff line number Diff line """ Simple Tests for Support Generation """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals __author__ = "Han Altae-Tran and Bharath Ramsundar" __copyright__ = "Copyright 2016, Stanford University" __license__ = "GPL" import numpy as np import unittest import tensorflow as tf import deepchem as dc class TestSupports(unittest.TestCase): """ Test that support generation happens properly. """ def test_get_task_support_simple(self): """Tests that get_task_support samples correctly.""" n_samples = 20 n_features = 3 n_tasks = 1 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) n_episodes = 20 n_pos = 1 n_neg = 5 supports = dc.data.get_task_support(dataset, n_episodes, n_pos, n_neg, task=0, log_every_n=10) assert len(supports) == n_episodes for support in supports: assert len(support) == n_pos + n_neg assert np.count_nonzero(support.y) == n_pos def test_get_task_support_missing(self): """Test that task support works in presence of missing data.""" n_samples = 20 n_features = 3 n_tasks = 1 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) # Set last n_samples/2 weights to 0 w[n_samples/2:] = 0 dataset = dc.data.NumpyDataset(X, y, w, ids) n_episodes = 20 n_pos = 1 n_neg = 2 supports = dc.data.get_task_support(dataset, n_episodes, n_pos, n_neg, task=0, log_every_n=10) assert len(supports) == n_episodes for support in supports: assert len(support) == n_pos + n_neg assert np.count_nonzero(support.y) == n_pos # Check that no support elements are sample from zero-weight samples for identifier in support.ids: assert identifier < n_samples/2 def test_get_task_test(self): """Tests that get_task_testsamples correctly.""" n_samples = 20 n_features = 3 n_tasks = 1 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) n_episodes = 20 n_test = 10 tests = dc.data.get_task_test(dataset, n_episodes, n_test, task=0, log_every_n=10) assert len(tests) == n_episodes for test in tests: assert len(test) == n_test def test_simple_support_generator(self): """Conducts simple test that support generator runs.""" n_samples = 20 n_features = 3 n_tasks = 1 n_pos = 1 n_neg = 5 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) # Create support generator supp_gen = dc.data.SupportGenerator(dataset, n_pos, n_neg, n_trials) def test_simple_episode_generator(self): """Conducts simple test that episode generator runs.""" n_samples = 20 n_features = 3 n_tasks = 1 n_pos = 1 n_neg = 5 n_test = 10 n_episodes = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) # Create support generator episode_gen = dc.data.EpisodeGenerator( dataset, n_pos, n_neg, n_test, n_episodes) n_episodes_found = 0 for (task, support, test) in episode_gen: assert task >= 0 assert task < n_tasks assert len(support) == n_pos + n_neg assert np.count_nonzero(support.y) == n_pos assert len(test) == n_test n_episodes_found += 1 assert n_episodes_found == n_episodes def test_get_task_minus_support_simple(self): """Test that fixed index support can be removed from dataset.""" n_samples = 20 n_support = 5 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) support_dataset = dc.data.NumpyDataset(X[:n_support], y[:n_support], w[:n_support], ids[:n_support]) task_dataset = dc.data.get_task_dataset_minus_support( dataset, support_dataset, task=0) # Assert all support elements have been removed assert len(task_dataset) == n_samples - n_support np.testing.assert_array_equal(task_dataset.X, X[n_support:]) np.testing.assert_array_equal(task_dataset.y, y[n_support:]) np.testing.assert_array_equal(task_dataset.w, w[n_support:]) np.testing.assert_array_equal(task_dataset.ids, ids[n_support:]) def test_get_task_minus_support(self): """Test that random index support can be removed from dataset.""" n_samples = 10 n_support = 4 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) support_inds = sorted(np.random.choice( np.arange(n_samples), (n_support,), replace=False)) support_dataset = dc.data.NumpyDataset(X[support_inds], y[support_inds], w[support_inds], ids[support_inds]) task_dataset = dc.data.get_task_dataset_minus_support( dataset, support_dataset, task=0) # Assert all support elements have been removed data_inds = sorted(list(set(range(n_samples)) - set(support_inds))) assert len(task_dataset) == n_samples - n_support np.testing.assert_array_equal(task_dataset.X, X[data_inds]) np.testing.assert_array_equal(task_dataset.y, y[data_inds]) np.testing.assert_array_equal(task_dataset.w, w[data_inds]) np.testing.assert_array_equal(task_dataset.ids, ids[data_inds]) def test_get_task_minus_support_missing(self): """Test that support can be removed from dataset with missing data""" n_samples = 20 n_support = 4 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) # Set last n_samples/2 weights to 0 w[n_samples/2:] = 0 dataset = dc.data.NumpyDataset(X, y, w, ids) # Sample from first n_samples/2 elements for support support_inds = sorted(np.random.choice( np.arange(n_samples/2), (n_support,), replace=False)) support_dataset = dc.data.NumpyDataset(X[support_inds], y[support_inds], w[support_inds], ids[support_inds]) task_dataset = dc.data.get_task_dataset_minus_support( dataset, support_dataset, task=0) # Should lie within first n_samples/2 samples only assert len(task_dataset) == n_samples/2 - n_support for identifier in task_dataset.ids: assert identifier < n_samples/2 def test_support_generator_correct_samples(self): """Tests that samples from support generator have desired shape.""" n_samples = 20 n_features = 3 n_tasks = 1 n_pos = 1 n_neg = 5 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) # Create support generator supp_gen = dc.data.SupportGenerator(dataset, n_pos, n_neg, n_trials) num_supports = 0 for (task, support) in supp_gen: assert support.X.shape == (n_pos + n_neg, n_features) num_supports += 1 assert task == 0 # Only one task in this example n_supp_pos = np.count_nonzero(support.y) assert n_supp_pos == n_pos assert num_supports == n_trials def test_evaluation_strategy(self): """Tests that sampling supports for eval works properly.""" n_samples = 2000 n_features = 3 n_tasks = 5 n_pos = 1 n_neg = 5 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.random.randint(2, size=(n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) support_generator = dc.data.SupportGenerator(dataset, n_pos, n_neg, n_trials) for ind, (task, support) in enumerate(support_generator): task_dataset = dc.data.get_task_dataset_minus_support( dataset, support, task) task_y = dataset.y[:, task] task_w = dataset.w[:, task] task_y = task_y[task_w != 0] assert len(task_y) == len(support) + len(task_dataset) print("Verifying that task_dataset doesn't overlap with support.") for task_id in task_dataset.ids: assert task_id not in set(support.ids) deepchem/models/tests/test_overfit.py +15 −21 Original line number Diff line number Diff line Loading @@ -667,9 +667,8 @@ class TestOverfit(test_util.TensorFlowTestCase): n_pos = 6 n_neg = 4 test_batch_size = 10 n_train_trials = 60 n_train_trials = 80 support_batch_size = n_pos + n_neg replace = False # Load mini log-solubility dataset. featurizer = dc.feat.ConvMolFeaturizer() Loading Loading @@ -705,10 +704,9 @@ class TestOverfit(test_util.TensorFlowTestCase): verbosity="high") # Fit trained model. Dataset has 6 positives and 4 negatives, so set # n_pos/n_neg accordingly. Set replace to false to ensure full dataset # is always passed in to support. model.fit(dataset, n_trials=n_train_trials, n_pos=n_pos, n_neg=n_neg, replace=False) # n_pos/n_neg accordingly. model.fit(dataset, n_episodes_per_epoch=n_train_trials, n_pos=n_pos, n_neg=n_neg) model.save() # Eval model on train. Dataset has 6 positives and 4 negatives, so set Loading @@ -718,7 +716,7 @@ class TestOverfit(test_util.TensorFlowTestCase): # model has mastered memorization of provided support. scores = model.evaluate(dataset, classification_metric, n_trials=5, n_pos=n_pos, n_neg=n_neg, exclude_support=False, replace=False) exclude_support=False) # Measure performance on 0-th task. assert scores[0] > .9 Loading @@ -738,8 +736,7 @@ class TestOverfit(test_util.TensorFlowTestCase): n_neg = 4 test_batch_size = 10 support_batch_size = n_pos + n_neg n_train_trials = 60 replace = False n_train_trials = 80 # Load mini log-solubility dataset. featurizer = dc.feat.ConvMolFeaturizer() Loading Loading @@ -780,10 +777,9 @@ class TestOverfit(test_util.TensorFlowTestCase): verbosity="high") # Fit trained model. Dataset has 6 positives and 4 negatives, so set # n_pos/n_neg accordingly. Set replace to false to ensure full dataset # is always passed in to support. model.fit(dataset, n_trials=n_train_trials, n_pos=n_pos, n_neg=n_neg, replace=False) # n_pos/n_neg accordingly. model.fit(dataset, n_episodes_per_epoch=n_train_trials, n_pos=n_pos, n_neg=n_neg) model.save() # Eval model on train. Dataset has 6 positives and 4 negatives, so set Loading @@ -793,7 +789,7 @@ class TestOverfit(test_util.TensorFlowTestCase): # model has mastered memorization of provided support. scores = model.evaluate(dataset, classification_metric, n_trials=5, n_pos=n_pos, n_neg=n_neg, exclude_support=False, replace=False) exclude_support=False) # Measure performance on 0-th task. assert scores[0] > .9 Loading @@ -811,8 +807,7 @@ class TestOverfit(test_util.TensorFlowTestCase): n_neg = 4 test_batch_size = 10 support_batch_size = n_pos + n_neg n_train_trials = 60 replace = False n_train_trials = 80 # Load mini log-solubility dataset. featurizer = dc.feat.ConvMolFeaturizer() Loading Loading @@ -853,11 +848,10 @@ class TestOverfit(test_util.TensorFlowTestCase): verbosity="high") # Fit trained model. Dataset has 6 positives and 4 negatives, so set # n_pos/n_neg accordingly. Set replace to false to ensure full dataset # is always passed in to support. # n_pos/n_neg accordingly. model.fit(dataset, n_trials=n_train_trials, n_pos=n_pos, n_neg=n_neg, replace=False) model.fit(dataset, n_episodes_per_epoch=n_train_trials, n_pos=n_pos, n_neg=n_neg) model.save() # Eval model on train. Dataset has 6 positives and 4 negatives, so set Loading @@ -867,7 +861,7 @@ class TestOverfit(test_util.TensorFlowTestCase): # model has mastered memorization of provided support. scores = model.evaluate(dataset, classification_metric, n_trials=5, n_pos=n_pos, n_neg=n_neg, exclude_support=False, replace=False) exclude_support=False) # Measure performance on 0-th task. assert scores[0] > .9 Loading
deepchem/data/__init__.py +1 −0 Original line number Diff line number Diff line Loading @@ -8,3 +8,4 @@ from __future__ import unicode_literals # TODO(rbharath): Get rid of * import from deepchem.data.datasets import * from deepchem.data.supports import * import deepchem.data.tests
deepchem/data/datasets.py +7 −0 Original line number Diff line number Diff line Loading @@ -567,6 +567,10 @@ class DiskDataset(Dataset): for i in range(num_shards): X, y, w, ids = dataset.get_shard(shard_perm[i]) n_samples = X.shape[0] # TODO(rbharath): This happens in tests sometimes, but don't understand why? # Handle edge case. if n_samples == 0: continue if not deterministic: sample_perm = np.random.permutation(n_samples) else: Loading Loading @@ -954,6 +958,9 @@ class DiskDataset(Dataset): # Updating counts indices_count += num_shard_elts count += shard_len # Break when all indices have been used up already if indices_count >= len(indices): break return DiskDataset(data_dir=select_dir, metadata_rows=metadata_rows, verbosity=self.verbosity) Loading
deepchem/data/supports.py +179 −40 Original line number Diff line number Diff line Loading @@ -5,6 +5,7 @@ from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time import numpy as np from deepchem.data import NumpyDataset Loading Loading @@ -56,7 +57,44 @@ def get_task_dataset(dataset, task): return NumpyDataset(X_task, y_task, w_task, ids_task) def get_task_test(dataset, batch_size, task, replace=True): def get_task_test(dataset, n_episodes, n_test, task, log_every_n=50): """Gets test set from specified task. Parameters ---------- dataset: dc.data.Dataset Dataset from which to sample. n_episodes: int Number of episodes to sample test sets for. n_test: int Number of compounds per test set. log_every_n: int, optional Prints every log_every_n supports sampled. """ w_task = dataset.w[:, task] X_task = dataset.X[w_task != 0] y_task = dataset.y[w_task != 0] ids_task = dataset.ids[w_task != 0] # Now just get weights for this task w_task = dataset.w[w_task != 0] n_samples = len(X_task) ids = np.random.choice(np.arange(n_samples), (n_episodes, n_test)) tests = [] for episode in range(n_episodes): if episode % log_every_n == 0: print("Sampling test %d" % episode) inds = ids[episode] X_batch = X_task[inds] y_batch = np.squeeze(y_task[inds, task]) w_batch = np.squeeze(w_task[inds, task]) ids_batch = ids_task[inds] tests.append(NumpyDataset(X_batch, y_batch, w_batch, ids_batch)) return tests def get_single_task_test(dataset, batch_size, task, replace=True): """Gets test set from specified task. Samples random subset of size batch_size from specified task of dataset. Loading @@ -76,8 +114,10 @@ def get_task_test(dataset, batch_size, task, replace=True): ids_batch = ids_task[inds] return NumpyDataset(X_batch, y_batch, w_batch, ids_batch) def get_task_support(dataset, n_pos, n_neg, task, replace=True): """Generates a support set purely for specified task. def get_single_task_support(dataset, n_pos, n_neg, task, replace=True): """Generates one support set purely for specified task. Parameters ---------- Loading @@ -92,36 +132,143 @@ def get_task_support(dataset, n_pos, n_neg, task, replace=True): replace: bool, optional Whether or not to use replacement when sampling supports. Returns ------- list List of NumpyDatasets, each of which is a support set. """ return get_task_support(dataset, 1, n_pos, n_neg, task)[0] def get_task_support(dataset, n_episodes, n_pos, n_neg, task, log_every_n=50): """Generates one support set purely for specified task. Parameters ---------- datasets: dc.data.Dataset Dataset from which supports are sampled. n_episodes: int Number of episodes for which supports have to be sampled from this task. n_pos: int Number of positive samples in support. n_neg: int Number of negative samples in support. task: int Index of current task. log_every_n: int, optional Prints every log_every_n supports sampled. Returns ------- list List of NumpyDatasets, each of which is a support set. """ y_task = dataset.y[:, task] w_task = dataset.w[:, task] # Split data into pos and neg lists. pos_mols = np.where(y_task == 1)[0] neg_mols = np.where(y_task == 0)[0] # Get randomly sampled pos/neg indices (with replacement) pos_inds = pos_mols[np.random.choice(len(pos_mols), (n_pos), replace=replace)] neg_inds = neg_mols[np.random.choice(len(neg_mols), (n_neg), replace=replace)] pos_mols = np.where(np.logical_and(y_task == 1, w_task != 0))[0] neg_mols = np.where(np.logical_and(y_task == 0, w_task != 0))[0] supports = [] for episode in range(n_episodes): if episode % log_every_n == 0: print("Sampling support %d" % episode) # No replacement allowed for supports pos_ids = np.random.choice(len(pos_mols), (n_pos,), replace=False) neg_ids = np.random.choice(len(neg_mols), (n_neg,), replace=False) pos_inds, neg_inds = pos_mols[pos_ids], neg_mols[neg_ids] # Handle one-d vs. non one-d feature matrices one_dimensional_features = (len(dataset.X.shape) == 1) if not one_dimensional_features: X_trial = np.vstack( X = np.vstack( [dataset.X[pos_inds], dataset.X[neg_inds]]) else: X_trial = np.concatenate( X = np.concatenate( [dataset.X[pos_inds], dataset.X[neg_inds]]) y_trial = np.concatenate( y = np.concatenate( [dataset.y[pos_inds, task], dataset.y[neg_inds, task]]) w_trial = np.concatenate( w = np.concatenate( [dataset.w[pos_inds, task], dataset.w[neg_inds, task]]) ids_trial = np.concatenate( ids = np.concatenate( [dataset.ids[pos_inds], dataset.ids[neg_inds]]) return NumpyDataset(X_trial, y_trial, w_trial, ids_trial) supports.append(NumpyDataset(X, y, w, ids)) return supports class EpisodeGenerator(object): """Generates (support, test) pairs for episodic training. Precomputes all (support, test) pairs at construction. Allows to reduce overhead from computation. """ def __init__(self, dataset, n_pos, n_neg, n_test, n_episodes_per_task): """ Parameters ---------- dataset: dc.data.Dataset Holds dataset from which support sets will be sampled. n_pos: int Number of positive samples n_neg: int Number of negative samples. n_test: int Number of samples in test set. n_episodes_per_task: int Number of (support, task) pairs to sample per task. replace: bool Whether to use sampling with or without replacement. """ time_start = time.time() self.tasks = range(len(dataset.get_task_names()) ) self.n_tasks = len(self.tasks) self.n_episodes_per_task = n_episodes_per_task self.dataset = dataset self.n_pos = n_pos self.n_neg = n_neg self.task_episodes = {} for task in range(self.n_tasks): task_supports = get_task_support( self.dataset, n_episodes_per_task, n_pos, n_neg, task) task_tests = get_task_test( self.dataset, n_episodes_per_task, n_test, task) self.task_episodes[task] = (task_supports, task_tests) # Init the iterator self.perm_tasks = np.random.permutation(self.tasks) # Set initial iterator state self.task_num = 0 self.trial_num = 0 time_end = time.time() print("Constructing EpisodeGenerator took %s seconds" % str(time_end-time_start)) def __iter__(self): return self def next(self): """Sample next (support, test) pair. Return from internal storage. """ if self.trial_num == self.n_episodes_per_task: raise StopIteration else: task = self.perm_tasks[self.task_num] # Get id from permutation #support = self.supports[task][self.trial_num] task_supports, task_tests = self.task_episodes[task] support, test = (task_supports[self.trial_num], task_tests[self.trial_num]) # Increment and update logic self.task_num += 1 if self.task_num == self.n_tasks: self.task_num = 0 # Reset self.perm_tasks = np.random.permutation(self.tasks) # Permute again self.trial_num += 1 # Upgrade trial index return (task, support, test) __next__ = next # Python 3.X compatibility class SupportGenerator(object): """Generate support sets from a dataset. Loading @@ -129,33 +276,27 @@ class SupportGenerator(object): Iterates over tasks and trials. For each trial, picks one support from each task, and returns in a randomized order """ def __init__(self, dataset, tasks, n_pos, n_neg, n_trials, replace): def __init__(self, dataset, n_pos, n_neg, n_trials): """ Parameters ---------- dataset: dc.data.Dataset Holds dataset from which support sets will be sampled. tasks: list Indices of tasks from which supports are sampled. TODO(rbharath): Can this be removed. n_pos: int Number of positive samples n_neg: int Number of negative samples. n_trials: int Number of passes over tasks to make. In total, n_tasks*n_trials Number of passes over dataset to make. In total, n_tasks*n_trials support sets will be sampled by algorithm. replace: bool Whether to use sampling with or without replacement. """ self.tasks = tasks self.n_tasks = len(tasks) self.tasks = range(len(dataset.get_task_names()) ) self.n_tasks = len(self.tasks) self.n_trials = n_trials self.dataset = dataset self.n_pos = n_pos self.n_neg = n_neg self.replace = replace # Init the iterator self.perm_tasks = np.random.permutation(self.tasks) Loading @@ -166,8 +307,6 @@ class SupportGenerator(object): def __iter__(self): return self # TODO(rbharath): This is generating data from one task at a time. Is it # wrong to have batches that mix information from multiple tasks? def next(self): """Sample next support. Loading @@ -179,9 +318,9 @@ class SupportGenerator(object): else: task = self.perm_tasks[self.task_num] # Get id from permutation #support = self.supports[task][self.trial_num] support = get_task_support( support = get_single_task_support( self.dataset, n_pos=self.n_pos, n_neg=self.n_neg, task=task, replace=self.replace) replace=False) # Increment and update logic self.task_num += 1 if self.task_num == self.n_tasks: Loading
deepchem/data/tests/test_support_generator.py 0 → 100644 +305 −0 Original line number Diff line number Diff line """ Simple Tests for Support Generation """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals __author__ = "Han Altae-Tran and Bharath Ramsundar" __copyright__ = "Copyright 2016, Stanford University" __license__ = "GPL" import numpy as np import unittest import tensorflow as tf import deepchem as dc class TestSupports(unittest.TestCase): """ Test that support generation happens properly. """ def test_get_task_support_simple(self): """Tests that get_task_support samples correctly.""" n_samples = 20 n_features = 3 n_tasks = 1 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) n_episodes = 20 n_pos = 1 n_neg = 5 supports = dc.data.get_task_support(dataset, n_episodes, n_pos, n_neg, task=0, log_every_n=10) assert len(supports) == n_episodes for support in supports: assert len(support) == n_pos + n_neg assert np.count_nonzero(support.y) == n_pos def test_get_task_support_missing(self): """Test that task support works in presence of missing data.""" n_samples = 20 n_features = 3 n_tasks = 1 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) # Set last n_samples/2 weights to 0 w[n_samples/2:] = 0 dataset = dc.data.NumpyDataset(X, y, w, ids) n_episodes = 20 n_pos = 1 n_neg = 2 supports = dc.data.get_task_support(dataset, n_episodes, n_pos, n_neg, task=0, log_every_n=10) assert len(supports) == n_episodes for support in supports: assert len(support) == n_pos + n_neg assert np.count_nonzero(support.y) == n_pos # Check that no support elements are sample from zero-weight samples for identifier in support.ids: assert identifier < n_samples/2 def test_get_task_test(self): """Tests that get_task_testsamples correctly.""" n_samples = 20 n_features = 3 n_tasks = 1 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) n_episodes = 20 n_test = 10 tests = dc.data.get_task_test(dataset, n_episodes, n_test, task=0, log_every_n=10) assert len(tests) == n_episodes for test in tests: assert len(test) == n_test def test_simple_support_generator(self): """Conducts simple test that support generator runs.""" n_samples = 20 n_features = 3 n_tasks = 1 n_pos = 1 n_neg = 5 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) # Create support generator supp_gen = dc.data.SupportGenerator(dataset, n_pos, n_neg, n_trials) def test_simple_episode_generator(self): """Conducts simple test that episode generator runs.""" n_samples = 20 n_features = 3 n_tasks = 1 n_pos = 1 n_neg = 5 n_test = 10 n_episodes = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) # Create support generator episode_gen = dc.data.EpisodeGenerator( dataset, n_pos, n_neg, n_test, n_episodes) n_episodes_found = 0 for (task, support, test) in episode_gen: assert task >= 0 assert task < n_tasks assert len(support) == n_pos + n_neg assert np.count_nonzero(support.y) == n_pos assert len(test) == n_test n_episodes_found += 1 assert n_episodes_found == n_episodes def test_get_task_minus_support_simple(self): """Test that fixed index support can be removed from dataset.""" n_samples = 20 n_support = 5 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) support_dataset = dc.data.NumpyDataset(X[:n_support], y[:n_support], w[:n_support], ids[:n_support]) task_dataset = dc.data.get_task_dataset_minus_support( dataset, support_dataset, task=0) # Assert all support elements have been removed assert len(task_dataset) == n_samples - n_support np.testing.assert_array_equal(task_dataset.X, X[n_support:]) np.testing.assert_array_equal(task_dataset.y, y[n_support:]) np.testing.assert_array_equal(task_dataset.w, w[n_support:]) np.testing.assert_array_equal(task_dataset.ids, ids[n_support:]) def test_get_task_minus_support(self): """Test that random index support can be removed from dataset.""" n_samples = 10 n_support = 4 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) support_inds = sorted(np.random.choice( np.arange(n_samples), (n_support,), replace=False)) support_dataset = dc.data.NumpyDataset(X[support_inds], y[support_inds], w[support_inds], ids[support_inds]) task_dataset = dc.data.get_task_dataset_minus_support( dataset, support_dataset, task=0) # Assert all support elements have been removed data_inds = sorted(list(set(range(n_samples)) - set(support_inds))) assert len(task_dataset) == n_samples - n_support np.testing.assert_array_equal(task_dataset.X, X[data_inds]) np.testing.assert_array_equal(task_dataset.y, y[data_inds]) np.testing.assert_array_equal(task_dataset.w, w[data_inds]) np.testing.assert_array_equal(task_dataset.ids, ids[data_inds]) def test_get_task_minus_support_missing(self): """Test that support can be removed from dataset with missing data""" n_samples = 20 n_support = 4 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) # Set last n_samples/2 weights to 0 w[n_samples/2:] = 0 dataset = dc.data.NumpyDataset(X, y, w, ids) # Sample from first n_samples/2 elements for support support_inds = sorted(np.random.choice( np.arange(n_samples/2), (n_support,), replace=False)) support_dataset = dc.data.NumpyDataset(X[support_inds], y[support_inds], w[support_inds], ids[support_inds]) task_dataset = dc.data.get_task_dataset_minus_support( dataset, support_dataset, task=0) # Should lie within first n_samples/2 samples only assert len(task_dataset) == n_samples/2 - n_support for identifier in task_dataset.ids: assert identifier < n_samples/2 def test_support_generator_correct_samples(self): """Tests that samples from support generator have desired shape.""" n_samples = 20 n_features = 3 n_tasks = 1 n_pos = 1 n_neg = 5 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) # Create support generator supp_gen = dc.data.SupportGenerator(dataset, n_pos, n_neg, n_trials) num_supports = 0 for (task, support) in supp_gen: assert support.X.shape == (n_pos + n_neg, n_features) num_supports += 1 assert task == 0 # Only one task in this example n_supp_pos = np.count_nonzero(support.y) assert n_supp_pos == n_pos assert num_supports == n_trials def test_evaluation_strategy(self): """Tests that sampling supports for eval works properly.""" n_samples = 2000 n_features = 3 n_tasks = 5 n_pos = 1 n_neg = 5 n_trials = 10 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.random.randint(2, size=(n_samples, n_tasks)) dataset = dc.data.NumpyDataset(X, y, w, ids) support_generator = dc.data.SupportGenerator(dataset, n_pos, n_neg, n_trials) for ind, (task, support) in enumerate(support_generator): task_dataset = dc.data.get_task_dataset_minus_support( dataset, support, task) task_y = dataset.y[:, task] task_w = dataset.w[:, task] task_y = task_y[task_w != 0] assert len(task_y) == len(support) + len(task_dataset) print("Verifying that task_dataset doesn't overlap with support.") for task_id in task_dataset.ids: assert task_id not in set(support.ids)
deepchem/models/tests/test_overfit.py +15 −21 Original line number Diff line number Diff line Loading @@ -667,9 +667,8 @@ class TestOverfit(test_util.TensorFlowTestCase): n_pos = 6 n_neg = 4 test_batch_size = 10 n_train_trials = 60 n_train_trials = 80 support_batch_size = n_pos + n_neg replace = False # Load mini log-solubility dataset. featurizer = dc.feat.ConvMolFeaturizer() Loading Loading @@ -705,10 +704,9 @@ class TestOverfit(test_util.TensorFlowTestCase): verbosity="high") # Fit trained model. Dataset has 6 positives and 4 negatives, so set # n_pos/n_neg accordingly. Set replace to false to ensure full dataset # is always passed in to support. model.fit(dataset, n_trials=n_train_trials, n_pos=n_pos, n_neg=n_neg, replace=False) # n_pos/n_neg accordingly. model.fit(dataset, n_episodes_per_epoch=n_train_trials, n_pos=n_pos, n_neg=n_neg) model.save() # Eval model on train. Dataset has 6 positives and 4 negatives, so set Loading @@ -718,7 +716,7 @@ class TestOverfit(test_util.TensorFlowTestCase): # model has mastered memorization of provided support. scores = model.evaluate(dataset, classification_metric, n_trials=5, n_pos=n_pos, n_neg=n_neg, exclude_support=False, replace=False) exclude_support=False) # Measure performance on 0-th task. assert scores[0] > .9 Loading @@ -738,8 +736,7 @@ class TestOverfit(test_util.TensorFlowTestCase): n_neg = 4 test_batch_size = 10 support_batch_size = n_pos + n_neg n_train_trials = 60 replace = False n_train_trials = 80 # Load mini log-solubility dataset. featurizer = dc.feat.ConvMolFeaturizer() Loading Loading @@ -780,10 +777,9 @@ class TestOverfit(test_util.TensorFlowTestCase): verbosity="high") # Fit trained model. Dataset has 6 positives and 4 negatives, so set # n_pos/n_neg accordingly. Set replace to false to ensure full dataset # is always passed in to support. model.fit(dataset, n_trials=n_train_trials, n_pos=n_pos, n_neg=n_neg, replace=False) # n_pos/n_neg accordingly. model.fit(dataset, n_episodes_per_epoch=n_train_trials, n_pos=n_pos, n_neg=n_neg) model.save() # Eval model on train. Dataset has 6 positives and 4 negatives, so set Loading @@ -793,7 +789,7 @@ class TestOverfit(test_util.TensorFlowTestCase): # model has mastered memorization of provided support. scores = model.evaluate(dataset, classification_metric, n_trials=5, n_pos=n_pos, n_neg=n_neg, exclude_support=False, replace=False) exclude_support=False) # Measure performance on 0-th task. assert scores[0] > .9 Loading @@ -811,8 +807,7 @@ class TestOverfit(test_util.TensorFlowTestCase): n_neg = 4 test_batch_size = 10 support_batch_size = n_pos + n_neg n_train_trials = 60 replace = False n_train_trials = 80 # Load mini log-solubility dataset. featurizer = dc.feat.ConvMolFeaturizer() Loading Loading @@ -853,11 +848,10 @@ class TestOverfit(test_util.TensorFlowTestCase): verbosity="high") # Fit trained model. Dataset has 6 positives and 4 negatives, so set # n_pos/n_neg accordingly. Set replace to false to ensure full dataset # is always passed in to support. # n_pos/n_neg accordingly. model.fit(dataset, n_trials=n_train_trials, n_pos=n_pos, n_neg=n_neg, replace=False) model.fit(dataset, n_episodes_per_epoch=n_train_trials, n_pos=n_pos, n_neg=n_neg) model.save() # Eval model on train. Dataset has 6 positives and 4 negatives, so set Loading @@ -867,7 +861,7 @@ class TestOverfit(test_util.TensorFlowTestCase): # model has mastered memorization of provided support. scores = model.evaluate(dataset, classification_metric, n_trials=5, n_pos=n_pos, n_neg=n_neg, exclude_support=False, replace=False) exclude_support=False) # Measure performance on 0-th task. assert scores[0] > .9
