Faster fit and fix for support sampling with missing data (8afc4c11) · Commits · 钟慕尧 / deepchem

deepchem/data/supports.py

+174 −7

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@ from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		import time
		import numpy as np
		from deepchem.data import NumpyDataset

		@@ -56,7 +57,44 @@ def get_task_dataset(dataset, task):

		return NumpyDataset(X_task, y_task, w_task, ids_task)

		def get_task_test(dataset, batch_size, task, replace=True):
		def get_task_test(dataset, n_episodes, n_test, task, log_every_n=50):
		"""Gets test set from specified task.

		Parameters
		----------
		dataset: dc.data.Dataset
		Dataset from which to sample.
		n_episodes: int
		Number of episodes to sample test sets for.
		n_test: int
		Number of compounds per test set.
		log_every_n: int, optional
		Prints every log_every_n supports sampled.
		"""
		w_task = dataset.w[:, task]
		X_task = dataset.X[w_task != 0]
		y_task = dataset.y[w_task != 0]
		ids_task = dataset.ids[w_task != 0]
		# Now just get weights for this task
		w_task = dataset.w[w_task != 0]

		n_samples = len(X_task)

		ids = np.random.choice(np.arange(n_samples), (n_episodes, n_test))

		tests = []
		for episode in range(n_episodes):
		if episode % log_every_n == 0:
		print("Sampling test %d" % episode)
		inds = ids[episode]
		X_batch = X_task[inds]
		y_batch = np.squeeze(y_task[inds, task])
		w_batch = np.squeeze(w_task[inds, task])
		ids_batch = ids_task[inds]
		tests.append(NumpyDataset(X_batch, y_batch, w_batch, ids_batch))
		return tests

		def get_single_task_test(dataset, batch_size, task, replace=True):
		"""Gets test set from specified task.

		Samples random subset of size batch_size from specified task of dataset.
		@@ -76,8 +114,10 @@ def get_task_test(dataset, batch_size, task, replace=True):
		ids_batch = ids_task[inds]
		return NumpyDataset(X_batch, y_batch, w_batch, ids_batch)

		def get_task_support(dataset, n_pos, n_neg, task, replace=True):
		"""Generates a support set purely for specified task.


		def get_single_task_support(dataset, n_pos, n_neg, task, replace=True):
		"""Generates one support set purely for specified task.

		Parameters
		----------
		@@ -123,6 +163,135 @@ def get_task_support(dataset, n_pos, n_neg, task, replace=True):
		[dataset.ids[pos_inds], dataset.ids[neg_inds]])
		return NumpyDataset(X_trial, y_trial, w_trial, ids_trial)

		def get_task_support(dataset, n_episodes, n_pos, n_neg, task, log_every_n=50):
		"""Generates one support set purely for specified task.

		Parameters
		----------
		datasets: dc.data.Dataset
		Dataset from which supports are sampled.
		n_episodes: int
		Number of episodes for which supports have to be sampled from this task.
		n_pos: int
		Number of positive samples in support.
		n_neg: int
		Number of negative samples in support.
		task: int
		Index of current task.
		log_every_n: int, optional
		Prints every log_every_n supports sampled.

		Returns
		-------
		list
		List of NumpyDatasets, each of which is a support set.
		"""
		y_task = dataset.y[:, task]
		w_task = dataset.w[:, task]

		# Split data into pos and neg lists.
		pos_mols = np.where(np.logical_and(y_task == 1, w_task != 0))[0]
		neg_mols = np.where(np.logical_and(y_task == 0, w_task != 0))[0]

		supports = []
		for episode in range(n_episodes):
		if episode % log_every_n == 0:
		print("Sampling support %d" % episode)
		# No replacement allowed for supports
		pos_ids = np.random.choice(len(pos_mols), (n_pos,), replace=False)
		neg_ids = np.random.choice(len(neg_mols), (n_neg,), replace=False)
		pos_inds, neg_inds = pos_mols[pos_ids], neg_mols[neg_ids]
		# Handle one-d vs. non one-d feature matrices
		one_dimensional_features = (len(dataset.X.shape) == 1)
		if not one_dimensional_features:
		X = np.vstack(
		[dataset.X[pos_inds], dataset.X[neg_inds]])
		else:
		X = np.concatenate(
		[dataset.X[pos_inds], dataset.X[neg_inds]])
		y = np.concatenate(
		[dataset.y[pos_inds, task], dataset.y[neg_inds, task]])
		w = np.concatenate(
		[dataset.w[pos_inds, task], dataset.w[neg_inds, task]])
		ids = np.concatenate(
		[dataset.ids[pos_inds], dataset.ids[neg_inds]])
		supports.append(NumpyDataset(X, y, w, ids))
		return supports

		class EpisodeGenerator(object):
		"""Generates (support, test) pairs for episodic training.

		Precomputes all (support, test) pairs at construction. Allows to reduce
		overhead from computation.
		"""
		def __init__(self, dataset, n_pos, n_neg, n_test, n_episodes_per_task):
		"""
		Parameters
		----------
		dataset: dc.data.Dataset
		Holds dataset from which support sets will be sampled.
		n_pos: int
		Number of positive samples
		n_neg: int
		Number of negative samples.
		n_test: int
		Number of samples in test set.
		n_episodes_per_task: int
		Number of (support, task) pairs to sample per task.
		replace: bool
		Whether to use sampling with or without replacement.
		"""
		time_start = time.time()
		self.tasks = range(len(dataset.get_task_names()) )
		self.n_tasks = len(self.tasks)
		self.n_episodes_per_task = n_episodes_per_task
		self.dataset = dataset
		self.n_pos = n_pos
		self.n_neg = n_neg
		self.task_episodes = {}

		for task in range(self.n_tasks):
		task_supports = get_task_support(
		self.dataset, n_episodes_per_task, n_pos, n_neg, task)
		task_tests = get_task_test(
		self.dataset, n_episodes_per_task, n_test, task)
		self.task_episodes[task] = (task_supports, task_tests)

		# Init the iterator
		self.perm_tasks = np.random.permutation(self.tasks)
		# Set initial iterator state
		self.task_num = 0
		self.trial_num = 0
		time_end = time.time()
		print("Constructing EpisodeGenerator took %s seconds"
		% str(time_end-time_start))

		def __iter__(self):
		return self

		def next(self):
		"""Sample next (support, test) pair.

		Return from internal storage.
		"""
		if self.trial_num == self.n_episodes_per_task:
		raise StopIteration
		else:
		task = self.perm_tasks[self.task_num] # Get id from permutation
		#support = self.supports[task][self.trial_num]
		task_supports, task_tests = self.task_episodes[task]
		support, test = (task_supports[self.trial_num],
		task_tests[self.trial_num])
		# Increment and update logic
		self.task_num += 1
		if self.task_num == self.n_tasks:
		self.task_num = 0 # Reset
		self.perm_tasks = np.random.permutation(self.tasks) # Permute again
		self.trial_num += 1 # Upgrade trial index

		return (task, support, test)


		class SupportGenerator(object):
		"""Generate support sets from a dataset.

		@@ -166,8 +335,6 @@ class SupportGenerator(object):
		def __iter__(self):
		return self

		# TODO(rbharath): This is generating data from one task at a time. Is it
		# wrong to have batches that mix information from multiple tasks?
		def next(self):
		"""Sample next support.

		@@ -179,7 +346,7 @@ class SupportGenerator(object):
		else:
		task = self.perm_tasks[self.task_num] # Get id from permutation
		#support = self.supports[task][self.trial_num]
		support = get_task_support(
		support = get_single_task_support(
		self.dataset, n_pos=self.n_pos, n_neg=self.n_neg, task=task,
		replace=self.replace)
		# Increment and update logic

deepchem/data/tests/test_support_generator.py

0 → 100644

+274 −0

Original line number	Diff line number	Diff line
		"""
		Simple Tests for Support Generation
		"""
		from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		__author__ = "Han Altae-Tran and Bharath Ramsundar"
		__copyright__ = "Copyright 2016, Stanford University"
		__license__ = "GPL"

		import numpy as np
		import unittest
		import tensorflow as tf
		import deepchem as dc
		class TestSupports(unittest.TestCase):
		"""
		Test that support generation happens properly.
		"""

		def test_get_task_support_simple(self):
		"""Tests that get_task_support samples correctly."""
		n_samples = 20
		n_features = 3
		n_tasks = 1
		n_trials = 10

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		n_episodes = 20
		n_pos = 1
		n_neg = 5
		supports = dc.data.get_task_support(dataset, n_episodes, n_pos, n_neg,
		task=0, log_every_n=10)
		assert len(supports) == n_episodes

		for support in supports:
		assert len(support) == n_pos + n_neg
		assert np.count_nonzero(support.y) == n_pos

		def test_get_task_support_missing(self):
		"""Test that task support works in presence of missing data."""
		n_samples = 20
		n_features = 3
		n_tasks = 1
		n_trials = 10

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		# Set last n_samples/2 weights to 0
		w[n_samples/2:] = 0
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		n_episodes = 20
		n_pos = 1
		n_neg = 2
		supports = dc.data.get_task_support(dataset, n_episodes, n_pos, n_neg,
		task=0, log_every_n=10)
		assert len(supports) == n_episodes

		for support in supports:
		assert len(support) == n_pos + n_neg
		assert np.count_nonzero(support.y) == n_pos
		# Check that no support elements are sample from zero-weight samples
		for identifier in support.ids:
		assert identifier < n_samples/2


		def test_get_task_test(self):
		"""Tests that get_task_testsamples correctly."""
		n_samples = 20
		n_features = 3
		n_tasks = 1
		n_trials = 10

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		n_episodes = 20
		n_test = 10
		tests = dc.data.get_task_test(dataset, n_episodes, n_test,
		task=0, log_every_n=10)

		assert len(tests) == n_episodes
		for test in tests:
		assert len(test) == n_test

		def test_simple_support_generator(self):
		"""Conducts simple test that support generator runs."""
		n_samples = 20
		n_features = 3
		n_tasks = 1
		n_pos = 1
		n_neg = 5
		n_trials = 10

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		# Create support generator
		supp_gen = dc.data.SupportGenerator(
		dataset, np.arange(n_tasks), n_pos, n_neg, n_trials, replace=True)

		def test_simple_episode_generator(self):
		"""Conducts simple test that episode generator runs."""
		n_samples = 20
		n_features = 3
		n_tasks = 1
		n_pos = 1
		n_neg = 5
		n_test = 10
		n_episodes = 10

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		# Create support generator
		episode_gen = dc.data.EpisodeGenerator(
		dataset, n_pos, n_neg, n_test, n_episodes)

		n_episodes_found = 0
		for (task, support, test) in episode_gen:
		assert task >= 0
		assert task < n_tasks
		assert len(support) == n_pos + n_neg
		assert np.count_nonzero(support.y) == n_pos
		assert len(test) == n_test
		n_episodes_found += 1
		assert n_episodes_found == n_episodes

		def test_get_task_minus_support_simple(self):
		"""Test that fixed index support can be removed from dataset."""
		n_samples = 20
		n_support = 5
		n_features = 3
		n_tasks = 1

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		support_dataset = dc.data.NumpyDataset(X[:n_support], y[:n_support],
		w[:n_support], ids[:n_support])

		task_dataset = dc.data.get_task_dataset_minus_support(
		dataset, support_dataset, task=0)

		# Assert all support elements have been removed
		assert len(task_dataset) == n_samples - n_support
		np.testing.assert_array_equal(task_dataset.X, X[n_support:])
		np.testing.assert_array_equal(task_dataset.y, y[n_support:])
		np.testing.assert_array_equal(task_dataset.w, w[n_support:])
		np.testing.assert_array_equal(task_dataset.ids, ids[n_support:])

		def test_get_task_minus_support(self):
		"""Test that random index support can be removed from dataset."""
		n_samples = 10
		n_support = 4
		n_features = 3
		n_tasks = 1

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		support_inds = sorted(np.random.choice(
		np.arange(n_samples), (n_support,), replace=False))
		support_dataset = dc.data.NumpyDataset(X[support_inds], y[support_inds],
		w[support_inds], ids[support_inds])

		task_dataset = dc.data.get_task_dataset_minus_support(
		dataset, support_dataset, task=0)

		# Assert all support elements have been removed
		data_inds = sorted(list(set(range(n_samples)) - set(support_inds)))
		assert len(task_dataset) == n_samples - n_support
		np.testing.assert_array_equal(task_dataset.X, X[data_inds])
		np.testing.assert_array_equal(task_dataset.y, y[data_inds])
		np.testing.assert_array_equal(task_dataset.w, w[data_inds])
		np.testing.assert_array_equal(task_dataset.ids, ids[data_inds])

		def test_get_task_minus_support_missing(self):
		"""Test that support can be removed from dataset with missing data"""
		n_samples = 20
		n_support = 4
		n_features = 3
		n_tasks = 1

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		# Set last n_samples/2 weights to 0
		w[n_samples/2:] = 0
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		# Sample from first n_samples/2 elements for support
		support_inds = sorted(np.random.choice(
		np.arange(n_samples/2), (n_support,), replace=False))
		support_dataset = dc.data.NumpyDataset(X[support_inds], y[support_inds],
		w[support_inds], ids[support_inds])

		task_dataset = dc.data.get_task_dataset_minus_support(
		dataset, support_dataset, task=0)

		# Should lie within first n_samples/2 samples only
		assert len(task_dataset) == n_samples/2 - n_support
		for identifier in task_dataset.ids:
		assert identifier < n_samples/2

		def test_support_generator_correct_samples(self):
		"""Tests that samples from support generator have desired shape."""
		n_samples = 20
		n_features = 3
		n_tasks = 1
		n_pos = 1
		n_neg = 5
		n_trials = 10

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.randint(2, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		# Create support generator
		supp_gen = dc.data.SupportGenerator(
		dataset, np.arange(n_tasks), n_pos, n_neg, n_trials, replace=True)
		num_supports = 0

		for (task, support) in supp_gen:
		assert support.X.shape == (n_pos + n_neg, n_features)
		num_supports += 1
		assert task == 0 # Only one task in this example
		n_supp_pos = np.count_nonzero(support.y)
		assert n_supp_pos == n_pos
		assert num_supports == n_trials

deepchem/models/tf_keras_models/support_classifier.py

+81 −12

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ from __future__ import unicode_literals
		import numpy as np
		import tensorflow as tf
		import sys
		import time
		from keras.layers import Input
		from keras import backend as K
		from deepchem.models import Model
		@@ -17,8 +18,9 @@ from deepchem.metrics import to_one_hot
		from deepchem.models.tf_keras_models.graph_topology import merge_dicts
		from deepchem.models.tensorflow_models import model_ops
		from deepchem.data import SupportGenerator
		from deepchem.data import get_task_test
		from deepchem.data import EpisodeGenerator
		from deepchem.data import get_task_dataset
		from deepchem.data import get_single_task_test
		from deepchem.data import get_task_dataset_minus_support

		class SupportGraphClassifier(Model):
		@@ -115,7 +117,7 @@ class SupportGraphClassifier(Model):
		feed_dict[K.learning_phase()] = training
		return feed_dict

		def fit(self, dataset, n_trials=1000, n_steps_per_trial=1, n_pos=1,
		def old_fit(self, dataset, n_trials=1000, n_steps_per_trial=1, n_pos=1,
		n_neg=9, log_every_n_samples=10, replace=True, **kwargs):
		"""Fits model on dataset.

		@@ -125,11 +127,6 @@ class SupportGraphClassifier(Model):
		task, and independently a test set from that same task. The
		SupportGenerator class iterates over the tasks in random order.

		# TODO(rbharath): Would it improve performance to sample multiple test sets
		for each support set or would that only harm performance?

		# TODO(rbharath): Should replace be an option for sampling the test sets?

		Parameters
		----------
		dataset: dc.data.Dataset
		@@ -147,9 +144,11 @@ class SupportGraphClassifier(Model):
		replace: bool, optional
		Whether or not to use replacement when sampling supports/tests.
		"""
		time_start = time.time()
		# Perform the optimization
		n_tasks = len(dataset.get_task_names())

		feed_total, run_total, test_total = 0, 0, 0
		# Create different support sets
		support_generator = SupportGenerator(dataset, range(n_tasks),
		n_pos, n_neg, n_trials, replace)
		@@ -158,12 +157,21 @@ class SupportGraphClassifier(Model):
		if ind % log_every_n_samples == 0:
		print("Sample %d from task %s" % (ind, str(task)))
		# Get batch to try it out on
		test = get_task_test(dataset, self.test_batch_size, task, replace)
		test_start = time.time()
		test = get_single_task_test(dataset, self.test_batch_size, task, replace)
		test_end = time.time()
		test_total += (test_end - test_start)
		feed_start = time.time()
		feed_dict = self.construct_feed_dict(test, support)
		feed_end = time.time()
		feed_total += (feed_end - feed_start)
		for step in range(n_steps_per_trial):
		# Train on support set, batch pair
		############################################################## DEBUG
		run_start = time.time()
		_, loss = self.sess.run([self.train_op, self.loss_op], feed_dict=feed_dict)
		run_end = time.time()
		run_total += (run_end - run_start)
		if ind % log_every_n_samples == 0:
		mean_loss = np.mean(np.array(recent_losses))
		print("\tmean loss is %s" % str(mean_loss))
		@@ -171,6 +179,69 @@ class SupportGraphClassifier(Model):
		else:
		recent_losses.append(loss)
		############################################################## DEBUG
		time_end = time.time()
		print("old_fit took %s seconds" % str(time_end-time_start))
		print("test_total: %s" % str(test_total))
		print("feed_total: %s" % str(feed_total))
		print("run_total: %s" % str(run_total))

		def fit(self, dataset, n_episodes_per_epoch=1000, nb_epochs=1, n_pos=1, n_neg=9,
		log_every_n_samples=10, **kwargs):
		"""Fits model on dataset using cached supports.

		For each epcoh, sample n_episodes_per_epoch (support, test) pairs and does
		gradient descent.

		Parameters
		----------
		dataset: dc.data.Dataset
		Dataset to fit model on.
		nb_epochs: int, optional
		number of epochs of training.
		n_episodes_per_epoch: int, optional
		Number of (support, test) pairs to sample and train on per epoch.
		n_pos: int, optional
		Number of positive examples per support.
		n_neg: int, optional
		Number of negative examples per support.
		log_every_n_samples: int, optional
		Displays info every this number of samples
		"""
		time_start = time.time()
		# Perform the optimization
		n_tasks = len(dataset.get_task_names())
		n_test = self.test_batch_size

		feed_total, run_total = 0, 0
		for epoch in range(nb_epochs):
		# Create different support sets
		episode_generator = EpisodeGenerator(dataset,
		n_pos, n_neg, n_test, n_episodes_per_epoch)
		recent_losses = []
		for ind, (task, support, test) in enumerate(episode_generator):
		if ind % log_every_n_samples == 0:
		print("Epoch %d, Sample %d from task %s" % (epoch, ind, str(task)))
		# Get batch to try it out on
		feed_start = time.time()
		feed_dict = self.construct_feed_dict(test, support)
		feed_end = time.time()
		feed_total += (feed_end - feed_start)
		# Train on support set, batch pair
		run_start = time.time()
		_, loss = self.sess.run([self.train_op, self.loss_op], feed_dict=feed_dict)
		run_end = time.time()
		run_total += (run_end - run_start)
		if ind % log_every_n_samples == 0:
		mean_loss = np.mean(np.array(recent_losses))
		print("\tmean loss is %s" % str(mean_loss))
		recent_losses = []
		else:
		recent_losses.append(loss)
		time_end = time.time()
		print("fit took %s seconds" % str(time_end-time_start))
		print("feed_total: %s" % str(feed_total))
		print("run_total: %s" % str(run_total))


		def save(self):
		"""Save all models
		@@ -348,6 +419,8 @@ class SupportGraphClassifier(Model):
		n_pos, n_neg, n_trials, replace)
		for ind, (task, support) in enumerate(support_generator):
		print("Eval sample %d from task %s" % (ind, str(task)))
		# TODO(rbharath): Add test for get_task_dataset_minus_support for
		# multitask case with missing data...
		if exclude_support:
		print("Removing support datapoints for eval.")
		task_dataset = get_task_dataset_minus_support(dataset, support, task)
		@@ -355,10 +428,6 @@ class SupportGraphClassifier(Model):
		print("Keeping support datapoints for eval.")
		task_dataset = get_task_dataset(dataset, task)
		y_pred = self.predict_proba(support, task_dataset)
		######################################################### DEBUG
		#print("task_dataset.y.shape, y_pred.shape, task_dataset.w.shape")
		#print(task_dataset.y.shape, y_pred.shape, task_dataset.w.shape)
		######################################################### DEBUG
		task_scores[task].append(metric.compute_metric(
		task_dataset.y, y_pred, task_dataset.w))

deepchem/models/tf_keras_models/tests/test_support_generator.py

deleted100644 → 0

+0 −100

File deleted.

Preview size limit exceeded, changes collapsed.

deepchem/splits/splitters.py

+5 −0

Original line number	Diff line number	Diff line
		@@ -331,6 +331,11 @@ class IndexSplitter(Splitter):
		train_cutoff = int(frac_train * num_datapoints)
		valid_cutoff = int((frac_train + frac_valid) * num_datapoints)
		indices = range(num_datapoints)
		################################################################ DEBUG
		print("IndexSplitter.split()")
		print("train_cutoff, valid_cutoff, len(dataset)")
		print(train_cutoff, valid_cutoff, len(dataset))
		################################################################ DEBUG
		return (indices[:train_cutoff], indices[train_cutoff:valid_cutoff],
		indices[valid_cutoff:])

Admin message