Merge pull request #147 from rbharath/balancing_transformer (7fc142d4) · Commits · 钟慕尧 / deepchem

deepchem/datasets/init.py

+21 −8

Original line number	Diff line number	Diff line
		@@ -66,7 +66,7 @@ class Dataset(object):
		metadata_rows,
		columns=('df_file', 'task_names', 'ids',
		'X', 'X-transformed', 'y', 'y-transformed',
		'w',
		'w', 'w-transformed',
		'X_sums', 'X_sum_squares', 'X_n',
		'y_sums', 'y_sum_squares', 'y_n'))
		self.save_to_disk()
		@@ -87,12 +87,14 @@ class Dataset(object):
		out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)
		out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)
		out_w = os.path.join(data_dir, "%s-w.joblib" % basename)
		out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)
		out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

		metadata_rows = []
		retval = ([df_file, tasks, out_ids, out_X,
		out_X_transformed, out_y,
		out_y_transformed, out_w,
		retval = ([df_file, tasks, out_ids,
		out_X, out_X_transformed,
		out_y, out_y_transformed,
		out_w, out_w_transformed,
		out_X_sums, out_X_sum_squares, out_X_n,
		out_y_sums, out_y_sum_squares, out_y_n])
		metadata_rows.append(retval)
		@@ -101,7 +103,7 @@ class Dataset(object):
		metadata_rows,
		columns=('df_file','task_names', 'ids',
		'X', 'X-transformed', 'y', 'y-transformed',
		'w',
		'w', 'w-transformed',
		'X_sums', 'X_sum_squares', 'X_n',
		'y_sums', 'y_sum_squares', 'y_n'))
		self.save_to_disk()
		@@ -159,7 +161,7 @@ class Dataset(object):
		for _, row in self.metadata_df.iterrows():
		X = load_from_disk(row['X-transformed'])
		y = load_from_disk(row['y-transformed'])
		w = load_from_disk(row['w'])
		w = load_from_disk(row['w-transformed'])
		ids = load_from_disk(row['ids'])
		yield (X, y, w, ids)

		@@ -213,6 +215,15 @@ class Dataset(object):
		ys.append(y_b)
		return np.vstack(ys)

		def get_weights(self):
		"""
		Returns all weights for this dataset.
		"""
		ws = []
		for (_, _, w_b, _) in self.itershards():
		ws.append(w_b)
		return np.vstack(ws)

		def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size):
		"""Fix batch to have exactly batch_size elements.

		@@ -339,10 +350,12 @@ def write_dataset_single(val, data_dir, feature_types, tasks):
		out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)
		out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)
		out_w = os.path.join(data_dir, "%s-w.joblib" % basename)
		out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)
		out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

		save_to_disk(X, out_X)
		save_to_disk(y, out_y)
		save_to_disk(w, out_w)
		# Write moments to disk
		save_to_disk(X_sums, out_X_sums)
		save_to_disk(X_sum_squares, out_X_sum_squares)
		@@ -353,12 +366,12 @@ def write_dataset_single(val, data_dir, feature_types, tasks):
		# Write X, y as transformed versions
		save_to_disk(X, out_X_transformed)
		save_to_disk(y, out_y_transformed)
		save_to_disk(w, out_w)
		save_to_disk(w, out_w_transformed)
		save_to_disk(ids, out_ids)
		# TODO(rbharath): Should X be saved to out_X_transformed as well? Since
		# itershards expects to loop over X-transformed? (Ditto for y/w)
		return([df_file, task_names, out_ids, out_X, out_X_transformed, out_y,
		out_y_transformed, out_w,
		out_y_transformed, out_w, out_w_transformed,
		out_X_sums, out_X_sum_squares, out_X_n,
		out_y_sums, out_y_sum_squares, out_y_n])

deepchem/datasets/tests/init.py

+14 −0

Original line number	Diff line number	Diff line
		@@ -80,6 +80,20 @@ class TestDatasetAPI(unittest.TestCase):
		input_transformer_classes, output_transformer_classes,
		input_file, task_types.keys())

		def _load_classification_data(self):
		"""Loads classification data from example.csv"""
		compound_featurizers = [CircularFingerprint(size=1024)]
		complex_featurizers = []
		input_transformer_classes = []
		output_transformer_classes = []
		task_types = {"outcome": "classification"}
		input_file = "example_classification.csv"
		return self._create_dataset(
		compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, task_types.keys())


		def _load_multitask_data(self):
		"""Load example multitask data."""
		compound_featurizers = [CircularFingerprint(size=1024)]

deepchem/transformers/init.py

+67 −12

Original line number	Diff line number	Diff line
		@@ -21,14 +21,17 @@ class Transformer(object):
		# Hack to allow for easy unpickling:
		# http://stefaanlippens.net/pickleproblem
		__module__ = os.path.splitext(os.path.basename(__file__))[0]
		def __init__(self, transform_X=False, transform_y=False, dataset=None):
		def __init__(self, transform_X=False, transform_y=False, transform_w=False,
		dataset=None):
		"""Initializes transformation based on dataset statistics."""
		self.dataset = dataset
		self.transform_X = transform_X
		self.transform_y = transform_y
		self.transform_w = transform_w
		# One, but not both, transform_X or tranform_y is true
		assert transform_X or transform_y
		assert not (transform_X and transform_y)
		assert transform_X or transform_y or transform_w
		# Use fact that bools add as ints in python
		assert (transform_X + transform_y + transform_w) == 1

		def transform_row(self, i, df):
		"""
		@@ -73,10 +76,12 @@ def _transform_row(i, df, transformer):

		class NormalizationTransformer(Transformer):

		def __init__(self, transform_X=False, transform_y=False, dataset=None):
		def __init__(self, transform_X=False, transform_y=False, transform_w=False,
		dataset=None):
		"""Initialize normalization transformation."""
		super(NormalizationTransformer, self).__init__(
		transform_X=transform_X, transform_y=transform_y, dataset=dataset)
		transform_X=transform_X, transform_y=transform_y,
		transform_w=transform_w, dataset=dataset)
		X_means, X_stds, y_means, y_stds = dataset.get_statistics()
		self.X_means = X_means
		self.X_stds = X_stds
		@@ -121,11 +126,12 @@ class NormalizationTransformer(Transformer):

		class ClippingTransformer(Transformer):

		def __init__(self, transform_X=False, transform_y=False, dataset=None,
		max_val=5.):
		def __init__(self, transform_X=False, transform_y=False,
		transform_w=False, dataset=None, max_val=5.):
		"""Initialize clipping transformation."""
		super(ClippingTransformer, self).__init__(transform_X=transform_X,
		transform_y=transform_y,
		transform_w=transform_w,
		dataset=dataset)
		self.max_val = max_val

		@@ -168,13 +174,62 @@ class LogTransformer(Transformer):
		"""Undoes the logarithmic transformation."""
		return np.exp(z)

		class BalancingTransformer(Transformer):
		"""Balance positive and negative examples for weights."""
		def __init__(self, transform_X=False, transform_y=False,
		transform_w=False, dataset=None, seed=None):
		super(BalancingTransformer, self).__init__(
		transform_X=transform_X, transform_y=transform_y,
		transform_w=transform_w, dataset=dataset)
		# BalancingTransformer can only transform weights.
		assert not transform_X
		assert not transform_y
		assert transform_w

		# Compute weighting factors from dataset.
		y = self.dataset.get_labels()
		w = self.dataset.get_weights()
		# Ensure dataset is binary
		np.testing.assert_allclose(sorted(np.unique(y)), np.array([0., 1.]))
		weights = []
		for ind, task in enumerate(self.dataset.get_task_names()):
		task_w = w[:, ind]
		task_y = y[:, ind]
		# Remove labels with zero weights
		task_y = task_y[task_w != 0]
		num_positives = np.count_nonzero(task_y)
		num_negatives = len(task_y) - num_positives
		if num_positives > 0:
		pos_weight = float(num_negatives)/num_positives
		else:
		pos_weight = 1
		neg_weight = 1
		weights.append((neg_weight, pos_weight))
		self.weights = weights

		def transform_row(self, i, df):
		"""Reweight the labels for this data."""
		row = df.iloc[i]
		y = load_from_disk(row['y-transformed'])
		w = load_from_disk(row['w-transformed'])
		w_balanced = np.zeros_like(w)
		for ind, task in enumerate(self.dataset.get_task_names()):
		task_y = y[:, ind]
		task_w = w[:, ind]
		zero_indices = np.logical_and(task_y==0, task_w != 0)
		one_indices = np.logical_and(task_y==1, task_w != 0)
		w_balanced[zero_indices, ind] = self.weights[ind][0]
		w_balanced[one_indices, ind] = self.weights[ind][1]
		save_to_disk(w_balanced, row['w-transformed'])

		class CoulombRandomizationTransformer(Transformer):

		def __init__(self, transform_X=False, transform_y=False, dataset=None,
		seed=None):
		def __init__(self, transform_X=False, transform_y=False,
		transform_w=False, dataset=None, seed=None):
		"""Iniitialize coulomb matrix randomization transformation. """
		super(CoulombRandomizationTransformer, self).__init__(
		transform_X=transform_X, transform_y=transform_y, dataset=dataset)
		transform_X=transform_X, transform_y=transform_y,
		transform_w=transform_w, dataset=dataset)
		self.seed = seed

		def construct_cm_from_triu(self, x):
		@@ -238,7 +293,8 @@ class CoulombRandomizationTransformer(Transformer):

		class CoulombBinarizationTransformer(Transformer):

		def __init__(self, transform_X=False, transform_y=False, dataset=None,
		def __init__(self, transform_X=False, transform_y=False,
		transform_w=False, dataset=None,
		theta=1, update_state=True):
		"""Initialize binarization transformation."""
		super(CoulombBinarizationTransformer, self).__init__(
		@@ -277,7 +333,6 @@ class CoulombBinarizationTransformer(Transformer):
		"""
		Binarizes data in dataset with sigmoid function
		"""

		row = df.iloc[i]
		X_bin = []
		if self.update_state:

deepchem/transformers/tests/init.py

0 → 100644

+0 −0

Empty file added.

deepchem/transformers/tests/test_transformers.py

0 → 100644

+172 −0

Original line number	Diff line number	Diff line
		"""
		Tests for transformer objects.
		"""
		from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		__author__ = "Bharath Ramsundar"
		__copyright__ = "Copyright 2016, Stanford University"
		__license__ = "LGPL"

		import unittest
		import numpy as np
		from deepchem.transformers import LogTransformer
		from deepchem.transformers import NormalizationTransformer
		from deepchem.transformers import BalancingTransformer
		from deepchem.datasets.tests import TestDatasetAPI

		class TestTransformerAPI(TestDatasetAPI):
		"""
		Test top-level API for transformer objects.
		"""

		def test_y_log_transformer(self):
		"""Tests logarithmic data transformer."""
		solubility_dataset = self._load_solubility_data()
		log_transformer = LogTransformer(
		transform_y=True, dataset=solubility_dataset)
		X, y, w, ids = solubility_dataset.to_numpy()
		log_transformer.transform(solubility_dataset)
		X_t, y_t, w_t, ids_t = solubility_dataset.to_numpy()

		# Check ids are unchanged.
		for id_elt, id_t_elt in zip(ids, ids_t):
		assert id_elt == id_t_elt
		# Check X is unchanged since this is a y transformer
		np.testing.assert_allclose(X, X_t)
		# Check w is unchanged since this is a y transformer
		np.testing.assert_allclose(w, w_t)
		# Check y is now a logarithmic version of itself
		np.testing.assert_allclose(y_t, np.log(y))

		# Check that untransform does the right thing.
		np.testing.assert_allclose(log_transformer.untransform(y_t), y)

		def test_X_log_transformer(self):
		"""Tests logarithmic data transformer."""
		solubility_dataset = self._load_solubility_data()
		log_transformer = LogTransformer(
		transform_X=True, dataset=solubility_dataset)
		X, y, w, ids = solubility_dataset.to_numpy()
		log_transformer.transform(solubility_dataset)
		X_t, y_t, w_t, ids_t = solubility_dataset.to_numpy()

		# Check ids are unchanged.
		for id_elt, id_t_elt in zip(ids, ids_t):
		assert id_elt == id_t_elt
		# Check y is unchanged since this is a X transformer
		np.testing.assert_allclose(y, y_t)
		# Check w is unchanged since this is a y transformer
		np.testing.assert_allclose(w, w_t)
		# Check y is now a logarithmic version of itself
		np.testing.assert_allclose(X_t, np.log(X))

		# Check that untransform does the right thing.
		np.testing.assert_allclose(log_transformer.untransform(X_t), X)

		def test_y_normalization_transformer(self):
		"""Tests normalization transformer."""
		solubility_dataset = self._load_solubility_data()
		normalization_transformer = NormalizationTransformer(
		transform_y=True, dataset=solubility_dataset)
		X, y, w, ids = solubility_dataset.to_numpy()
		normalization_transformer.transform(solubility_dataset)
		X_t, y_t, w_t, ids_t = solubility_dataset.to_numpy()
		# Check ids are unchanged.
		for id_elt, id_t_elt in zip(ids, ids_t):
		assert id_elt == id_t_elt
		# Check X is unchanged since this is a y transformer
		np.testing.assert_allclose(X, X_t)
		# Check w is unchanged since this is a y transformer
		np.testing.assert_allclose(w, w_t)
		# Check that y_t has zero mean, unit std.
		assert np.isclose(y_t.mean(), 0.)
		assert np.isclose(y_t.std(), 1.)

		# Check that untransform does the right thing.
		np.testing.assert_allclose(normalization_transformer.untransform(y_t), y)

		def test_X_normalization_transformer(self):
		"""Tests normalization transformer."""
		solubility_dataset = self._load_solubility_data()
		normalization_transformer = NormalizationTransformer(
		transform_X=True, dataset=solubility_dataset)
		X, y, w, ids = solubility_dataset.to_numpy()
		normalization_transformer.transform(solubility_dataset)
		X_t, y_t, w_t, ids_t = solubility_dataset.to_numpy()
		# Check ids are unchanged.
		for id_elt, id_t_elt in zip(ids, ids_t):
		assert id_elt == id_t_elt
		# Check y is unchanged since this is a X transformer
		np.testing.assert_allclose(y, y_t)
		# Check w is unchanged since this is a y transformer
		np.testing.assert_allclose(w, w_t)
		# Check that X_t has zero mean, unit std.
		#np.set_printoptions(threshold='nan')
		mean = X_t.mean(axis=0)
		assert np.amax(np.abs(mean-np.zeros_like(mean))) < 1e-7
		orig_std_array = X.std(axis=0)
		std_array = X_t.std(axis=0)
		# Entries with zero std are not normalized
		for orig_std, std in zip(orig_std_array, std_array):
		if not np.isclose(orig_std, 0):
		assert np.isclose(std, 1)

		# TODO(rbharath): Untransform doesn't work properly for binary feature
		# vectors. Need to figure out what's wrong here. (low priority)
		## Check that untransform does the right thing.
		#np.testing.assert_allclose(normalization_transformer.untransform(X_t), X)

		def test_singletask_balancing_transformer(self):
		"""Test balancing transformer on single-task dataset."""

		classification_dataset = self._load_classification_data()
		balancing_transformer = BalancingTransformer(
		transform_w=True, dataset=classification_dataset)
		X, y, w, ids = classification_dataset.to_numpy()
		balancing_transformer.transform(classification_dataset)
		X_t, y_t, w_t, ids_t = classification_dataset.to_numpy()
		# Check ids are unchanged.
		for id_elt, id_t_elt in zip(ids, ids_t):
		assert id_elt == id_t_elt
		# Check X is unchanged since this is a w transformer
		np.testing.assert_allclose(X, X_t)
		# Check y is unchanged since this is a w transformer
		np.testing.assert_allclose(y, y_t)
		for ind, task in enumerate(classification_dataset.get_task_names()):
		y_task = y_t[:, ind]
		w_task = w_t[:, ind]
		w_orig_task = w[:, ind]
		# Assert that entries with zero weight retain zero weight
		np.testing.assert_allclose(
		w_task[w_orig_task == 0], np.zeros_like(w_task[w_orig_task == 0]))
		# Check that sum of 0s equals sum of 1s in transformed for each task
		assert np.isclose(np.sum(w_task[y_task == 0]),
		np.sum(w_task[y_task == 1]))

		def test_multitask_balancing_transformer(self):
		"""Test balancing transformer on multitask dataset."""
		multitask_dataset = self._load_multitask_data()
		balancing_transformer = BalancingTransformer(
		transform_w=True, dataset=multitask_dataset)
		X, y, w, ids = multitask_dataset.to_numpy()
		balancing_transformer.transform(multitask_dataset)
		X_t, y_t, w_t, ids_t = multitask_dataset.to_numpy()
		# Check ids are unchanged.
		for id_elt, id_t_elt in zip(ids, ids_t):
		assert id_elt == id_t_elt
		# Check X is unchanged since this is a w transformer
		np.testing.assert_allclose(X, X_t)
		# Check y is unchanged since this is a w transformer
		np.testing.assert_allclose(y, y_t)
		for ind, task in enumerate(multitask_dataset.get_task_names()):
		y_task = y_t[:, ind]
		w_task = w_t[:, ind]
		w_orig_task = w[:, ind]
		# Assert that entries with zero weight retain zero weight
		np.testing.assert_allclose(
		w_task[w_orig_task == 0], np.zeros_like(w_task[w_orig_task == 0]))
		# Check that sum of 0s equals sum of 1s in transformed for each task
		assert np.isclose(np.sum(w_task[y_task == 0]),
		np.sum(w_task[y_task == 1]))

Admin message