Preliminary CDF and Power Transform work (9b00d5a6) · Commits · 钟慕尧 / deepchem

deepchem/datasets/tests/init.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -111,14 +111,14 @@ class TestDatasetAPI(TestAPI):
		"""Load example with numerical features sampled from Gaussian normal distribution."""
		if os.path.exists(self.data_dir):
		shutil.rmtree(self.data_dir)
		features = ["feat0"]
		features = ["feat0","feat1"]
		featurizer = UserDefinedFeaturizer(features)
		tasks = ["task0"]
		tasks = ["task0","task1"]
		input_file = os.path.join(
		self.current_dir, "../../models/tests/gaussian_cdf_example.csv")
		loader = DataLoader(
		tasks=tasks,
		featurizer=featurizer,
		id_field="id",
		verbosity="low")
		verbosity=None)
		return loader.featurize(input_file, self.data_dir)

deepchem/models/tests/gaussian_cdf_example.csv

+1002 −1002

File changed.

Preview size limit exceeded, changes collapsed.

deepchem/transformers/init.py

+59 −12

Original line number	Diff line number	Diff line
		@@ -12,7 +12,7 @@ from deepchem.utils.save import save_to_disk
		from deepchem.utils.save import load_from_disk
		from deepchem.utils import pad_array
		import shutil
		from deepchem.datasets import Dataset
		from deepchem.datasets import DiskDataset

		def undo_transforms(y, transformers):
		"""Undoes all transformations applied."""
		@@ -470,7 +470,7 @@ class CDFTransformer(Transformer):

		def transform(self, dataset, bins):
		"""Performs CDF transform on data."""
		X, y, w, ids = dataset.to_numpy()
		X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids)
		w_t = w
		ids_t = ids
		if self.transform_X:
		@@ -480,31 +480,78 @@ class CDFTransformer(Transformer):
		y_t = get_cdf_values(y,self.bins)
		X_t = X
		# TODO (rbharath): Find a more elegant solution to saving the data?
		# shutil.rmtree(dataset.data_dir)
		# os.makedirs(dataset.data_dir)
		# Dataset.from_numpy(dataset.data_dir, X_t, y_t, w_t, ids_t)

		return X_t, y_t, w_t, ids_t
		shutil.rmtree(dataset.data_dir)
		os.makedirs(dataset.data_dir)
		DiskDataset.from_numpy(dataset.data_dir, X_t, y_t, w_t, ids_t)

		def untransform(self, z):
		print("Cannot undo CDF Transformer.")
		# Need this for transform_y

		def get_cdf_values(array, bins):
		n_cols = array.shape[1]
		#array = np.transpose(array)
		n_rows = array.shape[0]
		n_cols = array.shape[1]
		array_t = np.zeros((n_rows,n_cols))
		parts = n_rows/bins
		hist_values = np.zeros(n_rows)
		sorted_hist_values = np.zeros(n_rows)
		for row in xrange(n_rows):
		for row in range(n_rows):
		if np.remainder(bins,2)==1:
		hist_values[row] = np.floor(np.divide(row,parts))/(bins-1)
		else:
		hist_values[row] = np.floor(np.divide(row,parts))/bins
		for col in xrange(n_cols):
		for col in range(n_cols):
		order = np.argsort(array[:,col], axis=0)
		sorted_hist_values = hist_values[order]
		array_t[:,col] = sorted_hist_values

		return array_t

		class PowerTransformer(Transformer):
		"""Takes power n transform of a column and adds it as a new column."""
		def __init__(self, transform_X=False,
		transform_y=False, features=None, tasks=None,
		n_powers=1, powers=[1]):
		self.transform_X = transform_X
		self.transform_y = transform_y
		self.features = features
		self.tasks = tasks
		self.n_powers = n_powers
		self.powers = powers
		if len(self.powers) != self.n_powers:
		print("Number of powers in list powers is not equal to n_powers.")

		def transform(self, dataset, n_powers, powers):
		"""Performs power transform on data."""
		X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids)
		X_t = X
		y_t = y
		w_t = w
		ids_t = ids
		if self.transform_X:
		for i in range(self.n_powers):
		X_temp = np.power(X, self.powers[i])
		X_t = np.append(X_t, X_temp, axis=1)
		new_features = self.features
		#for j in range(len(self.features)):

		if self.transform_y:
		for i in range(self.n_powers):
		y_temp = np.power(y, self.powers[i])
		y_t = np.append(y_t, y_temp, axis=1)
		new_tasks = self.tasks
		#for j in range(len(self.tasks)):

		# TODO (rbharath): Find a more elegant solution to saving the data?
		shutil.rmtree(dataset.data_dir)
		os.makedirs(dataset.data_dir)
		DiskDataset.from_numpy(dataset.data_dir, X_t, y_t, w_t, ids_t)

		def untransform(self, z):
		if self.transform_X:
		orig_len = (z.shape[1])/(self.n_powers+1)
		z = z[:,:orig_len]
		if self.transform_y:
		orig_len = (z.shape[1])/(self.n_powers+1)
		z = z[:,:orig_len]
		No newline at end of file

deepchem/transformers/tests/test_transformers.py

+31 −8

Original line number	Diff line number	Diff line
		@@ -14,6 +14,7 @@ import numpy as np
		import pandas as pd
		import numpy.random as random
		import os
		from deepchem.datasets import Dataset
		from deepchem.transformers import LogTransformer
		from deepchem.transformers import NormalizationTransformer
		from deepchem.transformers import BalancingTransformer
		@@ -184,20 +185,18 @@ class TestTransformerAPI(TestDatasetAPI):
		## Check that untransform does the right thing.
		#np.testing.assert_allclose(normalization_transformer.untransform(X_t), X)

		def test_cdf_transformer(self):
		def test_cdf_X_transformer(self):
		"""Test CDF transformer on Gaussian normal dataset."""
		target = np.array(np.transpose(np.linspace(0.,1.,1001)))
		target = np.array([target])
		target = np.transpose(np.array(np.append([target],[target], axis=0)))
		gaussian_dataset = self.load_gaussian_cdf_data()
		bins=1001
		cdf_transformer = CDFTransformer(
		transform_X=True, bins=bins)
		#cdf_transformer = CDFTransformer(
		# transform_X=True, bins=1001, dataset=gaussian_dataset)
		# cdf_transformer.transform(gaussian_dataset)
		# X_t, y_t, w_t, ids_t = gaussian_dataset.to_numpy()
		X_t, y_t, w_t, ids_t = cdf_transformer.transform(gaussian_dataset,bins=bins)
		X, y, w, ids = gaussian_dataset.to_numpy()
		cdf_transformer.transform(gaussian_dataset, bins=bins)
		gaussian_dataset = Dataset(data_dir=gaussian_dataset.data_dir, reload=True)
		X_t, y_t, w_t, ids_t = gaussian_dataset.to_numpy()

		# Check ids are unchanged.
		for id_elt, id_t_elt in zip(ids, ids_t):
		@@ -207,7 +206,31 @@ class TestTransformerAPI(TestDatasetAPI):
		# Check w is unchanged since this is an X transformer
		np.testing.assert_allclose(w, w_t)
		# Check X is now holding the proper values when sorted.
		sorted = np.sort(X_t)
		sorted = np.sort(X_t,axis=0)
		np.testing.assert_allclose(sorted, target)

		def test_cdf_y_transformer(self):
		"""Test CDF transformer on Gaussian normal dataset."""
		target = np.array(np.transpose(np.linspace(0.,1.,1001)))
		target = np.transpose(np.array(np.append([target],[target], axis=0)))
		gaussian_dataset = self.load_gaussian_cdf_data()
		bins=1001
		cdf_transformer = CDFTransformer(
		transform_y=True, bins=bins)
		X, y, w, ids = gaussian_dataset.to_numpy()
		cdf_transformer.transform(gaussian_dataset, bins=bins)
		gaussian_dataset = Dataset(data_dir=gaussian_dataset.data_dir, reload=True)
		X_t, y_t, w_t, ids_t = gaussian_dataset.to_numpy()

		# Check ids are unchanged.
		for id_elt, id_t_elt in zip(ids, ids_t):
		assert id_elt == id_t_elt
		# Check X is unchanged since this is an y transformer
		np.testing.assert_allclose(X, X_t)
		# Check w is unchanged since this is an y transformer
		np.testing.assert_allclose(w, w_t)
		# Check y is now holding the proper values when sorted.
		sorted = np.sort(y_t,axis=0)
		np.testing.assert_allclose(sorted, target)

		def test_singletask_balancing_transformer(self):

Admin message