Merge pull request #191 from rbharath/relative_paths (eea57425) · Commits · 钟慕尧 / deepchem

deepchem/datasets/init.py

+121 −113

Original line number	Diff line number	Diff line
		@@ -92,19 +92,19 @@ class Dataset(object):
		# Create an empty metadata dataframe to be filled at a later time
		basename = "metadata"
		df_file = "metadata.joblib"
		out_X = os.path.join(data_dir, "%s-X.joblib" % basename)
		out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename)
		out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename)
		out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename)
		out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename)
		out_y = os.path.join(data_dir, "%s-y.joblib" % basename)
		out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename)
		out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename)
		out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)
		out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)
		out_w = os.path.join(data_dir, "%s-w.joblib" % basename)
		out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)
		out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)
		out_X = "%s-X.joblib" % basename
		out_X_transformed = "%s-X-transformed.joblib" % basename
		out_X_sums = "%s-X_sums.joblib" % basename
		out_X_sum_squares = "%s-X_sum_squares.joblib" % basename
		out_X_n = "%s-X_n.joblib" % basename
		out_y = "%s-y.joblib" % basename
		out_y_transformed = "%s-y-transformed.joblib" % basename
		out_y_sums = "%s-y_sums.joblib" % basename
		out_y_sum_squares = "%s-y_sum_squares.joblib" % basename
		out_y_n = "%s-y_n.joblib" % basename
		out_w = "%s-w.joblib" % basename
		out_w_transformed = "%s-w-transformed.joblib" % basename
		out_ids = "%s-ids.joblib" % basename

		metadata_rows = []
		retval = ([df_file, tasks, out_ids,
		@@ -151,7 +151,9 @@ class Dataset(object):
		if not len(self.metadata_df):
		raise ValueError("No data in dataset.")
		sample_X = load_from_disk(
		self.metadata_df.iterrows().next()[1]['X-transformed'])[0]
		os.path.join(
		self.data_dir,
		self.metadata_df.iterrows().next()[1]['X-transformed']))[0]
		return np.shape(sample_X)

		def _get_metadata_filename(self):
		@@ -176,10 +178,14 @@ class Dataset(object):
		The order of shards returned is guaranteed to remain fixed.
		"""
		for _, row in self.metadata_df.iterrows():
		X = np.array(load_from_disk(row['X-transformed']))
		y = np.array(load_from_disk(row['y-transformed']))
		w = np.array(load_from_disk(row['w-transformed']))
		ids = np.array(load_from_disk(row['ids']), dtype=object)
		X = np.array(load_from_disk(
		os.path.join(self.data_dir, row['X-transformed'])))
		y = np.array(load_from_disk(
		os.path.join(self.data_dir, row['y-transformed'])))
		w = np.array(load_from_disk(
		os.path.join(self.data_dir, row['w-transformed'])))
		ids = np.array(load_from_disk(
		os.path.join(self.data_dir, row['ids'])), dtype=object)
		yield (X, y, w, ids)

		def iterbatches(self, batch_size=None, epoch=0):
		@@ -267,7 +273,7 @@ class Dataset(object):
		"""
		total = 0
		for _, row in self.metadata_df.iterrows():
		y = load_from_disk(row['y-transformed'])
		y = load_from_disk(os.path.join(self.data_dir, row['y-transformed']))
		total += len(y)
		return total

		@@ -285,13 +291,80 @@ class Dataset(object):
		return None, None, None, None
		self.update_moments()
		df = self.metadata_df
		X_means, X_stds, y_means, y_stds = compute_mean_and_std(df)
		X_means, X_stds, y_means, y_stds = self._compute_mean_and_std(df)
		return X_means, X_stds, y_means, y_stds

		def _compute_mean_and_std(self, df):
		"""
		Compute means/stds of X/y from sums/sum_squares of tensors.
		"""

		X_sums = []
		X_sum_squares = []
		X_n = []
		for _, row in df.iterrows():
		Xs = load_from_disk(os.path.join(self.data_dir, row['X_sums']))
		Xss = load_from_disk(os.path.join(self.data_dir, row['X_sum_squares']))
		Xn = load_from_disk(os.path.join(self.data_dir, row['X_n']))
		X_sums.append(np.array(Xs))
		X_sum_squares.append(np.array(Xss))
		X_n.append(np.array(Xn))

		# Note that X_n is a list of floats
		n = float(np.sum(X_n))
		X_sums = np.vstack(X_sums)
		X_sum_squares = np.vstack(X_sum_squares)
		overall_X_sums = np.sum(X_sums, axis=0)
		overall_X_means = overall_X_sums / n
		overall_X_sum_squares = np.sum(X_sum_squares, axis=0)

		X_vars = (overall_X_sum_squares - np.square(overall_X_sums)/n)/(n)

		y_sums = []
		y_sum_squares = []
		y_n = []
		for _, row in df.iterrows():
		ys = load_from_disk(os.path.join(self.data_dir, row['y_sums']))
		yss = load_from_disk(os.path.join(self.data_dir, row['y_sum_squares']))
		yn = load_from_disk(os.path.join(self.data_dir, row['y_n']))
		y_sums.append(np.array(ys))
		y_sum_squares.append(np.array(yss))
		y_n.append(np.array(yn))

		# Note y_n is a list of arrays of shape (n_tasks,)
		y_n = np.sum(y_n, axis=0)
		y_sums = np.vstack(y_sums)
		y_sum_squares = np.vstack(y_sum_squares)
		y_means = np.sum(y_sums, axis=0)/y_n
		y_vars = np.sum(y_sum_squares, axis=0)/y_n - np.square(y_means)
		return overall_X_means, np.sqrt(X_vars), y_means, np.sqrt(y_vars)


		def update_moments(self):
		"""Re-compute statistics of this dataset during transformation"""
		df = self.metadata_df
		update_mean_and_std(df)
		self._update_mean_and_std(df)

		def _update_mean_and_std(self, df):
		"""
		Compute means/stds of X/y from sums/sum_squares of tensors.
		"""
		X_transform = []
		for _, row in df.iterrows():
		Xt = load_from_disk(os.path.join(self.data_dir, row['X-transformed']))
		Xs = np.sum(Xt,axis=0)
		Xss = np.sum(np.square(Xt),axis=0)
		save_to_disk(Xs, os.path.join(self.data_dir, row['X_sums']))
		save_to_disk(Xss, os.path.join(self.data_dir, row['X_sum_squares']))

		y_transform = []
		for _, row in df.iterrows():
		yt = load_from_disk(os.path.join(self.data_dir, row['y-transformed']))
		ys = np.sum(yt,axis=0)
		yss = np.sum(np.square(yt),axis=0)
		save_to_disk(ys, os.path.join(self.data_dir, row['y_sums']))
		save_to_disk(yss, os.path.join(self.data_dir, row['y_sum_squares']))



		def compute_sums_and_nb_sample(tensor, W=None):
		@@ -346,35 +419,35 @@ def write_dataset_single(val, data_dir, feature_types=None, tasks=None,

		if feature_types is not None and tasks is not None:
		basename = os.path.splitext(os.path.basename(df_file))[0]
		out_X = os.path.join(data_dir, "%s-X.joblib" % basename)
		out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename)
		out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename)
		out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename)
		out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename)
		out_y = os.path.join(data_dir, "%s-y.joblib" % basename)
		out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename)
		out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename)
		out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)
		out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)
		out_w = os.path.join(data_dir, "%s-w.joblib" % basename)
		out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)
		out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

		save_to_disk(X, out_X)
		save_to_disk(y, out_y)
		save_to_disk(w, out_w)
		out_X = "%s-X.joblib" % basename
		out_X_transformed = "%s-X-transformed.joblib" % basename
		out_X_sums = "%s-X_sums.joblib" % basename
		out_X_sum_squares = "%s-X_sum_squares.joblib" % basename
		out_X_n = "%s-X_n.joblib" % basename
		out_y = "%s-y.joblib" % basename
		out_y_transformed = "%s-y-transformed.joblib" % basename
		out_y_sums = "%s-y_sums.joblib" % basename
		out_y_sum_squares = "%s-y_sum_squares.joblib" % basename
		out_y_n = "%s-y_n.joblib" % basename
		out_w = "%s-w.joblib" % basename
		out_w_transformed = "%s-w-transformed.joblib" % basename
		out_ids = "%s-ids.joblib" % basename

		save_to_disk(X, os.path.join(data_dir, out_X))
		save_to_disk(y, os.path.join(data_dir, out_y))
		save_to_disk(w, os.path.join(data_dir, out_w))
		# Write moments to disk
		save_to_disk(X_sums, out_X_sums)
		save_to_disk(X_sum_squares, out_X_sum_squares)
		save_to_disk(X_n, out_X_n)
		save_to_disk(y_sums, out_y_sums)
		save_to_disk(y_sum_squares, out_y_sum_squares)
		save_to_disk(y_n, out_y_n)
		save_to_disk(X_sums, os.path.join(data_dir, out_X_sums))
		save_to_disk(X_sum_squares, os.path.join(data_dir, out_X_sum_squares))
		save_to_disk(X_n, os.path.join(data_dir, out_X_n))
		save_to_disk(y_sums, os.path.join(data_dir, out_y_sums))
		save_to_disk(y_sum_squares, os.path.join(data_dir, out_y_sum_squares))
		save_to_disk(y_n, os.path.join(data_dir, out_y_n))
		# Write X, y as transformed versions
		save_to_disk(X, out_X_transformed)
		save_to_disk(y, out_y_transformed)
		save_to_disk(w, out_w_transformed)
		save_to_disk(ids, out_ids)
		save_to_disk(X, os.path.join(data_dir, out_X_transformed))
		save_to_disk(y, os.path.join(data_dir, out_y_transformed))
		save_to_disk(w, os.path.join(data_dir, out_w_transformed))
		save_to_disk(ids, os.path.join(data_dir, out_ids))
		return([df_file, tasks, out_ids, out_X, out_X_transformed, out_y,
		out_y_transformed, out_w, out_w_transformed,
		out_X_sums, out_X_sum_squares, out_X_n,
		@@ -438,68 +511,3 @@ def _df_to_numpy(df, feature_types, tasks):
		assert len(sorted_ids) == len(x) == len(y) == len(w)
		return sorted_ids, x.astype(float), y.astype(float), w.astype(float)

		def compute_mean_and_std(df):
		"""
		Compute means/stds of X/y from sums/sum_squares of tensors.
		"""

		X_sums = []
		X_sum_squares = []
		X_n = []
		for _, row in df.iterrows():
		Xs = load_from_disk(row['X_sums'])
		Xss = load_from_disk(row['X_sum_squares'])
		Xn = load_from_disk(row['X_n'])
		X_sums.append(np.array(Xs))
		X_sum_squares.append(np.array(Xss))
		X_n.append(np.array(Xn))

		# Note that X_n is a list of floats
		n = float(np.sum(X_n))
		X_sums = np.vstack(X_sums)
		X_sum_squares = np.vstack(X_sum_squares)
		overall_X_sums = np.sum(X_sums, axis=0)
		overall_X_means = overall_X_sums / n
		overall_X_sum_squares = np.sum(X_sum_squares, axis=0)

		X_vars = (overall_X_sum_squares - np.square(overall_X_sums)/n)/(n)

		y_sums = []
		y_sum_squares = []
		y_n = []
		for _, row in df.iterrows():
		ys = load_from_disk(row['y_sums'])
		yss = load_from_disk(row['y_sum_squares'])
		yn = load_from_disk(row['y_n'])
		y_sums.append(np.array(ys))
		y_sum_squares.append(np.array(yss))
		y_n.append(np.array(yn))

		# Note y_n is a list of arrays of shape (n_tasks,)
		y_n = np.sum(y_n, axis=0)
		y_sums = np.vstack(y_sums)
		y_sum_squares = np.vstack(y_sum_squares)
		y_means = np.sum(y_sums, axis=0)/y_n
		y_vars = np.sum(y_sum_squares, axis=0)/y_n - np.square(y_means)
		return overall_X_means, np.sqrt(X_vars), y_means, np.sqrt(y_vars)

		def update_mean_and_std(df):
		"""
		Compute means/stds of X/y from sums/sum_squares of tensors.
		"""
		X_transform = []
		for _, row in df.iterrows():
		Xt = load_from_disk(row['X-transformed'])
		Xs = np.sum(Xt,axis=0)
		Xss = np.sum(np.square(Xt),axis=0)
		save_to_disk(Xs, row['X_sums'])
		save_to_disk(Xss, row['X_sum_squares'])

		y_transform = []
		for _, row in df.iterrows():
		yt = load_from_disk(row['y-transformed'])
		ys = np.sum(yt,axis=0)
		yss = np.sum(np.square(yt),axis=0)
		save_to_disk(ys, row['y_sums'])
		save_to_disk(yss, row['y_sum_squares'])

deepchem/datasets/tests/test_load.py

+45 −0

Original line number	Diff line number	Diff line
		@@ -26,6 +26,51 @@ class TestLoad(TestAPI):
		Test singletask/multitask data loading.
		"""

		def test_move_load(self):
		"""Test that datasets can be moved and loaded."""
		verbosity = "high"
		current_dir = os.path.dirname(os.path.realpath(__file__))
		feature_dir = os.path.join(self.base_dir, "features")
		moved_feature_dir = os.path.join(self.base_dir, "moved_features")
		samples_dir = os.path.join(self.base_dir, "samples")
		moved_samples_dir = os.path.join(self.base_dir, "moved_samples")
		data_dir = os.path.join(self.base_dir, "data")
		moved_data_dir = os.path.join(self.base_dir, "moved_data")
		dataset_file = os.path.join(
		current_dir, "../../models/tests/example.csv")

		featurizers = [CircularFingerprint(size=1024)]
		tasks = ["log-solubility"]
		featurizer = DataFeaturizer(tasks=tasks,
		smiles_field="smiles",
		compound_featurizers=featurizers,
		verbosity=verbosity)
		featurized_samples = featurizer.featurize(
		dataset_file, feature_dir,
		samples_dir, reload=reload)
		dataset = Dataset(data_dir=data_dir, samples=featurized_samples,
		featurizers=featurizers, tasks=tasks,
		verbosity=verbosity, reload=reload)

		X, y, w, ids = dataset.to_numpy()
		shutil.move(feature_dir, moved_feature_dir)
		shutil.move(samples_dir, moved_samples_dir)
		shutil.move(data_dir, moved_data_dir)

		moved_dataset = Dataset(
		data_dir=moved_data_dir, samples=featurized_samples,
		featurizers=featurizers, tasks=tasks,
		verbosity=verbosity, reload=reload)

		X_moved, y_moved, w_moved, ids_moved = moved_dataset.to_numpy()

		np.testing.assert_allclose(X, X_moved)
		np.testing.assert_allclose(y, y_moved)
		np.testing.assert_allclose(w, w_moved)
		np.testing.assert_array_equal(ids, ids_moved)



		def test_multiload(self):
		"""Check can re-use featurization for multiple task selections."""
		# Only for debug!

deepchem/featurizers/tests/test_featurized_samples.py

+36 −10

Original line number	Diff line number	Diff line
		@@ -13,25 +13,18 @@ import os
		import unittest
		import tempfile
		import shutil
		from deepchem.models.tests import TestAPI
		from deepchem.splits import RandomSplitter
		from deepchem.splits import ScaffoldSplitter
		from deepchem.splits import SpecifiedSplitter
		from deepchem.featurizers.featurize import DataFeaturizer
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.featurizers.featurize import FeaturizedSamples

		class TestFeaturizedSamples(unittest.TestCase):
		class TestFeaturizedSamples(TestAPI):
		"""
		Test Featurized Samples class.
		"""
		def setUp(self):
		self.current_dir = os.path.dirname(os.path.abspath(__file__))
		self.smiles_field = "smiles"
		self.feature_dir = tempfile.mkdtemp()
		self.samples_dir = tempfile.mkdtemp()
		self.train_dir = tempfile.mkdtemp()
		self.valid_dir = tempfile.mkdtemp()
		self.test_dir = tempfile.mkdtemp()

		def _featurize_train_valid_test_split(self, splittype, input_file, tasks,
		frac_train, frac_valid, frac_test):
		# Featurize input
		@@ -141,3 +134,36 @@ class TestFeaturizedSamples(unittest.TestCase):
		frac_valid=0, frac_test=.2))
		assert len(train_samples) == 8
		assert len(test_samples) == 2

		def test_samples_move(self):
		"""Test that featurized samples can be moved and reloaded."""
		verbosity = "high"
		current_dir = os.path.dirname(os.path.realpath(__file__))
		feature_dir = os.path.join(self.base_dir, "features")
		moved_feature_dir = os.path.join(self.base_dir, "moved_features")
		samples_dir = os.path.join(self.base_dir, "samples")
		moved_samples_dir = os.path.join(self.base_dir, "moved_samples")
		dataset_file = os.path.join(
		current_dir, "../../models/tests/example.csv")

		featurizers = [CircularFingerprint(size=1024)]
		tasks = ["log-solubility"]
		featurizer = DataFeaturizer(tasks=tasks,
		smiles_field="smiles",
		compound_featurizers=featurizers,
		verbosity=verbosity)
		featurized_samples = featurizer.featurize(
		dataset_file, feature_dir,
		samples_dir, reload=reload)
		n_samples = len(featurized_samples)

		# Now perform move
		shutil.move(feature_dir, moved_feature_dir)
		shutil.move(samples_dir, moved_samples_dir)

		moved_featurized_samples = FeaturizedSamples(
		samples_dir=moved_samples_dir, featurizers=featurizers,
		reload=True)

		assert len(moved_featurized_samples) == n_samples

deepchem/models/tests/test_generalize.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -95,6 +95,7 @@ class TestGeneralization(TestAPI):
		train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train)
		test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test)


		# Eval model on train
		input_transformers = [
		NormalizationTransformer(transform_X=True, dataset=train_dataset),

deepchem/transformers/init.py

+36 −36

Original line number	Diff line number	Diff line
		@@ -40,7 +40,7 @@ class Transformer(object):
		# Use fact that bools add as ints in python
		assert (transform_X + transform_y + transform_w) == 1

		def transform_row(self, i, df):
		def transform_row(self, i, df, data_dir):
		"""
		Transforms the data (X, y, w, ...) in a single row).
		"""
		@@ -67,7 +67,7 @@ class Transformer(object):
		df = dataset.metadata_df
		indices = range(0, df.shape[0])
		transform_row_partial = partial(
		_transform_row, df=df, transformer=self)
		_transform_row, df=df, transformer=self, data_dir=dataset.data_dir)
		if parallel:
		pool = mp.Pool(int(mp.cpu_count()/4))
		pool.map(transform_row_partial, indices)
		@@ -84,13 +84,13 @@ class Transformer(object):
		X, y, w = self.transform_array(X, y, w)
		return X, y, w

		def _transform_row(i, df, transformer):
		def _transform_row(i, df, transformer, data_dir):
		"""
		Transforms the data (X, y, w,...) in a single row.

		Writes X-transforme,d y-transformed to disk.
		"""
		transformer.transform_row(i, df)
		transformer.transform_row(i, df, data_dir)

		class NormalizationTransformer(Transformer):

		@@ -111,21 +111,22 @@ class NormalizationTransformer(Transformer):
		dataset, parallel=parallel)


		def transform_row(self, i, df):
		def transform_row(self, i, df, data_dir):
		"""
		Normalizes the data (X, y, w, ...) in a single row).
		"""
		row = df.iloc[i]

		if self.transform_X:
		X = load_from_disk(row['X-transformed'])
		X = load_from_disk(
		os.path.join(data_dir, row['X-transformed']))
		X = np.nan_to_num((X - self.X_means) / self.X_stds)
		save_to_disk(X, row['X-transformed'])
		save_to_disk(X, os.path.join(data_dir, row['X-transformed']))

		if self.transform_y:
		y = load_from_disk(row['y-transformed'])
		y = load_from_disk(os.path.join(data_dir, row['y-transformed']))
		y = np.nan_to_num((y - self.y_means) / self.y_stds)
		save_to_disk(y, row['y-transformed'])
		save_to_disk(y, os.path.join(data_dir, row['y-transformed']))

		def untransform(self, z):
		"""
		@@ -134,7 +135,6 @@ class NormalizationTransformer(Transformer):
		if self.transform_X:
		return z * self.X_stds + self.X_means
		elif self.transform_y:
		out = z * self.y_stds + self.y_means
		return z * self.y_stds + self.y_means

		class ClippingTransformer(Transformer):
		@@ -148,21 +148,21 @@ class ClippingTransformer(Transformer):
		dataset=dataset)
		self.max_val = max_val

		def transform_row(self, i, df):
		def transform_row(self, i, df, data_dir):
		"""
		Clips outliers for the data (X, y, w, ...) in a single row).
		"""
		row = df.iloc[i]
		if self.transform_X:
		X = load_from_disk(row['X-transformed'])
		X = load_from_disk(os.path.join(data_dir, row['X-transformed']))
		X[X > self.max_val] = self.max_val
		X[X < (-1.0self.max_val)] = -1.0 self.max_val
		save_to_disk(X, row['X-transformed'])
		save_to_disk(X, os.path.join(data_dir, row['X-transformed']))
		if self.transform_y:
		y = load_from_disk(row['y-transformed'])
		y = load_from_disk(os.path.join(data_dir, row['y-transformed']))
		y[y > trunc] = trunc
		y[y < (-1.0trunc)] = -1.0 trunc
		save_to_disk(y, row['y-transformed'])
		save_to_disk(y, os.path.join(data_dir, row['y-transformed']))

		def untransform(self, z):
		warnings.warn("Clipping cannot be undone.")
		@@ -170,18 +170,18 @@ class ClippingTransformer(Transformer):

		class LogTransformer(Transformer):

		def transform_row(self, i, df):
		def transform_row(self, i, df, data_dir):
		"""Logarithmically transforms data in dataset."""
		row = df.iloc[i]
		if self.transform_X:
		X = load_from_disk(row['X-transformed'])
		X = load_from_disk(os.path.join(data_dir, row['X-transformed']))
		X = np.log(X)
		save_to_disk(X, row['X-transformed'])
		save_to_disk(X, os.path.join(data_dir, row['X-transformed']))

		if self.transform_y:
		y = load_from_disk(row['y-transformed'])
		y = load_from_disk(os.path.join(data_dir, row['y-transformed']))
		y = np.log(y)
		save_to_disk(y, row['y-transformed'])
		save_to_disk(y, os.path.join(data_dir, row['y-transformed']))

		def untransform(self, z):
		"""Undoes the logarithmic transformation."""
		@@ -220,11 +220,11 @@ class BalancingTransformer(Transformer):
		weights.append((neg_weight, pos_weight))
		self.weights = weights

		def transform_row(self, i, df):
		def transform_row(self, i, df, data_dir):
		"""Reweight the labels for this data."""
		row = df.iloc[i]
		y = load_from_disk(row['y-transformed'])
		w = load_from_disk(row['w-transformed'])
		y = load_from_disk(os.path.join(data_dir, row['y-transformed']))
		w = load_from_disk(os.path.join(data_dir, row['w-transformed']))
		w_balanced = np.zeros_like(w)
		for ind, task in enumerate(self.dataset.get_task_names()):
		task_y = y[:, ind]
		@@ -233,7 +233,7 @@ class BalancingTransformer(Transformer):
		one_indices = np.logical_and(task_y==1, task_w != 0)
		w_balanced[zero_indices, ind] = self.weights[ind][0]
		w_balanced[one_indices, ind] = self.weights[ind][1]
		save_to_disk(w_balanced, row['w-transformed'])
		save_to_disk(w_balanced, os.path.join(data_dir, row['w-transformed']))

		class CoulombRandomizationTransformer(Transformer):

		@@ -285,17 +285,17 @@ class CoulombRandomizationTransformer(Transformer):

		return rcm

		def transform_row(self, i, df):
		def transform_row(self, i, df, data_dir):
		"""
		Randomly permute a Coulomb Matrix in a dataset
		"""
		row = df.iloc[i]
		if self.transform_X:
		X = load_from_disk(row['X-transformed'])
		X = load_from_disk(os.path.join(data_dir, row['X-transformed']))
		for j in xrange(len(X)):
		cm = self.construct_cm_from_triu(X[j])
		X[j] = self.unpad_randomize_and_flatten(cm)
		save_to_disk(X, row['X-transformed'])
		save_to_disk(X, os.path.join(data_dir, row['X-transformed']))

		if self.transform_y:
		print("y will not be transformed by "
		@@ -330,10 +330,10 @@ class CoulombBinarizationTransformer(Transformer):
		self.feature_max = np.zeros(dataset.get_data_shape())
		self.update_state = update_state

		def set_max(self, df):
		def set_max(self, df, data_dir):

		for _, row in df.iterrows():
		X = load_from_disk(row['X-transformed'])
		X = load_from_disk(os.path.join(data_dir, row['X-transformed']))
		self.feature_max = np.maximum(self.feature_max,X.max(axis=0))

		def transform(self, dataset, parallel=False):
		@@ -345,7 +345,7 @@ class CoulombBinarizationTransformer(Transformer):
		Xt = []

		for _, row in df.iterrows():
		X_t = load_from_disk(row['X-transformed'])
		X_t = load_from_disk(os.path.join(dataset.data_dir, row['X-transformed']))
		Xt.append(np.array(X_t))

		X = np.vstack(Xt)
		@@ -354,25 +354,25 @@ class CoulombBinarizationTransformer(Transformer):

		for i, row in df.iterrows():
		X_t = (Xt[i]-X_means)/X_stds
		save_to_disk(X_t, row['X-transformed'])
		save_to_disk(X_t, os.path.join(dataset.data_dir, row['X-transformed']))

		def transform_row(self, i, df):
		def transform_row(self, i, df, data_dir):
		"""
		Binarizes data in dataset with sigmoid function
		"""
		row = df.iloc[i]
		X_bin = []
		if self.update_state:
		self.set_max(df)
		self.set_max(df, data_dir)
		self.update_state = False
		if self.transform_X:
		X = load_from_disk(row['X-transformed'])
		X = load_from_disk(os.path.join(data_dir, row['X-transformed']))
		for i in range(X.shape[1]):
		for k in np.arange(0,self.feature_max[i]+self.theta,self.theta):
		X_bin += [np.tanh((X[:,i]-k)/self.theta)]

		X_bin = np.array(X_bin).T
		save_to_disk(X_bin, row['X-transformed'])
		save_to_disk(X_bin, os.path.join(data_dir, row['X-transformed']))

		if self.transform_y:
		print("y will not be transformed by "
		@@ -385,7 +385,7 @@ class CoulombBinarizationTransformer(Transformer):

		X_bin = []
		if self.update_state:
		self.set_max(df)
		self.set_max(df, data_dir)
		self.update_state = False
		if self.transform_X:
		for i in range(X.shape[1]):

Admin message