Merge pull request #156 from joegomes/dynamic-transform (e89b2606) · Commits · 钟慕尧 / deepchem

deepchem/models/init.py

+22 −28

Original line number	Diff line number	Diff line
		@@ -34,13 +34,6 @@ class Model(object):
		self.task_types = task_types
		self.model_params = model_params
		self.fit_transformers = fit_transformers
		if self.fit_transformers:

		# Initialize batch_dataset
		self.batch_dataset = self.create_batch_dataset()

		else:
		self.batch_dataset = None

		self.raw_model = None
		assert verbosity in [None, "low", "high"]
		@@ -106,32 +99,18 @@ class Model(object):
		for (X_batch, y_batch, w_batch, _) in dataset.iterbatches(batch_size):
		if self.fit_transformers:
		X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch,
		w_batch, self.batch_dataset)
		w_batch)
		losses.append(self.fit_on_batch(X_batch, y_batch, w_batch))
		log("Avg loss for epoch %d: %f" % (epoch+1,np.array(losses).mean()),self.verbosity)


		def transform_on_batch(self, X, y, w, batch_dataset):
		def transform_on_batch(self, X, y, w):
		"""
		Transforms data in a 1-shard Dataset object with Transformer objects.
		"""
		# Save X, y, and w to batch_dataset
		# The save/load operations work correctly with 1-shard dataframe
		df = batch_dataset.metadata_df
		for _, row in df.iterrows():
		save_to_disk(X, row['X-transformed'])
		save_to_disk(y, row['y-transformed'])
		save_to_disk(w, row['w'])

		# Transform batch_dataset
		# Transform X, y, and w
		for transformer in self.fit_transformers:
		transformer.transform(batch_dataset)

		# Return numpy arrays from batch_dataset
		for _, row in df.iterrows():
		X = load_from_disk(row['X-transformed'])
		y = load_from_disk(row['y-transformed'])
		w = load_from_disk(row['w'])
		X, y, w = transformer.transform_on_array(X, y, w)

		return X, y, w

		@@ -168,15 +147,30 @@ class Model(object):
		batch_size = self.model_params["batch_size"]
		for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):

		# Apply fit_transformers if needed
		# HACK(JG): This was a hack to perform n-fold averaging of y_pred on
		# a given X_batch. If fit_transformers exist, we will apply them to
		# X_batch 1 times and average the resulting y_pred before we undo
		# transforms on y_pred and y. In the future the averaging will be
		# performed n_sample times, where n_sample can be user-specified.

		if self.fit_transformers:
		X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch,
		w_batch, self.batch_dataset)

		y_preds = []
		for i in xrange(1):
		X_b, y_b, w_b = self.transform_on_batch(X_batch, y_batch, w_batch)
		y_pred = self.predict_on_batch(X_b)
		y_pred = np.reshape(y_pred, np.shape(y_b))
		y_preds.append(y_pred)

		y_pred = np.array(y_preds).mean(axis=0)

		else:

		y_pred = self.predict_on_batch(X_batch)
		y_pred = np.reshape(y_pred, np.shape(y_batch))

		# Now undo transformations on y, y_pred

		y_raw, y_pred_raw = y_batch, y_pred
		y_batch = undo_transforms(y_batch, transformers)
		y_pred = undo_transforms(y_pred, transformers)

deepchem/models/keras_models/fcnet.py

+0 −3

Original line number	Diff line number	Diff line
		@@ -27,9 +27,6 @@ class MultiTaskDNN(KerasModel):
		verbosity=verbosity)
		if initialize_raw_model:
		sorted_tasks = sorted(task_types.keys())
		if fit_transformers:
		(n_inputs,) = model_params["init_data_shape"]
		else:
		(n_inputs,) = model_params["data_shape"]
		model = Graph()
		model.add_input(name="input", input_shape=(n_inputs,))

deepchem/transformers/init.py

+50 −1

Original line number	Diff line number	Diff line
		@@ -36,7 +36,11 @@ class Transformer(object):
		Transforms the data (X, y, w, ...) in a single row).
		"""
		raise NotImplementedError(
		"Each Transformer is responsible for its own tranform_row method.")
		"Each Transformer is responsible for its own transform_row method.")

		def transform_array(self, X, y, w):
		raise NotImplementedError(
		"Each Transformer is responsible for its own transform_array method.")

		def untransform(self, z):
		"""Reverses stored transformation on provided data."""
		@@ -64,6 +68,13 @@ class Transformer(object):
		transform_row_partial(index)
		dataset.save_to_disk()

		def transform_on_array(self, X, y, w):
		"""
		Transforms numpy arrays X, y, and w
		"""
		X, y, w = self.transform_array(X, y, w)
		return X, y, w

		def _transform_row(i, df, transformer):
		"""
		Transforms the data (X, y, w,...) in a single row.
		@@ -286,6 +297,20 @@ class CoulombRandomizationTransformer(Transformer):
		print("y will not be transformed by "
		"CoulombRandomizationTransformer.")

		def transform_array(self, X, y, w):
		"""
		Randomly permute a Coulomb Matrix passed as an array
		"""
		if self.transform_X:
		for j in xrange(len(X)):
		cm = self.construct_cm_from_triu(X[j])
		X[j] = self.unpad_randomize_and_flatten(cm)

		if self.transform_y:
		print("y will not be transformed by CoulombRandomizationTransformer.")

		return X, y, w

		def untransform(self, z):
		print("Cannot undo CoulombRandomizationTransformer.")

		@@ -349,5 +374,29 @@ class CoulombBinarizationTransformer(Transformer):
		print("y will not be transformed by "
		"CoulombBinarizationTransformer.")

		def transform_array(self,X, y, w):
		"""
		Binarizes data passed as arrays with sigmoid function
		"""

		X_bin = []
		if self.update_state:
		self.set_max(df)
		self.update_state = False
		if self.transform_X:
		for i in range(X.shape[1]):
		for k in np.arange(0,self.feature_max[i]+self.theta,self.theta):
		X_bin += [np.tanh((X[:,i]-k)/self.theta)]

		X = np.array(X_bin).T
		X_means = X.mean(axis=0)
		X_stds = (X-X_means).std()
		X = (X-X_means)/X_stds

		if self.transform_y:
		print("y will not be transformed by CoulombBinarizationTransformer.")

		return X, y, w

		def untranform(self, z):
		print("Cannot undo CoulombBinarizationTransformer.")

Admin message