Empty dataset creation + batch dataset creation (a217b7f8) · Commits · 钟慕尧 / deepchem

deepchem/datasets/init.py

+35 −0

Original line number	Diff line number	Diff line
		@@ -70,6 +70,41 @@ class Dataset(object):
		'X_sums', 'X_sum_squares', 'X_n',
		'y_sums', 'y_sum_squares', 'y_n'))
		self.save_to_disk()

		if samples is None and feature_types is not None: # Create an empty metadata dataframe to be filled at a later time

		basename = "metadata"
		df_file = "metadata.joblib"
		out_X = os.path.join(data_dir, "%s-X.joblib" % basename)
		out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename)
		out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename)
		out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename)
		out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename)
		out_y = os.path.join(data_dir, "%s-y.joblib" % basename)
		out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename)
		out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename)
		out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)
		out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)
		out_w = os.path.join(data_dir, "%s-w.joblib" % basename)
		out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

		metadata_rows = []
		retval = ([df_file, tasks, out_ids, out_X,
		out_X_transformed, out_y,
		out_y_transformed, out_w,
		out_X_sums, out_X_sum_squares, out_X_n,
		out_y_sums, out_y_sum_squares, out_y_n])
		metadata_rows.append(retval)

		self.metadata_df = pd.DataFrame(
		metadata_rows,
		columns=('df_file','task_names', 'ids',
		'X', 'X-transformed', 'y', 'y-transformed',
		'w',
		'X_sums', 'X_sum_squares', 'X_n',
		'y_sums', 'y_sum_squares', 'y_n'))
		self.save_to_disk()

		else:
		if os.path.exists(self._get_metadata_filename()):
		self.metadata_df = load_from_disk(self._get_metadata_filename())

deepchem/models/init.py

+23 −0

Original line number	Diff line number	Diff line
		@@ -9,6 +9,7 @@ import numpy as np
		import pandas as pd
		import joblib
		import os
		import tempfile
		from deepchem.datasets import Dataset
		from deepchem.utils.save import load_from_disk
		from deepchem.utils.save import save_to_disk
		@@ -103,9 +104,31 @@ class Model(object):
		Transforms data in a Dataset object with Transformer objects.
		"""
		# Create dataset
		batch_dataset = self.create_batch_dataset(X, y, w)
		for transformer in self.fit_transformers:
		transformer.transform(batch_dataset)

		def create_batch_dataset(self, X, y, w):
		"""
		Creates a new Dataset object from a batch of X, y and w
		"""
		# Create empty dataset
		data_dir = tempfile.mkdtemp()
		featurizers = None
		tasks = self.task_types.keys()
		batch_dataset = Dataset(data_dir=data_dir, samples=None,
		featurizers=featurizers, tasks=tasks,
		use_user_specified_features=True)

		# Save X, y, and w to batch_dataset
		df = batch_dataset.metadata_df
		for _, row in df.iterrows():
		save_to_disk(X, row['X-transformed'])
		save_to_disk(y, row['y-transformed'])
		save_to_disk(w, row['w'])

		return batch_dataset

		# TODO(rbharath): The structure of the produced df might be
		# complicated. Better way to model?
		def predict(self, dataset, transformers):

deepchem/transformers/init.py

+1 −2

Original line number	Diff line number	Diff line
		@@ -50,7 +50,6 @@ class Transformer(object):

		Adds X-transform, y-transform columns to metadata.
		"""
		dataset.update_moments()
		df = dataset.metadata_df
		indices = range(0, df.shape[0])
		transform_row_partial = partial(_transform_row, df=df, transformer=self)
		@@ -248,7 +247,7 @@ class CoulombBinarizationTransformer(Transformer):

		def set_max(self, df):

		for _, row in df.iterrows(): # Iterate over entire df by rows
		for _, row in df.iterrows():
		X = load_from_disk(row['X-transformed'])
		self.feature_max = np.maximum(self.feature_max,X.max(axis=0))

Admin message