Merge pull request #317 from rbharath/data_loader_cleanup (1cd28c35) · Commits · 钟慕尧 / deepchem

deepchem/init.py

+0 −1

Original line number	Diff line number	Diff line
		@@ -14,4 +14,3 @@ import deepchem.nn
		import deepchem.splits
		import deepchem.trans
		import deepchem.utils
		import deepchem.load

deepchem/data/init.py

+11 −1

Original line number	Diff line number	Diff line
		@@ -6,6 +6,16 @@ from __future__ import division
		from __future__ import unicode_literals

		# TODO(rbharath): Get rid of * import
		from deepchem.data.datasets import *
		from deepchem.data.datasets import pad_features
		from deepchem.data.datasets import pad_batch
		from deepchem.data.datasets import Dataset
		from deepchem.data.datasets import NumpyDataset
		from deepchem.data.datasets import DiskDataset
		from deepchem.data.datasets import sparsify_features
		from deepchem.data.datasets import densify_features
		from deepchem.data.supports import *
		from deepchem.data.data_loader import DataLoader
		from deepchem.data.data_loader import CSVLoader
		from deepchem.data.data_loader import UserCSVLoader
		from deepchem.data.data_loader import SDFLoader
		import deepchem.data.tests

deepchem/data/data_loader.py

0 → 100644

+212 −0

Original line number	Diff line number	Diff line
		"""
		Process an input dataset into a format suitable for machine learning.
		"""
		from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals
		import os
		import gzip
		import pandas as pd
		import numpy as np
		import csv
		import numbers
		import dill
		import tempfile
		from rdkit import Chem
		import time
		import sys
		from deepchem.utils.save import log
		from deepchem.utils.save import load_csv_files
		from deepchem.utils.save import load_sdf_files
		from deepchem.feat import UserDefinedFeaturizer
		from deepchem.data import DiskDataset

		def convert_df_to_numpy(df, tasks, id_field, verbose=False):
		"""Transforms a dataframe containing deepchem input into numpy arrays"""
		n_samples = df.shape[0]
		n_tasks = len(tasks)

		time1 = time.time()
		y = np.hstack([
		np.reshape(np.array(df[task].values), (n_samples, 1)) for task in tasks])
		time2 = time.time()

		w = np.ones((n_samples, n_tasks))
		missing = np.zeros_like(y).astype(int)
		feature_shape = None

		for ind in range(n_samples):
		for task in range(n_tasks):
		if y[ind, task] == "":
		missing[ind, task] = 1

		ids = df[id_field].values
		# Set missing data to have weight zero
		for ind in range(n_samples):
		for task in range(n_tasks):
		if missing[ind, task]:
		y[ind, task] = 0.
		w[ind, task] = 0.

		return ids, y.astype(float), w.astype(float)

		def featurize_smiles_df(df, featurizer, field, log_every_N=1000, verbose=True):
		"""Featurize individual compounds in dataframe.

		Given a featurizer that operates on individual chemical compounds
		or macromolecules, compute & add features for that compound to the
		features dataframe
		"""
		sample_elems = df[field].tolist()

		features = []
		for ind, elem in enumerate(sample_elems):
		mol = Chem.MolFromSmiles(elem)
		if ind % log_every_N == 0:
		log("Featurizing sample %d" % ind, verbose)
		features.append(featurizer.featurize([mol]))
		valid_inds = np.array([1 if elt.size > 0 else 0 for elt in features],
		dtype=bool)
		features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid]
		return np.squeeze(np.array(features)), valid_inds

		def get_user_specified_features(df, featurizer, verbose=True):
		"""Extract and merge user specified features.

		Merge features included in dataset provided by user
		into final features dataframe

		Three types of featurization here:

		1) Molecule featurization
		-) Smiles string featurization
		-) Rdkit MOL featurization
		2) Complex featurization
		-) PDB files for interacting molecules.
		3) User specified featurizations.
		"""
		time1 = time.time()
		df[featurizer.feature_fields] = df[featurizer.feature_fields].apply(pd.to_numeric)
		X_shard = df.as_matrix(columns=featurizer.feature_fields)
		time2 = time.time()
		log("TIMING: user specified processing took %0.3f s" % (time2-time1), verbose)
		return X_shard

		def featurize_mol_df(df, featurizer, field, verbose=True, log_every_N=1000):
		"""Featurize individual compounds in dataframe.

		Featurizes .sdf files, so the 3-D structure should be preserved
		so we use the rdkit "mol" object created from .sdf instead of smiles
		string. Some featurizers such as CoulombMatrix also require a 3-D
		structure. Featurizing from .sdf is currently the only way to
		perform CM feautization.
		"""
		sample_elems = df[field].tolist()

		features = []
		for ind, mol in enumerate(sample_elems):
		if ind % log_every_N == 0:
		log("Featurizing sample %d" % ind, verbose)
		features.append(featurizer.featurize([mol]))
		valid_inds = np.array([1 if elt.size > 0 else 0 for elt in features],
		dtype=bool)
		features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid]
		return np.squeeze(np.array(features)), valid_inds

		class DataLoader(object):
		"""
		Handles loading/featurizing of chemical samples (datapoints).

		Currently knows how to load csv-files/pandas-dataframes/SDF-files. Writes a
		dataframe object to disk as output.
		"""

		def __init__(self, tasks, smiles_field=None,
		id_field=None, mol_field=None, featurizer=None,
		verbose=True, log_every_n=1000):
		"""Extracts data from input as Pandas data frame"""
		if not isinstance(tasks, list):
		raise ValueError("tasks must be a list.")
		self.verbose = verbose
		self.tasks = tasks
		self.smiles_field = smiles_field
		if id_field is None:
		self.id_field = smiles_field
		else:
		self.id_field = id_field
		self.mol_field = mol_field
		self.user_specified_features = None
		if isinstance(featurizer, UserDefinedFeaturizer):
		self.user_specified_features = featurizer.feature_fields
		self.featurizer = featurizer
		self.log_every_n = log_every_n

		def featurize(self, input_files, data_dir=None, shard_size=8192):
		"""Featurize provided files and write to specified location."""
		log("Loading raw samples now.", self.verbose)
		log("shard_size: %d" % shard_size, self.verbose)

		if not isinstance(input_files, list):
		input_files = [input_files]
		def shard_generator():
		for shard_num, shard in enumerate(self.get_shards(input_files, shard_size)):
		time1 = time.time()
		X, valid_inds = self.featurize_shard(shard)
		ids, y, w = convert_df_to_numpy(shard, self.tasks, self.id_field)
		# Filter out examples where featurization failed.
		ids, y, w = (ids[valid_inds], y[valid_inds], w[valid_inds])
		assert len(X) == len(ids) == len(y) == len(w)
		time2 = time.time()
		log("TIMING: featurizing shard %d took %0.3f s" % (shard_num, time2-time1),
		self.verbose)
		yield X, y, w, ids
		return DiskDataset(shard_generator(), data_dir, self.tasks)

		def get_shards(self, input_files, shard_size):
		"""Stub for children classes."""
		raise NotImplementedError

		def featurize_shard(self, shard):
		"""Featurizes a shard of an input dataframe."""
		raise NotImplementedError

		class CSVLoader(DataLoader):
		"""
		Handles loading of CSV files.
		"""
		def get_shards(self, input_files, shard_size, verbose=True):
		"""Defines a generator which returns data for each shard"""
		return load_csv_files(input_files, shard_size, verbose=verbose)

		def featurize_shard(self, shard):
		"""Featurizes a shard of an input dataframe."""
		return featurize_smiles_df(shard, self.featurizer,
		field=self.smiles_field)

		class UserCSVLoader(DataLoader):
		"""
		Handles loading of CSV files with user-defined featurizers.
		"""
		def get_shards(self, input_files, shard_size):
		"""Defines a generator which returns data for each shard"""
		return load_csv_files(input_files, shard_size)

		def featurize_shard(self, shard):
		"""Featurizes a shard of an input dataframe."""
		assert isinstance(self.featurizer, UserDefinedFeaturizer)
		X = get_user_specified_features(shard, self.featurizer)
		return (X, np.ones(len(X), dtype=bool))

		class SDFLoader(DataLoader):
		"""
		Handles loading of SDF files.
		"""
		def get_shards(self, input_files, shard_size):
		"""Defines a generator which returns data for each shard"""
		return load_sdf_files(input_files)

		def featurize_shard(self, shard):
		"""Featurizes a shard of an input dataframe."""
		log("Currently featurizing feature_type: %s"
		% self.featurizer.__class__.__name__, self.verbose)
		return featurize_mol_df(shard, self.featurizer, field=self.mol_field)

deepchem/data/datasets.py

+147 −417

File changed.

Preview size limit exceeded, changes collapsed.

deepchem/data/tests/init.py

+14 −23

Original line number	Diff line number	Diff line
		@@ -23,12 +23,10 @@ def load_solubility_data():
		tasks = ["log-solubility"]
		task_type = "regression"
		input_file = os.path.join(current_dir, "../../models/tests/example.csv")
		featurizer = dc.load.DataLoader(
		tasks=tasks,
		smiles_field="smiles",
		featurizer=featurizer,
		verbosity="low")
		return featurizer.featurize(input_file)
		loader = dc.data.CSVLoader(
		tasks=tasks, smiles_field="smiles", featurizer=featurizer)

		return loader.featurize(input_file)

		def load_multitask_data():
		"""Load example multitask data."""
		@@ -39,11 +37,8 @@ def load_multitask_data():
		"task13", "task14", "task15", "task16"]
		input_file = os.path.join(
		current_dir, "../../models/tests/multitask_example.csv")
		loader = dc.load.DataLoader(
		tasks=tasks,
		smiles_field="smiles",
		featurizer=featurizer,
		verbosity="low")
		loader = dc.data.CSVLoader(
		tasks=tasks, smiles_field="smiles", featurizer=featurizer)
		return loader.featurize(input_file)

		def load_classification_data():
		@@ -54,9 +49,8 @@ def load_classification_data():
		task_type = "classification"
		input_file = os.path.join(
		current_dir, "../../models/tests/example_classification.csv")
		loader = dc.load.DataLoader(
		tasks=tasks, smiles_field="smiles",
		featurizer=featurizer, verbosity="low")
		loader = dc.data.CSVLoader(
		tasks=tasks, smiles_field="smiles", featurizer=featurizer)
		return loader.featurize(input_file)


		@@ -68,9 +62,8 @@ def load_sparse_multitask_dataset():
		"task7", "task8", "task9"]
		input_file = os.path.join(
		current_dir, "../../models/tests/sparse_multitask_example.csv")
		loader = dc.load.DataLoader(
		tasks=tasks, smiles_field="smiles",
		featurizer=featurizer, verbosity="low")
		loader = dc.data.CSVLoader(
		tasks=tasks, smiles_field="smiles", featurizer=featurizer)
		return loader.featurize(input_file)

		def load_feat_multitask_data():
		@@ -81,9 +74,8 @@ def load_feat_multitask_data():
		tasks = ["task0", "task1", "task2", "task3", "task4", "task5"]
		input_file = os.path.join(
		current_dir, "../../models/tests/feat_multitask_example.csv")
		loader = dc.load.DataLoader(
		tasks=tasks, featurizer=featurizer,
		id_field="id", verbosity="low")
		loader = dc.data.UserCSVLoader(
		tasks=tasks, featurizer=featurizer, id_field="id")
		return loader.featurize(input_file)

		def load_gaussian_cdf_data():
		@@ -96,7 +88,6 @@ def load_gaussian_cdf_data():
		tasks = ["task0","task1"]
		input_file = os.path.join(
		current_dir, "../../models/tests/gaussian_cdf_example.csv")
		loader = dc.load.DataLoader(
		tasks=tasks, featurizer=featurizer,
		id_field="id", verbosity=None)
		loader = dc.data.UserCSVLoader(
		tasks=tasks, featurizer=featurizer, id_field="id")
		return loader.featurize(input_file)

Admin message