Merge pull request #82 from rbharath/new_api (ab870e25) · Commits · 钟慕尧 / deepchem

datasets/pdbbind_core_5_df.pkl.gz

0 → 100644

+534 KiB

File added.

No diff preview for this file type.

deepchem/featurizers/init.py

+2 −30

Original line number	Diff line number	Diff line
		@@ -5,40 +5,12 @@ import types
		import numpy as np
		from rdkit import Chem
		from rdkit.Chem import rdGeometry, rdMolTransforms
		from deepchem.utils.save import log

		__author__ = "Steven Kearnes"
		__copyright__ = "Copyright 2014, Stanford University"
		__license__ = "BSD 3-clause"


		def get_featurizers():
		"""Compile a dict mapping strings to featurizer classes."""

		# import all Featurizer subclasses so __subclasses__ will work
		# these have to be local imports to avoid circular imports
		from .basic import MolecularWeight, SimpleDescriptors
		from .coulomb_matrices import CoulombMatrix
		from .dragon import DragonDescriptors
		from .esp import ESP
		from .fingerprints import CircularFingerprint
		from .images import MolImage
		from .scaffolds import ScaffoldGenerator
		from .shape_grid import ShapeGrid

		featurizers = {}
		for klass in Featurizer.__subclasses__():
		assert klass.name is not None, (klass.__name__ +
		" 'name' attribute is None.")
		if isinstance(klass.name, list):
		for name in klass.name:
		assert name not in featurizers
		featurizers[name] = klass
		else:
		assert klass.name not in featurizers
		featurizers[klass.name] = klass
		return featurizers


		def resolve_featurizer(name):
		"""
		Resolve featurizer class from a string.
		@@ -78,7 +50,7 @@ class ComplexFeaturizer(object):
		features = []
		for i, (mol_pdb, protein_pdb) in enumerate(zip(mol_pdbs, protein_pdbs)):
		if i % log_every_n == 0:
		print("Featurizing %d / %d" % (i, len(mol_pdbs)))
		log("Featurizing %d / %d" % (i, len(mol_pdbs)))
		features.append(self._featurize_complex(mol_pdb, protein_pdb))
		features = np.asarray(features)
		return features

deepchem/featurizers/basic.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -31,7 +31,7 @@ class MolecularWeight(Featurizer):
		return wt


		class SimpleDescriptors(Featurizer):
		class RDKitDescriptors(Featurizer):
		"""
		RDKit descriptors.

deepchem/featurizers/featurize.py

+91 −97

Original line number	Diff line number	Diff line
		@@ -11,7 +11,8 @@ import numpy as np
		import csv
		from rdkit import Chem
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.featurizers.basic import SimpleDescriptors
		from deepchem.featurizers.basic import RDKitDescriptors
		from deepchem.utils.save import log
		from deepchem.utils.save import save_to_disk
		from deepchem.utils.save import load_from_disk
		from deepchem.utils.save import load_pandas_from_disk
		@@ -78,30 +79,6 @@ def _get_fields(input_file):
		else:
		raise ValueError("Unrecognized extension for %s" % input_file)

		'''
		def _get_raw_samples(input_file, iterator=True):
		"""Returns an iterator over all rows in input_file"""
		input_type = _get_input_type(input_file)
		if input_type == "csv":
		with open(input_file, "rb") as inp_file_obj:
		for ind, row in enumerate(csv.reader(inp_file_obj)):
		# Skip labels
		if ind == 0:
		continue
		if row is not None:
		yield row
		elif input_type == "pandas-joblib":
		dataframe = load_from_disk(input_file)
		for _, row in dataframe.iterrows():
		yield row
		elif input_type == "pandas-pickle":
		dataframe = load_pickle_from_disk(input_file)
		for _, row in dataframe.iterrows():
		yield row
		else:
		raise ValueError("Unrecognized input type for %s" % input_file)
		'''

		class DataFeaturizer(object):
		"""
		Handles loading/featurizing of chemical samples (datapoints).
		@@ -114,6 +91,7 @@ class DataFeaturizer(object):
		id_field=None, threshold=None, user_specified_features=None,
		protein_pdb_field=None, ligand_pdb_field=None,
		ligand_mol2_field=None,
		compound_featurizers=[], complex_featurizers=[],
		verbose=False, log_every_n=1000):
		"""Extracts data from input as Pandas data frame"""
		if not isinstance(tasks, list):
		@@ -130,21 +108,27 @@ class DataFeaturizer(object):
		self.ligand_pdb_field = ligand_pdb_field
		self.ligand_mol2_field = ligand_mol2_field
		self.user_specified_features = user_specified_features
		self.compound_featurizers = compound_featurizers
		self.complex_featurizers = complex_featurizers
		self.verbose = verbose
		self.log_every_n = log_every_n

		def featurize(self, input_file, feature_types, feature_dir, shard_size=128):
		def featurize(self, input_file, feature_dir, samples_dir, shard_size=128):
		"""Featurize provided file and write to specified location."""
		input_type = _get_input_type(input_file)

		log("Loading raw samples now.", self.verbose)
		raw_df = load_pandas_from_disk(input_file)
		fields = raw_df.keys()
		log("Loaded raw data frame from file.", self.verbose)

		def process_raw_sample_helper(row, fields, input_type):
		return self._process_raw_sample(input_type, row, fields)
		process_raw_sample_helper_partial = partial(process_raw_sample_helper,
		fields=fields,
		input_type=input_type)


		#processed_rows = raw_df.apply(process_raw_sample_helper_partial, axis=1)
		raw_df = raw_df.apply(process_raw_sample_helper_partial, axis=1, reduce=False)
		#raw_df = pd.DataFrame.from_records(processed_rows)
		@@ -154,17 +138,33 @@ class DataFeaturizer(object):
		0, nb_sample, np.ceil(float(nb_sample)/shard_size)+1, dtype=int)
		shard_files = []
		for j in range(len(interval_points)-1):
		print("Sharding and standardizing into shard-%s / %s shards" % (str(j+1), len(interval_points)-1))
		log("Sharding and standardizing into shard-%s / %s shards" % (str(j+1), len(interval_points)-1), self.verbose)
		raw_df_shard = raw_df.iloc[range(interval_points[j], interval_points[j+1])]

		df = self._standardize_df(raw_df_shard)
		for feature_type in feature_types:
		print("Currently feauturizing feature_type: %s" % feature_type)
		self._featurize_df(df, feature_type)
		log("Aggregating User-Specified Features", self.verbose)
		self._add_user_specified_features(df)

		for compound_featurizer in self.compound_featurizers:
		log("Currently feauturizing feature_type: %s"
		% compound_featurizer.__class__.__name__, self.verbose)
		self._featurize_compounds(df, compound_featurizer)

		for complex_featurizer in self.complex_featurizers:
		log("Currently feauturizing feature_type: %s"
		% complex_featurizer.__class__.__name__, self.verbose)
		self._featurize_complexes(df, complex_featurizer)

		shard_out = os.path.join(feature_dir, "features_shard%d.joblib" % j)
		save_to_disk(df, shard_out)
		shard_files.append(shard_out)
		return shard_files

		featurizers = self.compound_featurizers + self.complex_featurizers
		samples = FeaturizedSamples(samples_dir=samples_dir, featurizers=featurizers,
		dataset_files=shard_files,
		reload_data=False)

		return samples

		def _process_raw_sample(self, input_type, row, fields):
		"""Extract information from row data."""
		@@ -187,8 +187,8 @@ class DataFeaturizer(object):

		def _standardize_df(self, ori_df):
		"""Copy specified columns to new df with standard column names."""
		df = pd.DataFrame([])
		df["mol_id"] = ori_df[[self.id_field]]
		df = pd.DataFrame(ori_df[[self.id_field]])
		df.columns = ["mol_id"]
		df["smiles"] = ori_df[[self.smiles_field]]
		for task in self.tasks:
		df[task] = ori_df[[task]]
		@@ -202,56 +202,54 @@ class DataFeaturizer(object):
		df["ligand_mol2"] = ori_df[[self.ligand_mol2_field]]
		return df

		def _featurize_df(self, df, feature_type):
		def _featurize_complexes(self, df, featurizer):
		"""Generates circular fingerprints for dataset."""
		if feature_type == "user-specified-features":
		if self.user_specified_features is not None:
		if self.verbose:
		print("Adding user-defined features.")
		features_data = []
		for _, row in df.iterrows():
		# pandas rows are tuples (row_num, row_data)
		feature_list = []
		for feature_name in self.user_specified_features:
		feature_list.append(row[feature_name])
		features_data.append({feature_type: np.array(feature_list)})
		df[feature_type] = pd.DataFrame(features_data)
		return
		elif feature_type in ["ECFP", "RDKIT-descriptors"]:
		if feature_type == "ECFP":
		if self.verbose:
		print("Generating ECFP circular fingerprints.")
		featurizer = CircularFingerprint(size=1024)
		elif feature_type == "RDKIT-descriptors":
		if self.verbose:
		print("Generating RDKIT descriptors.")
		featurizer = SimpleDescriptors()
		protein_pdbs = list(df["protein_pdb"])
		ligand_pdbs = list(df["ligand_pdb"])
		complexes = zip(ligand_pdbs, protein_pdbs)

		features = featurizer.featurize_complexes(ligand_pdbs, protein_pdbs)
		df[featurizer.__class__.__name__] = list(features)

		def _featurize_compounds(self, df, featurizer):
		"""Featurize individual compounds.

		Given a featurizer that operates on individual chemical compounds
		or macromolecules, compute & add features for that compound to the
		features dataframe
		"""
		features = []
		sample_smiles = df["smiles"].tolist()
		for ind, smiles in enumerate(sample_smiles):
		if ind % self.log_every_n == 0:
		print("Featurizing sample %d" % ind)
		log("Featurizing sample %d" % ind, self.verbose)
		mol = Chem.MolFromSmiles(smiles)
		features.append(featurizer.featurize([mol]))
		df[feature_type] = features
		elif feature_type == "NNScore":
		print("Currently conducting NNScore Featurization.")
		protein_pdbs = list(df["protein_pdb"])
		ligand_pdbs = list(df["ligand_pdb"])
		complexes = zip(ligand_pdbs, protein_pdbs)
		df[featurizer.__class__.__name__] = features

		pool = mp.Pool(processes=mp.cpu_count())
		features = pool.map(map_function, complexes)
		pool.terminate()
		def _add_user_specified_features(self, df):
		"""Merge user specified features.

		Merge features included in dataset provided by user
		into final features dataframe
		"""
		if self.user_specified_features is not None:
		log("Adding user-defined features.", self.verbose)
		features_data = []
		for _, row in df.iterrows():
		# pandas rows are tuples (row_num, row_data)
		feature_list = []
		for feature_name in self.user_specified_features:
		feature_list.append(row[feature_name])
		features_data.append({"user-specified-features": np.array(feature_list)})
		df["user-specified-features"] = pd.DataFrame(features_data)

		features = np.concatenate(features)
		df[feature_type] = list(features)
		else:
		raise ValueError("Unsupported feature_type requested.")

		def map_function(data_tuple):
		def map_function(data_tuple, featurizer):
		featurizer = NNScoreComplexFeaturizer()
		ligand_pdb, protein_pdb = data_tuple
		ind, ligand_pdb, protein_pdb = data_tuple
		print("Mapping on ind %d" % ind)
		print("ind, type(ligand_pdb), type(protein_pdb): %s " % str((ind, type(ligand_pdb), type(protein_pdb))))
		return featurizer.featurize_complexes([ligand_pdb], [protein_pdb])

		class FeaturizedSamples(object):
		@@ -261,36 +259,28 @@ class FeaturizedSamples(object):
		# The standard columns for featurized data.
		colnames = ["mol_id", "smiles", "split"]
		optional_colnames = ["protein_pdb", "ligand_pdb", "ligand_mol2"]
		feature_types = ["user-specified-features", "RDKIT-descriptors", "ECFP", "NNScore"]

		@staticmethod
		def get_sorted_task_names(df):
		"""
		Given metadata df, return sorted names of tasks.
		"""
		column_names = df.keys()
		task_names = (set(column_names) -
		set(FeaturizedSamples.colnames) -
		set(FeaturizedSamples.optional_colnames) -
		set(FeaturizedSamples.feature_types))
		return sorted(list(task_names))

		def __init__(self, feature_dir, dataset_files=None, overwrite=True,
		reload_data=False):
		def __init__(self, samples_dir, featurizers, dataset_files=None,
		overwrite=True, reload_data=False):
		"""
		Initialiize FeaturizedSamples

		If feature_dir does not exist, must specify dataset_files. Then feature_dir
		is created and populated. If feature_dir exists (created by previous call to
		If samples_dir does not exist, must specify dataset_files. Then samples_dir
		is created and populated. If samples_dir exists (created by previous call to
		FeaturizedSamples), then dataset_files cannot be specified. If overwrite is
		set and dataset_files is provided, will overwrite old dataset_files with
		new.
		"""
		self.dataset_files = dataset_files
		self.feature_types = (
		["user-specified-features"] +
		[featurizer.__class__.__name__ for featurizer in featurizers])

		self.featurizers = featurizers

		if not os.path.exists(feature_dir):
		os.makedirs(feature_dir)
		self.feature_dir = feature_dir
		if not os.path.exists(samples_dir):
		os.makedirs(samples_dir)
		self.samples_dir = samples_dir
		if os.path.exists(self._get_compounds_filename()) and reload_data:
		compounds_df = load_from_disk(self._get_compounds_filename())
		else:
		@@ -315,13 +305,13 @@ class FeaturizedSamples(object):
		"""
		Get standard location for file listing compounds in this dataframe.
		"""
		return os.path.join(self.feature_dir, "compounds.joblib")
		return os.path.join(self.samples_dir, "compounds.joblib")

		def _get_dataset_paths_filename(self):
		"""
		Get standard location for file listing dataset_files.
		"""
		return os.path.join(self.feature_dir, "datasets.joblib")
		return os.path.join(self.samples_dir, "datasets.joblib")

		def _get_compounds(self):
		"""
		@@ -362,7 +352,7 @@ class FeaturizedSamples(object):
		for ind, row in df.iterrows():
		if row["mol_id"] in compound_ids:
		visible_inds.append(ind)
		yield df.iloc[visible_inds]
		yield df.loc[visible_inds]

		def train_test_split(self, splittype, train_dir, test_dir, seed=None,
		frac_train=.8):
		@@ -377,9 +367,13 @@ class FeaturizedSamples(object):
		train_inds, test_inds = self._train_test_specified_split()
		else:
		raise ValueError("improper splittype.")
		train_samples = FeaturizedSamples(train_dir, self.dataset_files)
		train_samples = FeaturizedSamples(samples_dir=train_dir,
		dataset_files=self.dataset_files,
		featurizers=self.featurizers)
		train_samples._set_compound_df(self.compounds_df.iloc[train_inds])
		test_samples = FeaturizedSamples(test_dir, self.dataset_files)
		test_samples = FeaturizedSamples(samples_dir=test_dir,
		dataset_files=self.dataset_files,
		featurizers=self.featurizers)
		test_samples._set_compound_df(self.compounds_df.iloc[test_inds])

		return train_samples, test_samples

deepchem/featurizers/grid_featurizer.py

0 → 100644

+1120 −0

File added.

Preview size limit exceeded, changes collapsed.

Admin message