merged grid featurization into deepchem (15958094) · Commits · 钟慕尧 / deepchem

deepchem/featurizers/init.py

+0 −29

Original line number	Diff line number	Diff line
		@@ -11,35 +11,6 @@ __author__ = "Steven Kearnes"
		__copyright__ = "Copyright 2014, Stanford University"
		__license__ = "BSD 3-clause"


		def get_featurizers():
		"""Compile a dict mapping strings to featurizer classes."""

		# import all Featurizer subclasses so __subclasses__ will work
		# these have to be local imports to avoid circular imports
		from .basic import MolecularWeight, SimpleDescriptors
		from .coulomb_matrices import CoulombMatrix
		from .dragon import DragonDescriptors
		from .esp import ESP
		from .fingerprints import CircularFingerprint
		from .images import MolImage
		from .scaffolds import ScaffoldGenerator
		from .shape_grid import ShapeGrid

		featurizers = {}
		for klass in Featurizer.__subclasses__():
		assert klass.name is not None, (klass.__name__ +
		" 'name' attribute is None.")
		if isinstance(klass.name, list):
		for name in klass.name:
		assert name not in featurizers
		featurizers[name] = klass
		else:
		assert klass.name not in featurizers
		featurizers[klass.name] = klass
		return featurizers


		def resolve_featurizer(name):
		"""
		Resolve featurizer class from a string.

deepchem/featurizers/basic.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -31,7 +31,7 @@ class MolecularWeight(Featurizer):
		return wt


		class SimpleDescriptors(Featurizer):
		class RDKitDescriptors(Featurizer):
		"""
		RDKit descriptors.

deepchem/featurizers/featurize.py

+80 −64

Original line number	Diff line number	Diff line
		@@ -11,7 +11,7 @@ import numpy as np
		import csv
		from rdkit import Chem
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.featurizers.basic import SimpleDescriptors
		from deepchem.featurizers.basic import RDKitDescriptors
		from deepchem.utils.save import log
		from deepchem.utils.save import save_to_disk
		from deepchem.utils.save import load_from_disk
		@@ -90,7 +90,9 @@ class DataFeaturizer(object):
		def __init__(self, tasks, smiles_field, split_field=None,
		id_field=None, threshold=None, user_specified_features=None,
		protein_pdb_field=None, ligand_pdb_field=None,
		ligand_mol2_field=None, verbose=False, log_every_n=1000):
		ligand_mol2_field=None,
		compound_featurizers=[], complex_featurizers=[],
		verbose=False, log_every_n=1000):
		"""Extracts data from input as Pandas data frame"""
		if not isinstance(tasks, list):
		raise ValueError("tasks must be a list.")
		@@ -106,11 +108,12 @@ class DataFeaturizer(object):
		self.ligand_pdb_field = ligand_pdb_field
		self.ligand_mol2_field = ligand_mol2_field
		self.user_specified_features = user_specified_features
		self.compound_featurizers = compound_featurizers
		self.complex_featurizers = complex_featurizers
		self.verbose = verbose
		self.log_every_n = log_every_n

		def featurize(self, input_file, feature_types, feature_dir,
		shard_size=128):
		def featurize(self, input_file, feature_dir, shard_size=128):
		"""Featurize provided file and write to specified location."""
		input_type = _get_input_type(input_file)

		@@ -136,10 +139,24 @@ class DataFeaturizer(object):
		log("Sharding and standardizing into shard-%s / %s shards" % (str(j+1), len(interval_points)-1), self.verbose)
		raw_df_shard = raw_df.iloc[range(interval_points[j], interval_points[j+1])]

		print("featurize()")
		print("raw_df_shard.keys()")
		print(raw_df_shard.keys())
		print("self.id_field")
		print(self.id_field)
		df = self._standardize_df(raw_df_shard)
		for feature_type in feature_types:
		log("Currently feauturizing feature_type: %s" % feature_type, self.verbose)
		self._featurize_df(df, feature_type)
		log("Aggregating User-Specified Features", self.verbose)
		self._add_user_specified_features(df)

		for compound_featurizer in self.compound_featurizers:
		log("Currently feauturizing feature_type: %s"
		% compound_featurizer.__class__.__name__, self.verbose)
		self._featurize_compounds(df, compound_featurizer)

		for complex_featurizer in self.complex_featurizers:
		log("Currently feauturizing feature_type: %s"
		% complex_featurizer.__class__.__name__, self.verbose)
		self._featurize_complexes(df, complex_featurizer)

		shard_out = os.path.join(feature_dir, "features_shard%d.joblib" % j)
		save_to_disk(df, shard_out)
		@@ -169,6 +186,7 @@ class DataFeaturizer(object):
		def _standardize_df(self, ori_df):
		"""Copy specified columns to new df with standard column names."""
		df = pd.DataFrame(ori_df[[self.id_field]])
		df.columns = ["mol_id"]
		df["smiles"] = ori_df[[self.smiles_field]]
		for task in self.tasks:
		df[task] = ori_df[[task]]
		@@ -182,9 +200,37 @@ class DataFeaturizer(object):
		df["ligand_mol2"] = ori_df[[self.ligand_mol2_field]]
		return df

		def _featurize_df(self, df, feature_type):
		def _featurize_complexes(self, df, featurizer):
		"""Generates circular fingerprints for dataset."""
		if feature_type == "user-specified-features":
		protein_pdbs = list(df["protein_pdb"])
		ligand_pdbs = list(df["ligand_pdb"])
		complexes = zip(ligand_pdbs, protein_pdbs)

		features = featurizer.featurize_complexes(ligand_pdbs, protein_pdbs)
		df[featurizer.__class__.__name__] = list(features)

		def _featurize_compounds(self, df, featurizer):
		"""Featurize individual compounds.

		Given a featurizer that operates on individual chemical compounds
		or macromolecules, compute & add features for that compound to the
		features dataframe
		"""
		features = []
		sample_smiles = df["smiles"].tolist()
		for ind, smiles in enumerate(sample_smiles):
		if ind % self.log_every_n == 0:
		log("Featurizing sample %d" % ind, self.verbose)
		mol = Chem.MolFromSmiles(smiles)
		features.append(featurizer.featurize([mol]))
		df[featurizer.__class__.__name__] = features

		def _add_user_specified_features(self, df):
		"""Merge user specified features.

		Merge features included in dataset provided by user
		into final features dataframe
		"""
		if self.user_specified_features is not None:
		log("Adding user-defined features.", self.verbose)
		features_data = []
		@@ -193,39 +239,11 @@ class DataFeaturizer(object):
		feature_list = []
		for feature_name in self.user_specified_features:
		feature_list.append(row[feature_name])
		features_data.append({feature_type: np.array(feature_list)})
		df[feature_type] = pd.DataFrame(features_data)
		return
		elif feature_type in ["ECFP", "RDKIT-descriptors"]:
		if feature_type == "ECFP":
		log("Generating ECFP circular fingerprints.", self.verbose)
		featurizer = CircularFingerlog(size=1024)
		elif feature_type == "RDKIT-descriptors":
		log("Generating RDKIT descriptors.", self.verbose)
		featurizer = SimpleDescriptors()
		features = []
		sample_smiles = df["smiles"].tolist()
		for ind, smiles in enumerate(sample_smiles):
		if ind % self.log_every_n == 0:
		log("Featurizing sample %d" % ind, self.verbose)
		mol = Chem.MolFromSmiles(smiles)
		features.append(featurizer.featurize([mol]))
		df[feature_type] = features
		elif feature_type == "NNScore":
		log("Currently conducting NNScore Featurization.", self.verbose)
		protein_pdbs = list(df["protein_pdb"])
		ligand_pdbs = list(df["ligand_pdb"])
		complexes = zip(range(len(ligand_pdbs)), ligand_pdbs, protein_pdbs)
		features_data.append({"user-specified-features": np.array(feature_list)})
		df["user-specified-features"] = pd.DataFrame(features_data)

		pool = mp.Pool(processes=mp.cpu_count())
		features = pool.map(map_function, complexes)
		pool.terminate()
		features = np.concatenate(features)
		df[feature_type] = list(features)
		else:
		raise ValueError("Unsupported feature_type requested.")

		def map_function(data_tuple):
		def map_function(data_tuple, featurizer):
		featurizer = NNScoreComplexFeaturizer()
		ind, ligand_pdb, protein_pdb = data_tuple
		print("Mapping on ind %d" % ind)
		@@ -239,36 +257,28 @@ class FeaturizedSamples(object):
		# The standard columns for featurized data.
		colnames = ["mol_id", "smiles", "split"]
		optional_colnames = ["protein_pdb", "ligand_pdb", "ligand_mol2"]
		feature_types = ["user-specified-features", "RDKIT-descriptors", "ECFP", "NNScore"]

		@staticmethod
		def get_sorted_task_names(df):
		"""
		Given metadata df, return sorted names of tasks.
		"""
		column_names = df.keys()
		task_names = (set(column_names) -
		set(FeaturizedSamples.colnames) -
		set(FeaturizedSamples.optional_colnames) -
		set(FeaturizedSamples.feature_types))
		return sorted(list(task_names))

		def __init__(self, feature_dir, dataset_files=None, overwrite=True,
		reload_data=False):
		def __init__(self, samples_dir, featurizers, dataset_files=None,
		overwrite=True, reload_data=False):
		"""
		Initialiize FeaturizedSamples

		If feature_dir does not exist, must specify dataset_files. Then feature_dir
		is created and populated. If feature_dir exists (created by previous call to
		If samples_dir does not exist, must specify dataset_files. Then samples_dir
		is created and populated. If samples_dir exists (created by previous call to
		FeaturizedSamples), then dataset_files cannot be specified. If overwrite is
		set and dataset_files is provided, will overwrite old dataset_files with
		new.
		"""
		self.dataset_files = dataset_files
		self.feature_types = (
		["user-specified-features"] +
		[featurizer.__class__.__name__ for featurizer in featurizers])

		self.featurizers = featurizers

		if not os.path.exists(feature_dir):
		os.makedirs(feature_dir)
		self.feature_dir = feature_dir
		if not os.path.exists(samples_dir):
		os.makedirs(samples_dir)
		self.samples_dir = samples_dir
		if os.path.exists(self._get_compounds_filename()) and reload_data:
		compounds_df = load_from_disk(self._get_compounds_filename())
		else:
		@@ -293,13 +303,13 @@ class FeaturizedSamples(object):
		"""
		Get standard location for file listing compounds in this dataframe.
		"""
		return os.path.join(self.feature_dir, "compounds.joblib")
		return os.path.join(self.samples_dir, "compounds.joblib")

		def _get_dataset_paths_filename(self):
		"""
		Get standard location for file listing dataset_files.
		"""
		return os.path.join(self.feature_dir, "datasets.joblib")
		return os.path.join(self.samples_dir, "datasets.joblib")

		def _get_compounds(self):
		"""
		@@ -308,6 +318,8 @@ class FeaturizedSamples(object):
		compound_rows = []
		for dataset_file in self.dataset_files:
		df = load_from_disk(dataset_file)
		print("_get_compounds.df.keys()")
		print(df.keys())
		compound_ids = list(df["mol_id"])
		smiles = list(df["smiles"])
		if "split" in df.keys():
		@@ -355,9 +367,13 @@ class FeaturizedSamples(object):
		train_inds, test_inds = self._train_test_specified_split()
		else:
		raise ValueError("improper splittype.")
		train_samples = FeaturizedSamples(train_dir, self.dataset_files)
		train_samples = FeaturizedSamples(samples_dir=train_dir,
		dataset_files=self.dataset_files,
		featurizers=self.featurizers)
		train_samples._set_compound_df(self.compounds_df.iloc[train_inds])
		test_samples = FeaturizedSamples(test_dir, self.dataset_files)
		test_samples = FeaturizedSamples(samples_dir=test_dir,
		dataset_files=self.dataset_files,
		featurizers=self.featurizers)
		test_samples._set_compound_df(self.compounds_df.iloc[test_inds])

		return train_samples, test_samples

deepchem/featurizers/grid_featurizer.py

0 → 100644

+1120 −0

File added.

Preview size limit exceeded, changes collapsed.

deepchem/featurizers/tests/test_basic.py

+5 −5

Original line number	Diff line number	Diff line
		@@ -6,7 +6,7 @@ import unittest

		from rdkit import Chem

		from deepchem.featurizers.basic import MolecularWeight, SimpleDescriptors
		from deepchem.featurizers.basic import MolecularWeight, RDKitDescriptors


		class TestMolecularWeight(unittest.TestCase):
		@@ -28,9 +28,9 @@ class TestMolecularWeight(unittest.TestCase):
		assert np.allclose(self.engine([self.mol]), 180, atol=0.1)


		class TestSimpleDescriptors(unittest.TestCase):
		class TestRDKitDescriptors(unittest.TestCase):
		"""
		Test SimpleDescriptors.
		Test RDKitDescriptors.
		"""
		def setUp(self):
		"""
		@@ -38,9 +38,9 @@ class TestSimpleDescriptors(unittest.TestCase):
		"""
		smiles = 'CC(=O)OC1=CC=CC=C1C(=O)O'
		self.mol = Chem.MolFromSmiles(smiles)
		self.engine = SimpleDescriptors()
		self.engine = RDKitDescriptors()

		def testSimpleDescriptors(self):
		def testRDKitDescriptors(self):
		"""
		Test simple descriptors.
		"""

Admin message