added hyperparameter search, fixed memory errors (bdb4e4b2) · Commits · 钟慕尧 / deepchem

datasets/pdbbind_core_5_df.pkl.gz

0 → 100644

+534 KiB

File added.

No diff preview for this file type.

deepchem/featurizers/init.py

+2 −1

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@ import types
		import numpy as np
		from rdkit import Chem
		from rdkit.Chem import rdGeometry, rdMolTransforms
		from deepchem.utils.save import log

		__author__ = "Steven Kearnes"
		__copyright__ = "Copyright 2014, Stanford University"
		@@ -78,7 +79,7 @@ class ComplexFeaturizer(object):
		features = []
		for i, (mol_pdb, protein_pdb) in enumerate(zip(mol_pdbs, protein_pdbs)):
		if i % log_every_n == 0:
		print("Featurizing %d / %d" % (i, len(mol_pdbs)))
		log("Featurizing %d / %d" % (i, len(mol_pdbs)))
		features.append(self._featurize_complex(mol_pdb, protein_pdb))
		features = np.asarray(features)
		return features

deepchem/featurizers/featurize.py

+21 −46

Original line number	Diff line number	Diff line
		@@ -12,6 +12,7 @@ import csv
		from rdkit import Chem
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.featurizers.basic import SimpleDescriptors
		from deepchem.utils.save import log
		from deepchem.utils.save import save_to_disk
		from deepchem.utils.save import load_from_disk
		from deepchem.utils.save import load_pandas_from_disk
		@@ -78,30 +79,6 @@ def _get_fields(input_file):
		else:
		raise ValueError("Unrecognized extension for %s" % input_file)

		'''
		def _get_raw_samples(input_file, iterator=True):
		"""Returns an iterator over all rows in input_file"""
		input_type = _get_input_type(input_file)
		if input_type == "csv":
		with open(input_file, "rb") as inp_file_obj:
		for ind, row in enumerate(csv.reader(inp_file_obj)):
		# Skip labels
		if ind == 0:
		continue
		if row is not None:
		yield row
		elif input_type == "pandas-joblib":
		dataframe = load_from_disk(input_file)
		for _, row in dataframe.iterrows():
		yield row
		elif input_type == "pandas-pickle":
		dataframe = load_pickle_from_disk(input_file)
		for _, row in dataframe.iterrows():
		yield row
		else:
		raise ValueError("Unrecognized input type for %s" % input_file)
		'''

		class DataFeaturizer(object):
		"""
		Handles loading/featurizing of chemical samples (datapoints).
		@@ -113,8 +90,7 @@ class DataFeaturizer(object):
		def __init__(self, tasks, smiles_field, split_field=None,
		id_field=None, threshold=None, user_specified_features=None,
		protein_pdb_field=None, ligand_pdb_field=None,
		ligand_mol2_field=None,
		verbose=False, log_every_n=1000):
		ligand_mol2_field=None, verbose=False, log_every_n=1000):
		"""Extracts data from input as Pandas data frame"""
		if not isinstance(tasks, list):
		raise ValueError("tasks must be a list.")
		@@ -133,14 +109,15 @@ class DataFeaturizer(object):
		self.verbose = verbose
		self.log_every_n = log_every_n

		def featurize(self, input_file, feature_types, feature_dir, shard_size=128):
		def featurize(self, input_file, feature_types, feature_dir,
		shard_size=128):
		"""Featurize provided file and write to specified location."""
		input_type = _get_input_type(input_file)

		print("Loading raw samples now.")
		log("Loading raw samples now.", self.verbose)
		raw_df = load_pandas_from_disk(input_file)
		fields = raw_df.keys()
		print("Loaded raw data frame from file.")
		log("Loaded raw data frame from file.", self.verbose)
		def process_raw_sample_helper(row, fields, input_type):
		return self._process_raw_sample(input_type, row, fields)
		process_raw_sample_helper_partial = partial(process_raw_sample_helper,
		@@ -148,7 +125,7 @@ class DataFeaturizer(object):
		input_type=input_type)

		processed_rows = raw_df.apply(process_raw_sample_helper_partial, axis=1)
		print("finished processing rows")
		log("finished processing rows", self.verbose)
		raw_df = pd.DataFrame.from_records(processed_rows)

		nb_sample = raw_df.shape[0]
		@@ -156,11 +133,12 @@ class DataFeaturizer(object):
		0, nb_sample, np.ceil(float(nb_sample)/shard_size)+1, dtype=int)
		shard_files = []
		for j in range(len(interval_points)-1):
		print("Sharding and standardizing into shard-%s / %s shards" % (str(j+1), len(interval_points)-1))
		log("Sharding and standardizing into shard-%s / %s shards" % (str(j+1), len(interval_points)-1), self.verbose)
		raw_df_shard = raw_df.iloc[range(interval_points[j], interval_points[j+1])]

		df = self._standardize_df(raw_df_shard)
		for feature_type in feature_types:
		print("Currently feauturizing feature_type: %s" % feature_type)
		log("Currently feauturizing feature_type: %s" % feature_type, self.verbose)
		self._featurize_df(df, feature_type)

		shard_out = os.path.join(feature_dir, "features_shard%d.joblib" % j)
		@@ -190,8 +168,7 @@ class DataFeaturizer(object):

		def _standardize_df(self, ori_df):
		"""Copy specified columns to new df with standard column names."""
		df = pd.DataFrame([])
		df["mol_id"] = ori_df[[self.id_field]]
		df = pd.DataFrame(ori_df[[self.id_field]])
		df["smiles"] = ori_df[[self.smiles_field]]
		for task in self.tasks:
		df[task] = ori_df[[task]]
		@@ -209,8 +186,7 @@ class DataFeaturizer(object):
		"""Generates circular fingerprints for dataset."""
		if feature_type == "user-specified-features":
		if self.user_specified_features is not None:
		if self.verbose:
		print("Adding user-defined features.")
		log("Adding user-defined features.", self.verbose)
		features_data = []
		for _, row in df.iterrows():
		# pandas rows are tuples (row_num, row_data)
		@@ -222,31 +198,28 @@ class DataFeaturizer(object):
		return
		elif feature_type in ["ECFP", "RDKIT-descriptors"]:
		if feature_type == "ECFP":
		if self.verbose:
		print("Generating ECFP circular fingerprints.")
		featurizer = CircularFingerprint(size=1024)
		log("Generating ECFP circular fingerprints.", self.verbose)
		featurizer = CircularFingerlog(size=1024)
		elif feature_type == "RDKIT-descriptors":
		if self.verbose:
		print("Generating RDKIT descriptors.")
		log("Generating RDKIT descriptors.", self.verbose)
		featurizer = SimpleDescriptors()
		features = []
		sample_smiles = df["smiles"].tolist()
		for ind, smiles in enumerate(sample_smiles):
		if ind % self.log_every_n == 0:
		print("Featurizing sample %d" % ind)
		log("Featurizing sample %d" % ind, self.verbose)
		mol = Chem.MolFromSmiles(smiles)
		features.append(featurizer.featurize([mol]))
		df[feature_type] = features
		elif feature_type == "NNScore":
		print("Currently conducting NNScore Featurization.")
		log("Currently conducting NNScore Featurization.", self.verbose)
		protein_pdbs = list(df["protein_pdb"])
		ligand_pdbs = list(df["ligand_pdb"])
		complexes = zip(ligand_pdbs, protein_pdbs)
		complexes = zip(range(len(ligand_pdbs)), ligand_pdbs, protein_pdbs)

		pool = mp.Pool(processes=mp.cpu_count())
		features = pool.map(map_function, complexes)
		pool.terminate()

		features = np.concatenate(features)
		df[feature_type] = list(features)
		else:
		@@ -254,7 +227,9 @@ class DataFeaturizer(object):

		def map_function(data_tuple):
		featurizer = NNScoreComplexFeaturizer()
		ligand_pdb, protein_pdb = data_tuple
		ind, ligand_pdb, protein_pdb = data_tuple
		print("Mapping on ind %d" % ind)
		print("ind, type(ligand_pdb), type(protein_pdb): %s " % str((ind, type(ligand_pdb), type(protein_pdb))))
		return featurizer.featurize_complexes([ligand_pdb], [protein_pdb])

		class FeaturizedSamples(object):

deepchem/featurizers/nnscore_pdb.py

+6 −4

Original line number	Diff line number	Diff line
		@@ -26,6 +26,7 @@ from deepchem.featurizers.nnscore_utils import cross_product
		from deepchem.featurizers.nnscore_utils import dihedral
		from deepchem.featurizers.nnscore_utils import dot_product
		from deepchem.featurizers.nnscore_utils import vector_subtraction
		from deepchem.utils.save import log

		__author__ = "Bharath Ramsundar and Jacob Durrant"
		__license__ = "GNU General Public License"
		@@ -68,7 +69,7 @@ def remove_redundant_rings(rings):
		del ring_dict[snd_index]
		return ring_dict.values()

		def print_warning(atom, residue, need):
		def print_warning(atom, residue, need, verbose=False):
		"""
		Prints warning if residue has improper structure.

		@@ -88,6 +89,7 @@ def print_warning(atom, residue, need):
		'%s. If this residue is far from the ' % need +
		'active site, this warning may not affect the NNScore.')
		lines = textwrap.wrap(text, 80)
		if verbose:
		for line in lines:
		print line
		print
		@@ -384,12 +386,12 @@ class PDB(object):
		for line in pdb_lines:
		if "CONECT" in line:
		if len(line) < 31:
		warnings.warn(
		log(
		"Bad PDB! Improperly formatted CONECT line (too short)")
		continue
		atom_index = int(line[6:11].strip())
		if atom_index not in self.all_atoms:
		warnings.warn(
		log(
		"Bad PDB! Improper CONECT line: (atom index not loaded)")
		continue
		bonded_atoms = []
		@@ -400,7 +402,7 @@ class PDB(object):
		if line[lower:upper].strip():
		index = int(line[lower:upper])
		if index not in self.all_atoms:
		warnings.warn(
		log(
		"Bad PDB! Improper CONECT line: (bonded atom not loaded)")
		misformatted = True
		break

deepchem/featurizers/tests/data/3zso_ligand_hyd.pdb

0 → 100644

+98 −0

Original line number	Diff line number	Diff line
		COMPND 3zso ligand
		AUTHOR GENERATED BY OPEN BABEL 2.3.90
		ATOM 1 C1 O2N A 1 -11.198 48.636 5.765 1.00 0.00 C
		ATOM 2 C2 O2N A 1 -12.167 49.352 5.098 1.00 0.00 C
		ATOM 3 C3 O2N A 1 -10.134 48.143 5.020 1.00 0.00 C
		ATOM 4 C4 O2N A 1 -12.097 49.593 3.735 1.00 0.00 C
		ATOM 5 C5 O2N A 1 -11.433 45.761 -1.708 1.00 0.00 C
		ATOM 6 C6 O2N A 1 -7.511 45.057 -0.052 1.00 0.00 C
		ATOM 7 C7 O2N A 1 -9.086 44.128 1.474 1.00 0.00 C
		ATOM 8 C8 O2N A 1 -5.245 45.740 3.485 1.00 0.00 C
		ATOM 9 C9 O2N A 1 -6.862 44.158 4.205 1.00 0.00 C
		ATOM 10 C10 O2N A 1 -12.182 45.044 -2.695 1.00 0.00 C
		ATOM 11 C11 O2N A 1 -8.019 44.279 -1.100 1.00 0.00 C
		ATOM 12 C12 O2N A 1 -9.630 43.303 0.478 1.00 0.00 C
		ATOM 13 C13 O2N A 1 -4.369 45.191 4.419 1.00 0.00 C
		ATOM 14 C14 O2N A 1 -5.990 43.546 5.138 1.00 0.00 C
		ATOM 15 C15 O2N A 1 -10.047 48.370 3.658 1.00 0.00 C
		ATOM 16 C16 O2N A 1 -13.377 45.981 -0.234 1.00 0.00 C
		ATOM 17 C17 O2N A 1 -8.028 44.989 1.236 1.00 0.00 C
		ATOM 18 C18 O2N A 1 -6.524 45.211 3.362 1.00 0.00 C
		ATOM 19 C19 O2N A 1 -11.035 49.102 3.001 1.00 0.00 C
		ATOM 20 C20 O2N A 1 -12.013 46.219 -0.492 1.00 0.00 C
		ATOM 21 C21 O2N A 1 -13.513 44.860 -2.408 1.00 0.00 C
		ATOM 22 C22 O2N A 1 -14.081 45.291 -1.222 1.00 0.00 C
		ATOM 23 C23 O2N A 1 -9.062 43.371 -0.784 1.00 0.00 C
		ATOM 24 C24 O2N A 1 -4.734 44.116 5.222 1.00 0.00 C
		ATOM 25 C25 O2N A 1 -10.188 50.469 -1.841 1.00 0.00 C
		ATOM 26 C26 O2N A 1 -10.595 49.237 -1.487 1.00 0.00 C
		ATOM 27 C27 O2N A 1 -8.876 47.847 2.921 1.00 0.00 C
		ATOM 28 C28 O2N A 1 -14.103 46.347 1.041 1.00 0.00 C
		ATOM 29 C29 O2N A 1 -15.626 44.265 -2.377 1.00 0.00 C
		ATOM 30 C30 O2N A 1 -10.094 41.273 -1.577 1.00 0.00 C
		ATOM 31 C31 O2N A 1 -4.295 42.945 7.205 1.00 0.00 C
		ATOM 32 C32 O2N A 1 -10.989 49.402 1.539 1.00 0.00 C
		ATOM 33 C33 O2N A 1 -11.237 46.969 0.581 1.00 0.00 C
		ATOM 34 C34 O2N A 1 -11.872 49.040 -0.717 1.00 0.00 C
		ATOM 35 C35 O2N A 1 -7.456 45.879 2.354 1.00 0.00 C
		ATOM 36 N36 O2N A 1 -8.554 46.498 3.073 1.00 0.00 N
		ATOM 37 N37 O2N A 1 -11.766 48.441 0.652 1.00 0.00 N1+
		ATOM 38 O38 O2N A 1 -14.921 45.465 1.458 1.00 0.00 O
		ATOM 39 O39 O2N A 1 -8.234 48.643 2.230 1.00 0.00 O
		ATOM 40 O40 O2N A 1 -13.856 47.366 1.763 1.00 0.00 O
		ATOM 41 O41 O2N A 1 -14.435 44.230 -3.187 1.00 0.00 O
		ATOM 42 O42 O2N A 1 -15.447 44.991 -1.143 1.00 0.00 O
		ATOM 43 O43 O2N A 1 -9.561 42.589 -1.839 1.00 0.00 O
		ATOM 44 O44 O2N A 1 -3.748 43.655 6.094 1.00 0.00 O
		ATOM 45 H O2N A 1 -9.090 45.948 3.689 1.00 0.00 H
		ATOM 46 H O2N A 1 -12.679 48.334 1.094 1.00 0.00 H
		ATOM 47 H O2N A 1 -13.255 48.019 1.462 1.00 0.00 H
		CONECT 1 2 3
		CONECT 2 1 4
		CONECT 3 1 15
		CONECT 4 2 19
		CONECT 5 10 20
		CONECT 6 11 17
		CONECT 7 12 17
		CONECT 8 13 18
		CONECT 9 14 18
		CONECT 10 5 21
		CONECT 11 6 23
		CONECT 12 7 23
		CONECT 13 8 24
		CONECT 14 9 24
		CONECT 15 3 19 27
		CONECT 16 20 22 28
		CONECT 17 6 7 35
		CONECT 18 8 9 35
		CONECT 19 4 15 32
		CONECT 20 5 16 33
		CONECT 21 10 22 41
		CONECT 22 16 21 42
		CONECT 23 11 12 43
		CONECT 24 13 14 44
		CONECT 25 26
		CONECT 26 25 34
		CONECT 27 15 36 39
		CONECT 28 16 38 40
		CONECT 29 41 42
		CONECT 30 43
		CONECT 31 44
		CONECT 32 19 37
		CONECT 33 20 37
		CONECT 34 26 37
		CONECT 35 17 18 36
		CONECT 36 27 35 45
		CONECT 37 32 33 34 46
		CONECT 38 28
		CONECT 39 27
		CONECT 40 28 47
		CONECT 41 21 29
		CONECT 42 22 29
		CONECT 43 23 30
		CONECT 44 24 31
		CONECT 45 36
		CONECT 46 37
		CONECT 47 40
		MASTER 0 0 0 0 0 0 0 0 47 0 47 0
		END

Admin message