Removing verbosity from deepchem (b10de1da) · Commits · 钟慕尧 / deepchem

deepchem/data/datasets.py

+10 −70

Original line number	Diff line number	Diff line
		@@ -175,9 +175,9 @@ class Dataset(object):

		>>> newx, newy, neww = fn(x, y, w)

		It might be called only once with the whole dataset, or multiple times with different
		subsets of the data. Each time it is called, it should transform the samples and return
		the transformed data.
		It might be called only once with the whole dataset, or multiple times with
		different subsets of the data. Each time it is called, it should transform
		the samples and return the transformed data.

		Parameters
		----------
		@@ -317,7 +317,8 @@ class NumpyDataset(Dataset):
		>>> print(x, y, w, id)
		"""
		n_samples = self._X.shape[0]
		return ((self._X[i], self._y[i], self._w[i], self._ids[i]) for i in range(n_samples))
		return ((self._X[i], self._y[i], self._w[i], self._ids[i])
		for i in range(n_samples))

		def transform(self, fn, **args):
		"""Construct a new dataset by applying a transformation to every sample in this dataset.
		@@ -326,9 +327,9 @@ class NumpyDataset(Dataset):

		>>> newx, newy, neww = fn(x, y, w)

		It might be called only once with the whole dataset, or multiple times with different
		subsets of the data. Each time it is called, it should transform the samples and return
		the transformed data.
		It might be called only once with the whole dataset, or multiple times with
		different subsets of the data. Each time it is called, it should transform
		the samples and return the transformed data.

		Parameters
		----------
		@@ -416,8 +417,7 @@ class DiskDataset(Dataset):
		"""
		metadata_df = pd.DataFrame(
		metadata_entries,
		columns=('basename','task_names', 'ids',
		'X', 'y', 'w'))
		columns=('basename','task_names', 'ids', 'X', 'y', 'w'))
		return metadata_df

		@staticmethod
		@@ -597,48 +597,9 @@ class DiskDataset(Dataset):
		newx, newy, neww = fn(X, y, w)
		basename = "dataset-%d" % shard_num
		metadata_rows.append(DiskDataset.write_data_to_disk(
		out_dir, basename, tasks, newx, newy, neww, ids, False))
		out_dir, basename, tasks, newx, newy, neww, ids))
		return DiskDataset(data_dir=out_dir, metadata_rows=metadata_rows)

		def reshard(self, shard_size):
		"""Reshards data to have specified shard size."""
		# Create temp directory to store resharded version
		reshard_dir = tempfile.mkdtemp()
		new_metadata = []
		# Write data in new shards
		ind = 0
		tasks = self.get_task_names()
		X_next = np.zeros((0,) + self.get_data_shape())
		y_next = np.zeros((0,) + (len(tasks),))
		w_next = np.zeros((0,) + (len(tasks),))
		ids_next = np.zeros((0,), dtype=object)
		for (X, y, w, ids) in self.itershards():
		X_next = np.vstack([X_next, X])
		y_next = np.vstack([y_next, y])
		w_next = np.vstack([w_next, w])
		ids_next = np.concatenate([ids_next, ids])
		while len(X_next) > shard_size:
		X_batch, X_next = X_next[:shard_size], X_next[shard_size:]
		y_batch, y_next = y_next[:shard_size], y_next[shard_size:]
		w_batch, w_next = w_next[:shard_size], w_next[shard_size:]
		ids_batch, ids_next = ids_next[:shard_size], ids_next[shard_size:]
		new_basename = "reshard-%d" % ind
		new_metadata.append(DiskDataset.write_data_to_disk(
		reshard_dir, new_basename, tasks, X_batch, y_batch, w_batch, ids_batch))
		ind += 1
		# Handle spillover from last shard
		new_basename = "reshard-%d" % ind
		new_metadata.append(DiskDataset.write_data_to_disk(
		reshard_dir, new_basename, tasks, X_next, y_next, w_next, ids_next))
		ind += 1
		# Get new metadata rows
		resharded_dataset = DiskDataset(
		data_dir=reshard_dir, tasks=tasks, metadata_rows=new_metadata)
		shutil.rmtree(self.data_dir)
		shutil.move(reshard_dir, self.data_dir)
		self.metadata_df = resharded_dataset.metadata_df
		self.save_to_disk()

		@staticmethod
		def from_numpy(X, y, w=None, ids=None, tasks=None, data_dir=None):
		"""Creates a DiskDataset object from specified Numpy arrays."""
		@@ -696,27 +657,6 @@ class DiskDataset(Dataset):
		subset_dir, basename, tasks, X, y, w, ids))
		return DiskDataset(data_dir=subset_dir, metadata_rows=metadata_rows)

		def reshard_shuffle(self, reshard_size=10, num_reshards=3):
		"""Shuffles by resharding, shuffling shards, undoing resharding."""
		######################################################### TIMING
		time1 = time.time()
		######################################################### TIMING
		for i in range(num_reshards):
		orig_shard_size = self.get_shard_size()
		log("Resharding to shard-size %d." % reshard_size, self.verbose)
		self.reshard(shard_size=reshard_size)
		log("Shuffling shard order.", self.verbose)
		self.shuffle_shards()
		log("Resharding to original shard-size %d." % orig_shard_size,
		self.verbose)
		self.reshard(shard_size=orig_shard_size)
		self.shuffle_each_shard()
		######################################################### TIMING
		time2 = time.time()
		log("TIMING: reshard_shuffle took %0.3f s" % (time2-time1),
		self.verbose)
		######################################################### TIMING

		def sparse_shuffle(self):
		"""Shuffling that exploits data sparsity to shuffle large datasets.

deepchem/feat/init.py

+3 −2

Original line number	Diff line number	Diff line
		@@ -9,8 +9,9 @@ __author__ = "Bharath Ramsundar"
		__copyright__ = "Copyright 2016, Stanford University"
		__license__ = "GPL"

		# TODO(rbharath): Handle this * import and replace with explicit imports later
		from deepchem.feat.base_classes import *
		from deepchem.feat.base_classes import Featurizer
		from deepchem.feat.base_classes import ComplexFeaturizer
		from deepchem.feat.base_classes import UserDefinedFeaturizer
		from deepchem.feat.graph_features import ConvMolFeaturizer
		from deepchem.feat.fingerprints import CircularFingerprint
		from deepchem.feat.basic import RDKitDescriptors

deepchem/feat/base_classes.py

+14 −59

Original line number	Diff line number	Diff line
		@@ -11,17 +11,6 @@ __author__ = "Steven Kearnes"
		__copyright__ = "Copyright 2014, Stanford University"
		__license__ = "BSD 3-clause"

		def resolve_featurizer(name):
		"""
		Resolve featurizer class from a string.

		Parameters
		----------
		name : str
		Featurizer name.
		"""
		return get_featurizers()[name]

		class ComplexFeaturizer(object):
		""""
		Abstract class for calculating features for mol/protein complexes.
		@@ -33,7 +22,8 @@ class ComplexFeaturizer(object):
		"""
		name = None

		def featurize_complexes(self, mol_pdbs, protein_pdbs, verbosity=None, log_every_n=1000):
		def featurize_complexes(self, mol_pdbs, protein_pdbs, verbose=True,
		log_every_n=1000):
		"""
		Calculate features for mol/protein complexes.

		@@ -49,7 +39,7 @@ class ComplexFeaturizer(object):

		features = []
		for i, (mol_pdb, protein_pdb) in enumerate(zip(mol_pdbs, protein_pdbs)):
		if verbosity is not None and i % log_every_n == 0:
		if verbose and i % log_every_n == 0:
		log("Featurizing %d / %d" % (i, len(mol_pdbs)))
		features.append(self._featurize_complex(mol_pdb, protein_pdb))
		features = np.asarray(features)
		@@ -98,8 +88,7 @@ class Featurizer(object):
		name = None
		topo_view = False

		def featurize(self, mols, parallel=False, client_kwargs=None,
		view_flags=None, verbosity=None, log_every_n=1000):
		def featurize(self, mols, verbose=True, log_every_n=1000):
		"""
		Calculate features for molecules.

		@@ -107,39 +96,13 @@ class Featurizer(object):
		----------
		mols : iterable
		RDKit Mol objects.
		parallel : bool, optional
		Whether to train subtrainers in parallel using
		IPython.parallel (default False).
		client_kwargs : dict, optional
		Keyword arguments for IPython.parallel Client.
		view_flags : dict, optional
		Flags for IPython.parallel LoadBalancedView.
		"""
		if self.conformers and isinstance(mols, types.GeneratorType):
		mols = list(mols)
		assert verbosity in [None, "low", "high"]

		if parallel:
		from IPython.parallel import Client

		if client_kwargs is None:
		client_kwargs = {}
		if view_flags is None:
		view_flags = {}
		client = Client(**client_kwargs)
		client.direct_view().use_dill() # use dill
		view = client.load_balanced_view()
		view.set_flags(**view_flags)
		call = view.map(self._featurize, mols, block=False)
		features = call.get()

		# get output from engines
		call.display_outputs()

		else:
		features = []
		for i, mol in enumerate(mols):
		if verbosity is not None and i % log_every_n == 0:
		if verbose and i % log_every_n == 0:
		log("Featurizing %d / %d" % (i, len(mols)))
		if mol is not None:
		features.append(self._featurize(mol))
		@@ -163,8 +126,7 @@ class Featurizer(object):
		"""
		raise NotImplementedError('Featurizer is not defined.')

		def __call__(self, mols, parallel=False, client_kwargs=None,
		view_flags=None):
		def __call__(self, mols):
		"""
		Calculate features for molecules.

		@@ -172,15 +134,8 @@ class Featurizer(object):
		----------
		mols : iterable
		RDKit Mol objects.
		parallel : bool, optional
		Whether to train subtrainers in parallel using
		IPython.parallel (default False).
		client_kwargs : dict, optional
		Keyword arguments for IPython.parallel Client.
		view_flags : dict, optional
		Flags for IPython.parallel LoadBalancedView.
		"""
		return self.featurize(mols, parallel, client_kwargs, view_flags)
		"""
		return self.featurize(mols)

		def conformer_container(self, mols, features):
		"""

deepchem/feat/grid_featurizer.py

+15 −32

Original line number	Diff line number	Diff line
		@@ -20,9 +20,7 @@ import tempfile
		import os
		import shutil
		import multiprocessing as mp
		############################################################## DEBUG
		import time
		############################################################## DEBUG


		"""
		@@ -842,12 +840,8 @@ class GridFeaturizer(ComplexFeaturizer):
		save_intermediates=False, ligand_only=False,
		box_width=16.0, voxel_width=1.0, voxelize_features=True,
		voxel_feature_types=[], flatten=False, parallel=False,
		verbosity=None, **kwargs):
		self.verbosity = verbosity
		#################################################### DEBUG
		print("self.verbosity")
		print(self.verbosity)
		#################################################### DEBUG
		verbose=True, **kwargs):
		self.verbose = verbose
		self.parallel = parallel
		self.flatten = flatten

		@@ -893,8 +887,7 @@ class GridFeaturizer(ComplexFeaturizer):
		mol_f.writelines(ligand_pdb_lines)
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: Writing ligand took %0.3f s" % (time2-time1),
		self.verbosity)
		log("TIMING: Writing ligand took %0.3f s" % (time2-time1), self.verbose)
		############################################################## TIMING

		############################################################## TIMING
		@@ -905,8 +898,7 @@ class GridFeaturizer(ComplexFeaturizer):
		protein_f.writelines(protein_pdb_lines)
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: Writing protein took %0.3f s" % (time2-time1),
		self.verbosity)
		log("TIMING: Writing protein took %0.3f s" % (time2-time1), self.verbose)
		############################################################## TIMING

		features_dict = self._transform(protein_pdb_file, ligand_pdb_file)
		@@ -959,7 +951,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: Loading protein coordinates took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING
		############################################################## TIMING
		time1 = time.time()
		@@ -969,18 +961,9 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: Loading ligand coordinates took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		############################################################# DEBUG
		print("self.feature_types")
		print(self.feature_types)
		############################################################# DEBUG
		############################################################# DEBUG
		print("self.voxel_feature_types")
		print(self.voxel_feature_types)
		############################################################# DEBUG

		if "ecfp" in self.feature_types:
		ecfp_array = _compute_ecfp_features(
		ligand_ob, self.ecfp_degree, self.ecfp_power)
		@@ -996,7 +979,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: Centroid processing took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		if "splif" in self.feature_types:
		@@ -1020,7 +1003,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: ecfp voxel computataion took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING
		if "splif" in self.voxel_feature_types:
		############################################################## TIMING
		@@ -1032,7 +1015,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: splif voxel computataion took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		if "hbond" in self.voxel_feature_types:
		@@ -1047,7 +1030,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: hbond voxel computataion took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		if "sybyl" in self.voxel_feature_types:
		@@ -1060,7 +1043,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: sybyl voxel computataion took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		if "pi_stack" in self.voxel_feature_types:
		@@ -1073,7 +1056,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: pi_stack voxel computataion took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		if "cation_pi" in self.voxel_feature_types:
		@@ -1086,7 +1069,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: cation_pi voxel computataion took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		if "salt_bridge" in self.voxel_feature_types:
		@@ -1099,7 +1082,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: salt_bridge voxel computataion took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		if "charge" in self.voxel_feature_types:
		@@ -1111,7 +1094,7 @@ class GridFeaturizer(ComplexFeaturizer):
		############################################################## TIMING
		time2 = time.time()
		log("TIMING: charge voxel computataion took %0.3f s" % (time2-time1),
		self.verbosity)
		self.verbose)
		############################################################## TIMING

		transformed_systems = {}

deepchem/hyper/init.py

+13 −14

Original line number	Diff line number	Diff line
		@@ -17,10 +17,9 @@ class HyperparamOpt(object):
		Provides simple hyperparameter search capabilities.
		"""

		def __init__(self, model_class, verbosity="high"):
		def __init__(self, model_class, verbose=True):
		self.model_class = model_class
		assert verbosity in [None, "low", "high"]
		self.verbosity = verbosity
		self.verbose = verbose

		# TODO(rbharath): This function is complicated and monolithic. Is there a nice
		# way to refactor this?
		@@ -53,20 +52,20 @@ class HyperparamOpt(object):
		for ind, hyperparameter_tuple in enumerate(itertools.product(*hyperparam_vals)):
		model_params = {}
		log("Fitting model %d/%d" % (ind+1, number_combinations),
		self.verbosity, "high")
		self.verbose)
		for hyperparam, hyperparam_val in zip(hyperparams, hyperparameter_tuple):
		model_params[hyperparam] = hyperparam_val
		log("hyperparameters: %s" % str(model_params),
		self.verbosity, "high")
		log("hyperparameters: %s" % str(model_params), self.verbose)

		if logdir is not None:
		model_dir = os.path.join(logdir, str(ind))
		log("model_dir is %s" % model_dir, self.verbosity, "high")
		log("model_dir is %s" % model_dir, self.verbose)
		try:
		os.makedirs(model_dir)
		except OSError:
		if not os.path.isdir(model_dir):
		log("Error creating model_dir, using tempfile directory", self.verbosity, "high")
		log("Error creating model_dir, using tempfile directory",
		self.verbose)
		model_dir = tempfile.mkdtemp()
		else:
		model_dir = tempfile.mkdtemp()
		@@ -94,11 +93,11 @@ class HyperparamOpt(object):

		log("Model %d/%d, Metric %s, Validation set %s: %f" %
		(ind+1, number_combinations, metric.name, ind, valid_score),
		self.verbosity, "low")
		self.verbose)
		log("\tbest_validation_score so far: %f" % best_validation_score,
		self.verbosity, "low")
		self.verbose)
		if best_model is None:
		log("No models trained correctly.", self.verbosity, "low")
		log("No models trained correctly.", self.verbose)
		# arbitrarily return last model
		best_model, best_hyperparams = model, hyperparameter_tuple
		return best_model, best_hyperparams, all_scores
		@@ -109,7 +108,7 @@ class HyperparamOpt(object):
		[metric], train_csv_out.name, train_stats_out.name)
		train_score = multitask_scores[metric.name]
		log("Best hyperparameters: %s" % str(best_hyperparams),
		self.verbosity, "low")
		log("train_score: %f" % train_score, self.verbosity, "low")
		log("validation_score: %f" % best_validation_score, self.verbosity, "low")
		self.verbose)
		log("train_score: %f" % train_score, self.verbose)
		log("validation_score: %f" % best_validation_score, self.verbose)
		return best_model, best_hyperparams, all_scores

Admin message