Merge pull request #155 from rbharath/singletask_MUV (7b294483) · Commits · 钟慕尧 / deepchem

deepchem/datasets/init.py

+1 −5

Original line number	Diff line number	Diff line
		@@ -165,21 +165,17 @@ class Dataset(object):
		ids = load_from_disk(row['ids'])
		yield (X, y, w, ids)

		def iterbatches(self, batch_size=None, epoch=1):
		def iterbatches(self, batch_size=None, epoch=0):
		"""
		Returns minibatches from dataset.
		"""
		if batch_size == None:
		batch_size = len(self)
		for i, (X, y, w, ids) in enumerate(self.itershards()):
		log("Iterating on shard-%s/epoch-%s" % (str(i+1), str(epoch+1)),
		self.verbosity)
		nb_sample = np.shape(X)[0]
		interval_points = np.linspace(
		0, nb_sample, np.ceil(float(nb_sample)/batch_size)+1, dtype=int)
		for j in range(len(interval_points)-1):
		log("Iterating on batch-%s/shard-%s/epoch-%s" %
		(str(j+1), str(i+1), str(epoch+1)), self.verbosity)
		indices = range(interval_points[j], interval_points[j+1])
		X_batch = X[indices, :]
		y_batch = y[indices]

deepchem/featurizers/featurize.py

+14 −1

Original line number	Diff line number	Diff line
		@@ -112,7 +112,11 @@ class DataFeaturizer(object):
		reload=False):
		"""Featurize provided file and write to specified location."""
		# If we are not to reload data, or data has not already been featurized.
		if not reload or not os.path.exists(feature_dir):
		perform_featurization = (not os.path.exists(feature_dir)
		or not self._shard_files_exist(feature_dir)
		or not reload)

		if perform_featurization:
		if not os.path.exists(feature_dir):
		os.makedirs(feature_dir)
		input_type = _get_input_type(input_file)
		@@ -166,6 +170,13 @@ class DataFeaturizer(object):

		return samples

		def _shard_files_exist(self, feature_dir):
		"""Checks if data shard files already exist."""
		for filename in os.listdir(feature_dir):
		if "features_shard" in filename:
		return True
		return False

		def _process_raw_sample(self, input_type, row, fields):
		"""Extract information from row data."""
		data = {}
		@@ -327,8 +338,10 @@ class FeaturizedSamples(object):
		self.dataset_files = load_from_disk(self._get_dataset_paths_filename())

		if os.path.exists(self._get_compounds_filename()) and reload:
		log("Loading prexisting compounds from disk", self.verbosity)
		compounds_df = load_from_disk(self._get_compounds_filename())
		else:
		log("Saving compounds to disk", self.verbosity)
		compounds_df = self._get_compounds()
		# compounds_df is not altered by any method after initialization, so it's
		# safe to keep a copy in memory and on disk.

deepchem/hyperparameters/init.py

+14 −14

Original line number	Diff line number	Diff line
		@@ -50,28 +50,28 @@ class HyperparamOpt(object):
		all_scores = {}
		for ind, hyperparameter_tuple in enumerate(itertools.product(*hyperparam_vals)):
		model_params = {}
		log("Fitting model %d/%d" % (ind+1, number_combinations),
		self.verbosity, "high")
		for hyperparam, hyperparam_val in zip(hyperparams, hyperparameter_tuple):
		model_params[hyperparam] = hyperparam_val
		log("hyperparameters: %s" % str(model_params),
		self.verbosity, "high")

		if logdir is not None:
		model_dir = logdir
		else:
		model_dir = tempfile.mkdtemp()
		if logdir is not None:
		#TODO(JG) Fit transformers for TF models
		model = self.model_class(self.task_types, model_params, model_dir,
		verbosity=self.verbosity)
		else:
		if self.fit_transformers:
		model = self.model_class(self.task_types, model_params,
		model = self.model_class(self.task_types, model_params, model_dir,
		fit_transformers=self.fit_transformers,
		verbosity=self.verbosity)
		else:
		model = self.model_class(self.task_types, model_params,
		model = self.model_class(self.task_types, model_params, model_dir,
		verbosity=self.verbosity)

		model.fit(train_dataset)
		model.save(model_dir)
		model.save()

		evaluator = Evaluator(model, valid_dataset, output_transformers)
		df, scores_df, multitask_scores = evaluator.compute_model_performance(
		@@ -94,13 +94,13 @@ class HyperparamOpt(object):
		shutil.rmtree(model_dir)

		log("Model %d/%d, Metric %s, Validation set %s: %f" %
		(ind, number_combinations, metric.name, ind, valid_score),
		self.verbosity)
		(ind+1, number_combinations, metric.name, ind, valid_score),
		self.verbosity, "low")
		log("\tbest_validation_score so far: %f" % best_validation_score,
		self.verbosity)
		self.verbosity, "low")

		if best_model is None:
		log("No models trained correctly.", self.verbosity)
		log("No models trained correctly.", self.verbosity, "low")
		return best_model, best_hyperparams, all_scores
		train_csv_out = tempfile.NamedTemporaryFile()
		train_stats_out = tempfile.NamedTemporaryFile()
		@@ -112,7 +112,7 @@ class HyperparamOpt(object):
		else:
		train_score = multitask_scores[metric.name]
		log("Best hyperparameters: %s" % str(best_hyperparams),
		self.verbosity)
		log("train_score: %f" % train_score, self.verbosity)
		log("validation_score: %f" % best_validation_score, self.verbosity)
		self.verbosity, "low")
		log("train_score: %f" % train_score, self.verbosity, "low")
		log("validation_score: %f" % best_validation_score, self.verbosity, "low")
		return best_model, best_hyperparams, all_scores

deepchem/hyperparameters/tests/test_hyperparam_opt.py

+6 −5

Original line number	Diff line number	Diff line
		@@ -24,7 +24,7 @@ from deepchem.models.multitask import SingletaskToMultitask
		from sklearn.linear_model import LogisticRegression
		from sklearn.ensemble import RandomForestRegressor

		def rf_model_builder(task_types, params_dict, logdir=None, verbosity=None):
		def rf_model_builder(task_types, params_dict, model_dir, verbosity=None):
		"""Builds random forests given hyperparameters.

		Last two arguments only for tensorflow models and ignored.
		@@ -32,7 +32,7 @@ def rf_model_builder(task_types, params_dict, logdir=None, verbosity=None):
		n_estimators = params_dict["n_estimators"]
		max_features = params_dict["max_features"]
		return SklearnModel(
		task_types, params_dict,
		task_types, params_dict, model_dir,
		model_instance=RandomForestRegressor(n_estimators=n_estimators,
		max_features=max_features))

		@@ -87,12 +87,13 @@ class TestHyperparamOptAPI(TestAPI):
		"data_shape": [train_dataset.get_data_shape()],
		}
		classification_metric = Metric(metrics.matthews_corrcoef, np.mean)
		def model_builder(task_types, model_params, verbosity=None):
		return SklearnModel(task_types, model_params,
		def model_builder(task_types, model_params, task_model_dir, verbosity=None):
		return SklearnModel(task_types, model_params, task_model_dir,
		model_instance=LogisticRegression())
		def multitask_model_builder(task_types, params_dict, logdir=None,
		verbosity=None):
		return SingletaskToMultitask(task_types, params_dict, model_builder)
		return SingletaskToMultitask(task_types, params_dict, self.model_dir,
		model_builder)
		self._hyperparam_opt(multitask_model_builder, params_dict, train_dataset,
		valid_dataset, output_transformers, task_types,
		classification_metric)

deepchem/models/init.py

+10 −7

Original line number	Diff line number	Diff line
		@@ -27,10 +27,13 @@ class Model(object):
		"""
		Abstract base class for different ML models.
		"""
		def __init__(self, task_types, model_params, fit_transformers=None,
		def __init__(self, task_types, model_params, model_dir, fit_transformers=None,
		model_instance=None, initialize_raw_model=True,
		verbosity=None, **kwargs):
		self.model_class = model_instance.__class__
		self.model_dir = model_dir
		if not os.path.exists(self.model_dir):
		os.makedirs(self.model_dir)
		self.task_types = task_types
		self.model_params = model_params
		self.fit_transformers = fit_transformers
		@@ -66,25 +69,25 @@ class Model(object):
		return self.raw_model

		@staticmethod
		def get_model_filename(out_dir):
		def get_model_filename(model_dir):
		"""
		Given model directory, obtain filename for the model itself.
		"""
		return os.path.join(out_dir, "model.joblib")
		return os.path.join(model_dir, "model.joblib")

		@staticmethod
		def get_params_filename(out_dir):
		def get_params_filename(model_dir):
		"""
		Given model directory, obtain filename for the model itself.
		"""
		return os.path.join(out_dir, "model_params.joblib")
		return os.path.join(model_dir, "model_params.joblib")

		def save(self, out_dir):
		def save(self):
		"""Dispatcher function for saving."""
		params = {"model_params" : self.model_params,
		"task_types" : self.task_types,
		"model_class": self.__class__}
		save_to_disk(params, Model.get_params_filename(out_dir))
		save_to_disk(params, Model.get_params_filename(self.model_dir))

		def fit(self, dataset):
		"""

Admin message