Merge pull request #170 from rbharath/muv (1746304b) · Commits · 钟慕尧 / deepchem

deepchem/datasets/init.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -57,7 +57,7 @@ class Dataset(object):
		for ind, (df_file, df) in enumerate(
		zip(samples.dataset_files, samples.iterdataframes())):
		log("Writing data from file %s, number %d/%d"
		% (df_file, ind, len(samples.dataset_files)), self.verbosity)
		% (df_file, ind+1, len(samples.dataset_files)), self.verbosity)
		retval = write_dataset_single_partial((df_file, df))
		if retval is not None:
		metadata_rows.append(retval)

deepchem/metrics/init.py

+7 −1

Original line number	Diff line number	Diff line
		@@ -175,6 +175,7 @@ class Metric(object):
		"""
		y_true = np.array(np.squeeze(y_true[w != 0]))
		y_pred = np.array(np.squeeze(y_pred[w != 0]))

		if len(y_true.shape) == 0:
		n_samples = 1
		else:
		@@ -183,9 +184,13 @@ class Metric(object):
		if not y_true.size:
		return np.nan
		y_true = np.reshape(y_true, (n_samples,))

		if self.mode == "classification":
		n_classes = y_pred.shape[-1]
		if self.name == "roc_auc_score":
		# TODO(rbharath): This has been a major source of bugs. Is there a more
		# robust characterization of which metrics require class-probs and which
		# don't?
		if "roc_auc_score" in self.name:
		y_true = to_one_hot(y_true).astype(int)
		y_pred = np.reshape(y_pred, (n_samples, n_classes))
		else:
		@@ -196,6 +201,7 @@ class Metric(object):
		else:
		y_pred = np.reshape(y_pred, (n_samples,))


		if self.threshold is not None:
		y_pred = np.greater(y_pred, threshold)
		try:

deepchem/models/init.py

+7 −6

Original line number	Diff line number	Diff line
		@@ -145,10 +145,12 @@ class Model(object):

		# The iterbatches does padding with zero-weight examples on the last batch.
		# Remove padded examples.
		y_pred = y_pred[:len(dataset)]
		n_samples, n_tasks = len(dataset), len(self.tasks)
		y_pred = y_pred[:n_samples]
		y_pred = np.reshape(y_pred, (n_samples, n_tasks))
		return y_pred

		def predict_proba(self, dataset, transformers):
		def predict_proba(self, dataset, transformers, n_classes=2):
		"""
		TODO: Do transformers even make sense here?

		@@ -157,12 +159,9 @@ class Model(object):
		"""
		y_preds = []
		batch_size = self.model_params["batch_size"]
		n_classes = None
		n_tasks = len(self.tasks)
		for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):
		y_pred_batch = self.predict_proba_on_batch(X_batch)
		if n_classes is None:
		n_classes = y_pred_batch.shape[-1]
		batch_size = len(y_batch)
		y_pred_batch = np.squeeze(
		np.reshape(y_pred_batch, (batch_size, n_tasks, n_classes)))
		@@ -171,7 +170,9 @@ class Model(object):
		y_pred = np.vstack(y_preds)
		# The iterbatches does padding with zero-weight examples on the last batch.
		# Remove padded examples.
		y_pred = y_pred[:len(dataset)]
		n_samples, n_tasks = len(dataset), len(self.tasks)
		y_pred = y_pred[:n_samples]
		y_pred = np.reshape(y_pred, (n_samples, n_tasks, n_classes))
		return y_pred

		def get_task_type(self):

deepchem/models/sklearn_models/init.py

+0 −3

Original line number	Diff line number	Diff line
		@@ -58,9 +58,6 @@ class SklearnModel(Model):
		"""
		return self.raw_model.predict_proba(X)

		def predict_proba_on_batch(self, X):
		return self.raw_model.predict_proba(X)

		def predict(self, X, transformers):
		"""
		Makes predictions on dataset.

deepchem/models/test/test_overfit.py

+159 −26

Original line number	Diff line number	Diff line
		@@ -68,10 +68,7 @@ class TestOverfitAPI(TestAPI):
		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		with tempfile.NamedTemporaryFile() as csv_out:
		with tempfile.NamedTemporaryFile() as stats_out:
		scores = evaluator.compute_model_performance(
		[regression_metric], csv_out.name, stats_out)
		scores = evaluator.compute_model_performance([regression_metric])

		assert scores[regression_metric.name] > .7

		@@ -113,10 +110,50 @@ class TestOverfitAPI(TestAPI):
		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		with tempfile.NamedTemporaryFile() as csv_out:
		with tempfile.NamedTemporaryFile() as stats_out:
		scores = evaluator.compute_model_performance(
		[classification_metric], csv_out.name, stats_out)
		scores = evaluator.compute_model_performance([classification_metric])

		assert scores[classification_metric.name] > .9

		def test_sklearn_skewed_classification_overfit(self):
		"""Test sklearn models can overfit 0/1 datasets with few actives."""
		tasks = ["task0"]
		task_types = {task: "classification" for task in tasks}
		n_samples = 100
		n_features = 3
		n_tasks = len(tasks)

		# Generate dummy dataset
		np.random.seed(123)
		p = .05
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.binomial(1, p, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))

		dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)

		model_params = {
		"batch_size": None,
		"data_shape": dataset.get_data_shape()
		}

		verbosity = "high"
		classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)
		model = SklearnModel(tasks, task_types, model_params, self.model_dir,
		mode="classification",
		model_instance=RandomForestClassifier())

		# Fit trained model
		model.fit(dataset)
		model.save()

		y_pred_model = model.predict(dataset, transformers=[])
		y_pred_proba_model = model.predict_proba(dataset, transformers=[])

		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		scores = evaluator.compute_model_performance([classification_metric])

		assert scores[classification_metric.name] > .9

		@@ -167,10 +204,7 @@ class TestOverfitAPI(TestAPI):
		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		with tempfile.NamedTemporaryFile() as csv_out:
		with tempfile.NamedTemporaryFile() as stats_out:
		scores = evaluator.compute_model_performance(
		[regression_metric], csv_out.name, stats_out)
		scores = evaluator.compute_model_performance([regression_metric])

		assert scores[regression_metric.name] > .7

		@@ -223,10 +257,7 @@ class TestOverfitAPI(TestAPI):
		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		with tempfile.NamedTemporaryFile() as csv_out:
		with tempfile.NamedTemporaryFile() as stats_out:
		scores = evaluator.compute_model_performance(
		[regression_metric], csv_out.name, stats_out)
		scores = evaluator.compute_model_performance([regression_metric])

		assert scores[regression_metric.name] < .1

		@@ -274,16 +305,64 @@ class TestOverfitAPI(TestAPI):

		y_pred_model = model.predict(dataset, transformers=[])
		y_pred_proba_model = model.predict_proba(dataset, transformers=[])
		print("y_pred_proba_model.shape")
		print(y_pred_proba_model.shape)

		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		with tempfile.NamedTemporaryFile() as csv_out:
		with tempfile.NamedTemporaryFile() as stats_out:
		scores = evaluator.compute_model_performance(
		[classification_metric], csv_out.name, stats_out)
		scores = evaluator.compute_model_performance([classification_metric])

		assert scores[classification_metric.name] > .9

		def test_keras_skewed_classification_overfit(self):
		"""Test keras models can overfit 0/1 datasets with few actives."""
		tasks = ["task0"]
		task_types = {task: "classification" for task in tasks}
		n_samples = 100
		n_features = 3
		n_tasks = len(tasks)

		# Generate dummy dataset
		np.random.seed(123)
		p = .05
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.binomial(1, p, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))

		dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)

		model_params = {
		"nb_hidden": 1000,
		"activation": "relu",
		"dropout": .0,
		"learning_rate": .15,
		"momentum": .9,
		"nesterov": False,
		"decay": 1e-4,
		"batch_size": n_samples,
		"nb_epoch": 200,
		"init": "glorot_uniform",
		"nb_layers": 1,
		"batchnorm": False,
		"data_shape": dataset.get_data_shape()
		}

		verbosity = "high"
		classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)
		model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir,
		verbosity=verbosity)

		# Fit trained model
		model.fit(dataset)
		model.save()

		y_pred_model = model.predict(dataset, transformers=[])
		y_pred_proba_model = model.predict_proba(dataset, transformers=[])

		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		scores = evaluator.compute_model_performance([classification_metric])

		assert scores[classification_metric.name] > .9

		@@ -340,9 +419,63 @@ class TestOverfitAPI(TestAPI):
		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		with tempfile.NamedTemporaryFile() as csv_out:
		with tempfile.NamedTemporaryFile() as stats_out:
		scores = evaluator.compute_model_performance(
		[classification_metric], csv_out.name, stats_out)
		scores = evaluator.compute_model_performance([classification_metric])

		assert scores[classification_metric.name] > .9

		def test_tf_skewed_classification_overfit(self):
		"""Test tensorflow models can overfit 0/1 datasets with few actives."""
		tasks = ["task0"]
		task_types = {task: "classification" for task in tasks}
		n_samples = 100
		n_features = 3
		n_tasks = len(tasks)
		n_classes = 2

		# Generate dummy dataset
		np.random.seed(123)
		p = .05
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.random.binomial(1, p, size=(n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))

		dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)

		model_params = {
		"layer_sizes": [1500],
		"dropouts": [.0],
		"learning_rate": 0.001,
		"momentum": .9,
		"batch_size": n_samples,
		"num_classification_tasks": 1,
		"num_classes": n_classes,
		"num_features": n_features,
		"weight_init_stddevs": [.3],
		"bias_init_consts": [1.],
		"nb_epoch": 200,
		"penalty": 0.0,
		"optimizer": "adam",
		"data_shape": dataset.get_data_shape()
		}

		verbosity = "high"
		classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)
		model = TensorflowModel(
		tasks, task_types, model_params, self.model_dir,
		tf_class=TensorflowMultiTaskClassifier,
		verbosity=verbosity)

		# Fit trained model
		model.fit(dataset)
		model.save()

		y_pred_model = model.predict(dataset, transformers=[])
		y_pred_proba_model = model.predict_proba(dataset, transformers=[])

		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		scores = evaluator.compute_model_performance([classification_metric])

		assert scores[classification_metric.name] > .8

Admin message