Merge pull request #176 from rbharath/no_pad (9f3b9c03) · Commits · 钟慕尧 / deepchem

deepchem/datasets/init.py

+0 −23

Original line number	Diff line number	Diff line
		@@ -196,8 +196,6 @@ class Dataset(object):
		y_batch = y[indices]
		w_batch = w[indices]
		ids_batch = ids[indices]
		(X_batch, y_batch, w_batch, ids_batch) = self._pad_batch(
		X_batch, y_batch, w_batch, ids_batch, shard_batch_size)
		yield (X_batch, y_batch, w_batch, ids_batch)

		@staticmethod
		@@ -249,27 +247,6 @@ class Dataset(object):
		ws.append(np.array(w_b))
		return np.vstack(ws)

		def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size):
		"""Fix batch to have exactly batch_size elements.

		Due to rounding issues, some batches will not have exactly batch_size
		elements. Handle these batches by zero padding all arrays.
		"""
		n, feature_shape = np.shape(X_b)[0], np.shape(X_b)[1:]
		_, num_tasks = np.shape(y_b)
		if n == batch_size:
		return (X_b, y_b, w_b, ids_b)
		else:
		X_batch = np.zeros((batch_size,) + feature_shape)
		y_batch = np.zeros((batch_size, num_tasks))
		w_batch = np.zeros((batch_size, num_tasks))
		ids_batch = np.zeros((batch_size,), dtype=object)
		X_batch[:n] = X_b
		y_batch[:n] = y_b
		w_batch[:n] = w_b
		ids_batch[:n] = ids_b
		return X_batch, y_batch, w_batch, ids_batch

		def __len__(self):
		"""
		Finds number of elements in dataset.

+3 −6

Original line number	Diff line number	Diff line
		@@ -98,9 +98,6 @@ class TensorflowGraph(object):
		self.placeholder_root = 'placeholders'
		with tf.name_scope(self.placeholder_root) as scope:
		self.placeholder_scope = scope
		self.valid = tf.placeholder(tf.bool,
		shape=[model_params["batch_size"]],
		name='valid')

		self.setup()
		if train:
		@@ -333,7 +330,7 @@ class TensorflowGraph(object):
		for task in xrange(self.num_tasks):
		with tf.name_scope(self.placeholder_scope):
		weights.append(tf.identity(
		tf.placeholder(tf.float32, shape=[self.model_params["batch_size"]],
		tf.placeholder(tf.float32, shape=[None],
		name='weights_%d' % task)))
		self.weights = weights

		@@ -473,7 +470,7 @@ class TensorflowClassifier(TensorflowGraph):
		for task in xrange(self.num_tasks):
		with tf.name_scope(self.placeholder_scope):
		labels.append(tf.identity(
		tf.placeholder(tf.float32, shape=[batch_size, num_classes],
		tf.placeholder(tf.float32, shape=[None, num_classes],
		name='labels_%d' % task)))
		self.labels = labels

		@@ -527,7 +524,7 @@ class TensorflowRegressor(TensorflowGraph):
		for task in xrange(self.num_tasks):
		with tf.name_scope(self.placeholder_scope):
		labels.append(tf.identity(
		tf.placeholder(tf.float32, shape=[batch_size],
		tf.placeholder(tf.float32, shape=[None],
		name='labels_%d' % task)))
		self.labels = labels

+2 −4

Original line number	Diff line number	Diff line
		@@ -103,8 +103,7 @@ class TensorflowMultiTaskClassifier(TensorflowClassifier):
		with tf.name_scope(self.placeholder_scope):
		self.mol_features = tf.placeholder(
		tf.float32,
		shape=[self.model_params["batch_size"],
		num_features],
		shape=[None, num_features],
		name='mol_features')

		layer_sizes = self.model_params["layer_sizes"]
		@@ -231,8 +230,7 @@ class TensorflowMultiTaskRegressor(TensorflowRegressor):
		with tf.name_scope(self.placeholder_scope):
		self.mol_features = tf.placeholder(
		tf.float32,
		shape=[self.model_params["batch_size"],
		num_features],
		shape=[None, num_features],
		name='mol_features')

		layer_sizes = self.model_params["layer_sizes"]

+2 −1

Original line number	Diff line number	Diff line
		@@ -546,7 +546,8 @@ class TestOverfitAPI(TestAPI):
		#"batch_size": n_samples/8,
		#"batch_size": n_samples/16,
		#"batch_size": n_samples/32,
		"batch_size": n_samples/64,
		#"batch_size": n_samples/64,
		"batch_size": 75,
		# TODO(rbharath): Is there a bug in the padding code? Why does it fail to
		# learn for non-multiples?
		#"batch_size": 600,

+2 −11

Original line number	Diff line number	Diff line
		@@ -50,7 +50,7 @@ valid_dir = os.path.join(base_dir, "valid_dataset")
		test_dir = os.path.join(base_dir, "test_dataset")
		model_dir = os.path.join(base_dir, "model")

		# Remove existing model directory since TF doesn't overwrite by default...
		# Remove existing model directory since keras doesn't overwrite by default...
		if os.path.exists(model_dir):
		shutil.rmtree(model_dir)
		os.makedirs(model_dir)
		@@ -93,15 +93,6 @@ train_samples, valid_samples, test_samples = \
		featurized_samples, train_dir, valid_dir, test_dir,
		log_every_n=1000, reload=reload)

		len_train_samples, len_valid_samples, len_test_samples = \
		len(train_samples), len(valid_samples), len(test_samples)
		assert relative_difference(
		len(train_samples), frac_train * len(featurized_samples)) < 1e-3
		assert relative_difference(
		len(valid_samples), frac_valid * len(featurized_samples)) < 1e-3
		assert relative_difference(
		len(test_samples), frac_test * len(featurized_samples)) < 1e-3

		# Generate datasets
		print("About to create datasets")
		print("MUV_tasks")
		@@ -153,7 +144,7 @@ for transformer in transformers:
		for transformer in transformers:
		transformer.transform(test_dataset)

		# Fit tensorflow models
		# Fit keras models
		MUV_task_types = {task: "classification" for task in MUV_tasks}
		classification_metric = Metric(metrics.roc_auc_score, np.mean,
		verbosity=verbosity,