Pylinted all the code. (578bea4c) · Commits · 钟慕尧 / deepchem

deepchem/models/init.py

+5 −7

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@ class Model(object):
		self.model_type = model_type
		self.task_types = task_types
		self.model_params = model_params
		self.raw_model = None

		def fit_on_batch(self, X, y, w):
		"""
		@@ -49,7 +50,7 @@ class Model(object):
		"""
		Return raw model.
		"""
		return(self.raw_model)
		return self.raw_model

		@staticmethod
		def get_model_filename(out_dir):
		@@ -130,7 +131,7 @@ class Model(object):
		print("Training on batch-%s/epoch-%s" % (str(i+1), str(epoch+1)))
		nb_sample = np.shape(X)[0]
		interval_points = np.linspace(
		0, nb_sample, np.ceil(float(nb_sample)/batch_size)+1).astype(int)
		0, nb_sample, np.ceil(float(nb_sample)/batch_size)+1, dtype=int)
		for j in range(len(interval_points)-1):
		indices = range(interval_points[j], interval_points[j+1])
		X_batch = X[indices, :]
		@@ -138,9 +139,6 @@ class Model(object):
		w_batch = w[indices]
		self.fit_on_batch(X_batch, y_batch, w_batch)

		# TODO(rbharath): What does this function do when y is not provided. Suspect
		# it breaks. Need to fix.

		# TODO(rbharath): The structure of the produced df might be
		# complicated. Better way to model?
		def predict(self, dataset):
		@@ -158,9 +156,9 @@ class Model(object):
		for (X, y, w, ids) in dataset.itershards():
		nb_sample = np.shape(X)[0]
		interval_points = np.linspace(
		0, nb_sample, np.ceil(float(nb_sample)/batch_size)+1).astype(int)
		0, nb_sample, np.ceil(float(nb_sample)/batch_size)+1, dtype=int)
		y_preds = []
		for j in range(0,len(interval_points)-1):
		for j in range(len(interval_points)-1):
		indices = range(interval_points[j], interval_points[j+1])
		y_preds.append(self.predict_on_batch(X[indices, :]))
		y_pred = np.concatenate(y_preds)

deepchem/models/deep.py

+5 −2

Original line number	Diff line number	Diff line
		@@ -66,7 +66,8 @@ class MultiTaskDNN(KerasModel):
		Dense(model_params["nb_hidden"], init='uniform',
		activation=model_params["activation"]),
		name="dense", input="input")
		model.add_node(Dropout(model_params["dropout"]), name="dropout",
		model.add_node(Dropout(model_params["dropout"]),
		name="dropout",
		input="dense")
		top_layer = "dropout"
		for ind, task in enumerate(sorted_tasks):
		@@ -96,6 +97,7 @@ class MultiTaskDNN(KerasModel):
		self.raw_model = model

		def get_data_dict(self, X, y=None):
		"""Wrap data X in dict for graph computations (Keras graph only for now)."""
		data = {}
		data["input"] = X
		for ind, task in enumerate(sorted(self.task_types.keys())):
		@@ -110,7 +112,7 @@ class MultiTaskDNN(KerasModel):
		def get_sample_weight(self, w):
		"""Get dictionaries needed to fit models"""
		sample_weight = {}
		for ind, task in enumerate(sorted(self.task_types.keys())):
		for ind in range(len(sorted(self.task_types.keys()))):
		sample_weight["task%d" % ind] = w[:, ind]
		return sample_weight

		@@ -124,6 +126,7 @@ class MultiTaskDNN(KerasModel):
		data = self.get_data_dict(X, y)
		sample_weight = self.get_sample_weight(w)
		loss = self.raw_model.train_on_batch(data, sample_weight=sample_weight)
		return loss

		def predict_on_batch(self, X):
		"""

deepchem/models/deep3d.py

+8 −1

Original line number	Diff line number	Diff line
		@@ -15,16 +15,23 @@ from deepchem.models import Model
		from deepchem.models.deep import KerasModel

		def shuffle_shape(shape):
		"""
		Shuffle shape of form (N, N, N, C) into (C, N, N, N).
		"""
		(axis_length, _, _, n_channels) = shape
		shuffled_shape = (n_channels, axis_length, axis_length, axis_length)
		return shuffled_shape

		def shuffle_data(X):
		"""
		Make data of shape (C, N, N, N) from (N, N, N, C)

		C is n_channels, N is axis_length.
		"""
		(n_samples, axis_length, _, _, n_channels) = np.shape(X)
		X = np.reshape(X, (n_samples, n_channels, axis_length, axis_length, axis_length))
		return X


		class DockingDNN(KerasModel):
		"""
		Wrapper class for fitting 3D convolutional networks for deep docking.

deepchem/scripts/modeler.py

+14 −17

Original line number	Diff line number	Diff line
		@@ -7,6 +7,7 @@ from __future__ import unicode_literals
		import argparse
		import glob
		import os
		import multiprocessing as mp
		from functools import partial
		from deepchem.utils.featurize import DataFeaturizer
		from deepchem.utils.featurize import FeaturizedSamples
		@@ -59,10 +60,6 @@ def add_transforms_group(cmd):
		choices=["normalize", "log"],
		help="Supported transforms are 'log' and 'normalize'. 'None' will be taken\n"
		"to mean no transforms are required.")
		transform_group.add_argument(
		"--mode", default="singletask",
		choices=["singletask", "multitask"],
		help="Type of model being built.")
		transform_group.add_argument(
		"--feature-types", nargs="+", required=1,
		choices=["user-specified-features", "ECFP", "RDKIT-descriptors"],
		@@ -175,9 +172,10 @@ def extract_model_params(args):
		"activation", "momentum", "nesterov"]

		model_params = {param : getattr(args, param) for param in params}
		return(model_params)
		return model_params

		def ensure_exists(dirs):
		"""Creates dirs if they don't exist."""
		for directory in dirs:
		if not os.path.exists(directory):
		os.makedirs(directory)
		@@ -201,7 +199,6 @@ def create_model(args):
		args.data_dir)
		ensure_exists([feature_dir, data_dir, model_dir])


		if args.featurize:
		print("+++++++++++++++++++++++++++++++++")
		print("Perform featurization")
		@@ -214,7 +211,7 @@ def create_model(args):
		print("+++++++++++++++++++++++++++++++++")
		print("Generate dataset for featurized samples")
		samples_dir = os.path.join(data_dir, "samples")
		samples = FeaturizedSamples(samples_dir, reload=True)
		samples = FeaturizedSamples(samples_dir, reload_data=True)

		print("Generating dataset.")
		full_data_dir = os.path.join(data_dir, "full-data")
		@@ -227,10 +224,9 @@ def create_model(args):
		if args.train_test_split:
		print("+++++++++++++++++++++++++++++++++")
		print("Perform train-test split")
		paths = [feature_dir]
		train_test_split(
		paths, args.input_transforms, args.output_transforms, args.feature_types,
		args.splittype, args.mode, data_dir)
		args.input_transforms, args.output_transforms, args.feature_types,
		args.splittype, data_dir)

		if args.fit:
		print("+++++++++++++++++++++++++++++++++")
		@@ -281,6 +277,7 @@ def featurize_inputs(feature_dir, data_dir, input_files,
		user_specified_features, tasks, smiles_field,
		split_field, id_field, threshold, parallel):

		"""Allows for parallel data featurization."""
		featurize_input_partial = partial(featurize_input,
		feature_dir=feature_dir,
		user_specified_features=user_specified_features,
		@@ -302,7 +299,7 @@ def featurize_inputs(feature_dir, data_dir, input_files,

		print("Writing samples to disk.")
		samples_dir = os.path.join(data_dir, "samples")
		samples = FeaturizedSamples(samples_dir, dataset_files)
		FeaturizedSamples(samples_dir, dataset_files)

		def featurize_input(input_file, feature_dir, user_specified_features, tasks,
		smiles_field, split_field, id_field, threshold):
		@@ -318,18 +315,18 @@ def featurize_input(input_file, feature_dir, user_specified_features, tasks,
		feature_dir, "%s.joblib" %(os.path.splitext(os.path.basename(input_file))[0]))
		featurizer.featurize(input_file, FeaturizedSamples.feature_types, out)

		def train_test_split(paths, input_transforms, output_transforms,
		feature_types, splittype, mode, data_dir):
		def train_test_split(input_transforms, output_transforms,
		feature_types, splittype, data_dir):
		"""Saves transformed model."""

		samples_dir = os.path.join(data_dir, "samples")
		samples = FeaturizedSamples(samples_dir, reload=True)
		samples = FeaturizedSamples(samples_dir, reload_data=True)

		print("Split data into train/test")
		train_samples_dir = os.path.join(data_dir, "train-samples")
		test_samples_dir = os.path.join(data_dir, "test-samples")
		train_samples, test_samples = samples.train_test_split(splittype,
		train_samples_dir, test_samples_dir)
		train_samples, test_samples = samples.train_test_split(
		splittype, train_samples_dir, test_samples_dir)

		train_data_dir = os.path.join(data_dir, "train-data")
		test_data_dir = os.path.join(data_dir, "test-data")
		@@ -366,7 +363,7 @@ def eval_trained_model(model_type, model_dir, data_dir,
		data = Dataset(data_dir)

		evaluator = Evaluator(model, data, verbose=True)
		pred_y_df, perf_df = evaluator.compute_model_performance(csv_out, stats_out)
		_, perf_df = evaluator.compute_model_performance(csv_out, stats_out)
		print("Model Performance.")
		print(perf_df)

deepchem/utils/dataset.py

+27 −13

Original line number	Diff line number	Diff line
		@@ -7,8 +7,8 @@ from __future__ import unicode_literals
		import os
		import numpy as np
		import pandas as pd
		import multiprocessing as mp
		from functools import partial
		from rdkit import Chem
		from deepchem.utils.save import save_to_disk
		from deepchem.utils.save import load_from_disk
		from deepchem.utils.featurize import FeaturizedSamples
		@@ -110,8 +110,7 @@ class Dataset(object):
		"""
		Iterates over all shards in dataset.
		"""
		nb_shards = self.get_number_shards()
		for i, row in self.metadata_df.iterrows():
		for _, row in self.metadata_df.iterrows():
		X = load_from_disk(row['X-transformed'])
		y = load_from_disk(row['y-transformed'])
		w = load_from_disk(row['w'])
		@@ -119,8 +118,13 @@ class Dataset(object):
		yield (X, y, w, ids)

		def transform(self, input_transforms, output_transforms, parallel=False):
		(normalize_X, truncate_x, normalize_y,
		truncate_y, log_X, log_y) = False, False, False, False, False, False
		"""
		Transforms all internally stored data.

		Adds X-transform, y-transform columns to metadata.
		"""
		(normalize_X, truncate_x, normalize_y, truncate_y, log_X, log_y) = (
		False, False, False, False, False, False)

		if "truncate" in input_transforms:
		truncate_x = True
		@@ -141,10 +145,12 @@ class Dataset(object):
		log_X, log_y,
		parallel=parallel)
		nrow = self.metadata_df.shape[0]
		self.metadata_df['X_means'] = [X_means for i in range(nrow)]
		self.metadata_df['X_stds'] = [X_stds for i in range(nrow)]
		self.metadata_df['y_means'] = [y_means for i in range(nrow)]
		self.metadata_df['y_stds'] = [y_stds for i in range(nrow)]
		# TODO(rbharath): These lines are puzzling. Better way to avoid storage
		# duplication here?
		self.metadata_df['X_means'] = [X_means for _ in range(nrow)]
		self.metadata_df['X_stds'] = [X_stds for _ in range(nrow)]
		self.metadata_df['y_means'] = [y_means for _ in range(nrow)]
		self.metadata_df['y_stds'] = [y_stds for _ in range(nrow)]
		save_to_disk(
		self.metadata_df, self._get_metadata_filename())
		self.transforms = (input_transforms, output_transforms)
		@@ -152,26 +158,30 @@ class Dataset(object):
		self.transforms, self._get_transforms_filename())

		def get_label_means(self):
		"""Return pandas series of label means."""
		return self.metadata_df["y_means"]

		def get_label_stds(self):
		"""Return pandas series of label stds."""
		return self.metadata_df["y_stds"]

		def get_input_transforms(self):
		"""Returns stored input transforms."""
		(input_transforms, _) = self.transforms
		return input_transforms

		def get_output_transforms(self):
		"""Returns stored output transforms."""
		(_, output_transforms) = self.transforms
		return output_transforms

		def _transform(self, normalize_X=True, normalize_y=True,
		truncate_X=True, truncate_y=True,
		log_X=False, log_y=False, parallel=False):
		"""Helper to (parallel) transform all indexed data."""
		df = self.metadata_df
		trunc = 5.0
		X_means, X_stds, y_means, y_stds = compute_mean_and_std(df)
		total = df.shape[0]
		indices = range(0, df.shape[0])
		transform_row_partial = partial(_transform_row, df=df, normalize_X=normalize_X,
		normalize_y=normalize_y, truncate_X=truncate_X,
		@@ -190,7 +200,11 @@ class Dataset(object):

		def _transform_row(i, df, normalize_X, normalize_y, truncate_X, truncate_y,
		log_X, log_y, X_means, X_stds, y_means, y_stds, trunc):
		total = df.shape[0]
		"""
		Transforms the data (X, y, w,...) in a single row.

		Writes X-transforme,d y-transformed to disk.
		"""
		row = df.iloc[i]
		X = load_from_disk(row['X'])
		if normalize_X or log_X:
		@@ -205,7 +219,6 @@ def _transform_row(i, df, normalize_X, normalize_y, truncate_X, truncate_y,
		save_to_disk(X, row['X-transformed'])

		y = load_from_disk(row['y'])
		w = load_from_disk(row['w'])
		if normalize_y or log_y:
		if normalize_y:
		y = np.nan_to_num((y - y_means) / y_stds)
		@@ -249,6 +262,7 @@ def compute_sums_and_nb_sample(tensor, W=None):
		# make it easy to use multiprocessing.

		def write_dataset_single(val, data_dir, feature_types):
		"""Writes files for single row (X, y, w, X-transformed, ...) to disk."""
		(df_file, df) = val
		# TODO(rbharath): This is a hack. clean up.
		if not len(df):
		@@ -290,7 +304,7 @@ def _df_to_numpy(df, feature_types):
		y = np.reshape(y, (n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		tensors = []
		for i, datapoint in df.iterrows():
		for _, datapoint in df.iterrows():
		feature_list = []
		for feature_type in feature_types:
		feature_list.append(datapoint[feature_type])

Admin message