Merge branch 'master' of https://github.com/deepchem/deepchem into dynamic-transform (3ba48112) · Commits · 钟慕尧 / deepchem

datasets/mini_muv.csv.gz

0 → 100644

+19.3 KiB

File added.

No diff preview for this file type.

deepchem/datasets/init.py

+36 −10

Original line number	Diff line number	Diff line
		@@ -66,7 +66,7 @@ class Dataset(object):
		metadata_rows,
		columns=('df_file', 'task_names', 'ids',
		'X', 'X-transformed', 'y', 'y-transformed',
		'w',
		'w', 'w-transformed',
		'X_sums', 'X_sum_squares', 'X_n',
		'y_sums', 'y_sum_squares', 'y_n'))
		self.save_to_disk()
		@@ -87,12 +87,14 @@ class Dataset(object):
		out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)
		out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)
		out_w = os.path.join(data_dir, "%s-w.joblib" % basename)
		out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)
		out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

		metadata_rows = []
		retval = ([df_file, tasks, out_ids, out_X,
		out_X_transformed, out_y,
		out_y_transformed, out_w,
		retval = ([df_file, tasks, out_ids,
		out_X, out_X_transformed,
		out_y, out_y_transformed,
		out_w, out_w_transformed,
		out_X_sums, out_X_sum_squares, out_X_n,
		out_y_sums, out_y_sum_squares, out_y_n])
		metadata_rows.append(retval)
		@@ -101,7 +103,7 @@ class Dataset(object):
		metadata_rows,
		columns=('df_file','task_names', 'ids',
		'X', 'X-transformed', 'y', 'y-transformed',
		'w',
		'w', 'w-transformed',
		'X_sums', 'X_sum_squares', 'X_n',
		'y_sums', 'y_sum_squares', 'y_n'))
		self.save_to_disk()
		@@ -148,14 +150,18 @@ class Dataset(object):
		"""
		return self.metadata_df.shape[0]

		def _itershards(self):
		def itershards(self):
		"""
		Iterates over all shards in dataset.

		Datasets are stored in sharded fashion on disk. Each call to next() for the
		generator defined by this function returns the data from a particular shard.
		The order of shards returned is guaranteed to remain fixed.
		"""
		for _, row in self.metadata_df.iterrows():
		X = load_from_disk(row['X-transformed'])
		y = load_from_disk(row['y-transformed'])
		w = load_from_disk(row['w'])
		w = load_from_disk(row['w-transformed'])
		ids = load_from_disk(row['ids'])
		yield (X, y, w, ids)

		@@ -165,7 +171,7 @@ class Dataset(object):
		"""
		if batch_size == None:
		batch_size = len(self)
		for i, (X, y, w, ids) in enumerate(self._itershards()):
		for i, (X, y, w, ids) in enumerate(self.itershards()):
		log("Iterating on shard-%s/epoch-%s" % (str(i+1), str(epoch+1)),
		self.verbosity)
		nb_sample = np.shape(X)[0]
		@@ -200,6 +206,24 @@ class Dataset(object):
		return (np.vstack(Xs), np.vstack(ys), np.vstack(ws),
		np.squeeze(np.vstack(ids)))

		def get_labels(self):
		"""
		Returns all labels for this dataset.
		"""
		ys = []
		for (_, y_b, _, _) in self.itershards():
		ys.append(y_b)
		return np.vstack(ys)

		def get_weights(self):
		"""
		Returns all weights for this dataset.
		"""
		ws = []
		for (_, _, w_b, _) in self.itershards():
		ws.append(w_b)
		return np.vstack(ws)

		def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size):
		"""Fix batch to have exactly batch_size elements.

		@@ -307,10 +331,12 @@ def write_dataset_single(val, data_dir, feature_types, tasks):
		out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)
		out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)
		out_w = os.path.join(data_dir, "%s-w.joblib" % basename)
		out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)
		out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

		save_to_disk(X, out_X)
		save_to_disk(y, out_y)
		save_to_disk(w, out_w)
		# Write moments to disk
		save_to_disk(X_sums, out_X_sums)
		save_to_disk(X_sum_squares, out_X_sum_squares)
		@@ -321,12 +347,12 @@ def write_dataset_single(val, data_dir, feature_types, tasks):
		# Write X, y as transformed versions
		save_to_disk(X, out_X_transformed)
		save_to_disk(y, out_y_transformed)
		save_to_disk(w, out_w)
		save_to_disk(w, out_w_transformed)
		save_to_disk(ids, out_ids)
		# TODO(rbharath): Should X be saved to out_X_transformed as well? Since
		# itershards expects to loop over X-transformed? (Ditto for y/w)
		return([df_file, task_names, out_ids, out_X, out_X_transformed, out_y,
		out_y_transformed, out_w,
		out_y_transformed, out_w, out_w_transformed,
		out_X_sums, out_X_sum_squares, out_X_n,
		out_y_sums, out_y_sum_squares, out_y_n])

deepchem/datasets/tests/init.py

+93 −0

Original line number	Diff line number	Diff line
		"""
		General API for testing dataset objects
		"""
		from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		__author__ = "Bharath Ramsundar"
		__copyright__ = "Copyright 2016, Stanford University"
		__license__ = "LGPL"

		import unittest
		import tempfile
		import os
		import shutil
		import numpy as np
		from deepchem.datasets import Dataset
		from deepchem.featurizers.featurize import DataFeaturizer
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.transformers import NormalizationTransformer
		from deepchem.splits.tests import TestSplitAPI

		class TestDatasetAPI(TestSplitAPI):
		"""
		Shared API for testing with dataset objects.
		"""
		# TODO(rbharath): There should be a more natural way to create a dataset
		# object, perhaps just starting from (Xs, ys, ws)
		def _create_dataset(self, compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, tasks,
		protein_pdb_field=None, ligand_pdb_field=None,
		user_specified_features=None,
		split_field=None,
		shard_size=100):
		featurizers = compound_featurizers + complex_featurizers
		samples = self._gen_samples(
		compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, tasks,
		protein_pdb_field=protein_pdb_field,
		ligand_pdb_field=ligand_pdb_field,
		user_specified_features=user_specified_features,
		split_field=split_field,
		shard_size=shard_size)
		use_user_specified_features = (user_specified_features is not None)
		dataset = Dataset(data_dir=self.data_dir, samples=samples,
		featurizers=featurizers, tasks=tasks,
		use_user_specified_features=use_user_specified_features)
		return dataset

		def _load_solubility_data(self):
		"""Loads solubility data from example.csv"""
		compound_featurizers = [CircularFingerprint(size=1024)]
		complex_featurizers = []
		input_transformer_classes = []
		output_transformer_classes = []
		task_types = {"log-solubility": "regression"}
		input_file = "example.csv"
		return self._create_dataset(
		compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, task_types.keys())

		def _load_classification_data(self):
		"""Loads classification data from example.csv"""
		compound_featurizers = [CircularFingerprint(size=1024)]
		complex_featurizers = []
		input_transformer_classes = []
		output_transformer_classes = []
		task_types = {"outcome": "classification"}
		input_file = "example_classification.csv"
		return self._create_dataset(
		compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, task_types.keys())

		def _load_multitask_data(self):
		"""Load example multitask data."""
		compound_featurizers = [CircularFingerprint(size=1024)]
		complex_featurizers = []
		output_transformer_classes = []
		input_transformer_classes = []
		tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",
		"task7", "task8", "task9", "task10", "task11", "task12",
		"task13", "task14", "task15", "task16"]
		task_types = {task: "classification" for task in tasks}
		input_file = "multitask_example.csv"
		return self._create_dataset(
		compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, task_types.keys())

deepchem/datasets/tests/test_datasets.py

+13 −59

Original line number	Diff line number	Diff line
		@@ -18,78 +18,32 @@ from deepchem.datasets import Dataset
		from deepchem.featurizers.featurize import DataFeaturizer
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.transformers import NormalizationTransformer
		from deepchem.datasets.tests import TestDatasetAPI

		class TestAPI(unittest.TestCase):
		class TestBasicDatasetAPI(TestDatasetAPI):
		"""
		Test top-level API for ML models."
		Test basic top-level API for dataset objects.
		"""
		def setUp(self):
		self.current_dir = os.path.dirname(os.path.abspath(__file__))
		self.test_data_dir = os.path.join(self.current_dir, "../../models/test")
		self.smiles_field = "smiles"
		self.feature_dir = tempfile.mkdtemp()
		self.samples_dir = tempfile.mkdtemp()
		self.data_dir = tempfile.mkdtemp()

		def tearDown(self):
		shutil.rmtree(self.feature_dir)
		shutil.rmtree(self.samples_dir)
		shutil.rmtree(self.data_dir)

		# TODO(rbharath): There should be a more natural way to create a dataset
		# object, perhaps just starting from (Xs, ys, ws)
		def _create_dataset(self, compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, tasks,
		protein_pdb_field=None, ligand_pdb_field=None,
		user_specified_features=None,
		split_field=None,
		shard_size=100):
		# Featurize input
		featurizers = compound_featurizers + complex_featurizers

		input_file = os.path.join(self.test_data_dir, input_file)
		featurizer = DataFeaturizer(tasks=tasks,
		smiles_field=self.smiles_field,
		protein_pdb_field=protein_pdb_field,
		ligand_pdb_field=ligand_pdb_field,
		compound_featurizers=compound_featurizers,
		complex_featurizers=complex_featurizers,
		user_specified_features=user_specified_features,
		split_field=split_field,
		verbosity="low")

		samples = featurizer.featurize(input_file, self.feature_dir, self.samples_dir,
		shard_size=shard_size)
		use_user_specified_features = (user_specified_features is not None)
		dataset = Dataset(data_dir=self.data_dir, samples=samples,
		featurizers=featurizers, tasks=tasks,
		use_user_specified_features=use_user_specified_features)
		return dataset

		def _load_solubility_data(self):
		"""Loads solubility data from example.csv"""
		compound_featurizers = [CircularFingerprint(size=1024)]
		complex_featurizers = []
		input_transformer_classes = []
		output_transformer_classes = [NormalizationTransformer]
		task_types = {"log-solubility": "regression"}
		input_file = "example.csv"
		return self._create_dataset(
		compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, task_types.keys())

		def test_get_task_names(self):
		"""Test that get_task_names returns correct task_names"""
		solubility_dataset = self._load_solubility_data()
		assert solubility_dataset.get_task_names() == ["log-solubility"]

		multitask_dataset = self._load_multitask_data()
		assert sorted(multitask_dataset.get_task_names()) == sorted(["task0",
		"task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8",
		"task9", "task10", "task11", "task12", "task13", "task14", "task15",
		"task16"])


		def test_get_data_shape(self):
		"""Test that get_data_shape returns currect data shape"""
		solubility_dataset = self._load_solubility_data()
		assert solubility_dataset.get_data_shape() == (1024,)

		multitask_dataset = self._load_multitask_data()
		assert multitask_dataset.get_data_shape() == (1024,)

		def test_len(self):
		"""Test that len(dataset) works."""
		solubility_dataset = self._load_solubility_data()

deepchem/featurizers/featurize.py

+0 −136

Original line number	Diff line number	Diff line
		@@ -17,19 +17,11 @@ from deepchem.utils.save import log
		from deepchem.utils.save import save_to_disk
		from deepchem.utils.save import load_from_disk
		from deepchem.utils.save import load_pandas_from_disk
		from deepchem.utils import ScaffoldGenerator
		from deepchem.featurizers.nnscore import NNScoreComplexFeaturizer
		import multiprocessing as mp
		from functools import partial
		import dill

		def generate_scaffold(smiles, include_chirality=False):
		"""Compute the Bemis-Murcko scaffold for a SMILES string."""
		mol = Chem.MolFromSmiles(smiles)
		engine = ScaffoldGenerator(include_chirality=include_chirality)
		scaffold = engine.get_scaffold(mol)
		return scaffold

		def _check_validity(compounds_df):
		"""Ensure that columns of compound_df contain required elements."""
		if not set(FeaturizedSamples.colnames).issubset(compounds_df.keys()):
		@@ -330,8 +322,6 @@ class FeaturizedSamples(object):
		if dataset_files is not None:
		if not reload:
		save_to_disk(dataset_files, self._get_dataset_paths_filename())
		else:
		raise ValueError("Can't change dataset_files already stored on disk")
		else:
		save_to_disk(dataset_files, self._get_dataset_paths_filename())
		self.dataset_files = load_from_disk(self._get_dataset_paths_filename())
		@@ -417,129 +407,3 @@ class FeaturizedSamples(object):
		if row["mol_id"] in compound_ids:
		visible_inds.append(ind)
		yield df.loc[visible_inds]

		def train_valid_test_split(self, splittype, train_dir=None,
		valid_dir=None, test_dir=None, frac_train=.8,
		frac_valid=.1, frac_test=.1, seed=None,
		log_every_n=1000, reload=False):
		"""
		Splits self into train/validation/test sets.

		Returns FeaturizedDataset objects.
		"""
		if not reload:
		if splittype == "random":
		train_inds, valid_inds, test_inds = self._random_split(
		seed=seed, frac_train=frac_train, frac_test=frac_test,
		frac_valid=frac_valid)
		elif splittype == "scaffold":
		train_inds, valid_inds, test_inds = self._scaffold_split(
		frac_train=frac_train, frac_test=frac_test,
		frac_valid=frac_valid, log_every_n=log_every_n)
		elif splittype == "specified":
		train_inds, valid_inds, test_inds = self._specified_split()
		else:
		raise ValueError("improper splittype.")
		train_samples, valid_samples, test_samples = None, None, None
		dataset_files = self.dataset_files
		if train_dir is not None:
		train_samples = FeaturizedSamples(samples_dir=train_dir,
		dataset_files=dataset_files,
		featurizers=self.featurizers,
		verbosity=self.verbosity,
		reload=False)
		if not reload:
		train_samples._set_compound_df(self.compounds_df.iloc[train_inds])
		if test_dir is not None:
		test_samples = FeaturizedSamples(samples_dir=test_dir,
		dataset_files=dataset_files,
		featurizers=self.featurizers,
		verbosity=self.verbosity,
		reload=False)
		if not reload:
		test_samples._set_compound_df(self.compounds_df.iloc[test_inds])
		if valid_dir is not None:
		valid_samples = FeaturizedSamples(samples_dir=valid_dir,
		dataset_files=dataset_files,
		featurizers=self.featurizers,
		verbosity=self.verbosity,
		reload=False)
		if not reload:
		valid_samples._set_compound_df(self.compounds_df.iloc[valid_inds])

		return train_samples, valid_samples, test_samples

		def train_test_split(self, splittype, train_dir, test_dir, seed=None,
		frac_train=.8, reload=False):
		"""
		Splits self into train/test sets.

		Returns FeaturizedDataset objects.
		"""
		train_samples, _, test_samples = self.train_valid_test_split(
		splittype, train_dir, valid_dir=None, test_dir=test_dir,
		frac_train=frac_train, frac_test=1-frac_train, frac_valid=0.,
		reload=False)
		return train_samples, test_samples

		def _random_split(self, seed=None, frac_train=.8, frac_valid=.1,
		frac_test=.1):
		"""
		Splits internal compounds randomly into train/validation/test.
		"""
		np.testing.assert_almost_equal(frac_train + frac_valid + frac_test, 1.)
		np.random.seed(seed)
		train_cutoff = frac_train * len(self.compounds_df)
		valid_cutoff = (frac_train+frac_valid) * len(self.compounds_df)
		shuffled = np.random.permutation(range(len(self.compounds_df)))
		return (shuffled[:train_cutoff], shuffled[train_cutoff:valid_cutoff],
		shuffled[valid_cutoff:])

		def _scaffold_split(self, frac_train=.8, frac_valid=.1, frac_test=.1, log_every_n=1000):
		"""
		Splits internal compounds into train/validation/test by scaffold.
		"""
		np.testing.assert_almost_equal(frac_train + frac_valid + frac_test, 1.)
		scaffolds = {}
		log("About to generate scaffolds", self.verbosity)
		for ind, row in self.compounds_df.iterrows():
		if self.verbosity is not None and ind % log_every_n == 0:
		log("Generating scaffold %d/%d" % (ind, len(self.compounds_df)),
		self.verbosity)
		scaffold = generate_scaffold(row["smiles"])
		if scaffold not in scaffolds:
		scaffolds[scaffold] = [ind]
		else:
		scaffolds[scaffold].append(ind)
		# Sort from largest to smallest scaffold sets
		scaffold_sets = [scaffold_set for (scaffold, scaffold_set) in
		sorted(scaffolds.items(), key=lambda x: -len(x[1]))]
		train_cutoff = frac_train * len(self.compounds_df)
		valid_cutoff = (frac_train+frac_valid) * len(self.compounds_df)
		train_inds, valid_inds, test_inds = [], [], []
		log("About to sort in scaffold sets", self.verbosity)
		for scaffold_set in scaffold_sets:
		if len(train_inds) + len(scaffold_set) > train_cutoff:
		if len(train_inds) + len(valid_inds) + len(scaffold_set) > valid_cutoff:
		test_inds += scaffold_set
		else:
		valid_inds += scaffold_set
		else:
		train_inds += scaffold_set
		return train_inds, valid_inds, test_inds

		def _specified_split(self):
		"""
		Splits internal compounds into train/validation/test by user-specification.
		"""
		train_inds, valid_inds, test_inds = [], [], []
		for ind, row in self.compounds_df.iterrows():
		if row["split"].lower() == "train":
		train_inds.append(ind)
		elif row["split"].lower() in ["valid", "validation"]:
		valid_inds.append(ind)
		elif row["split"].lower() == "test":
		test_inds.append(ind)
		else:
		raise ValueError("Missing required split information.")
		return train_inds, valid_inds, test_inds

Admin message