Merge pull request #154 from rbharath/reload_test (53c02d0b) · Commits · 钟慕尧 / deepchem

datasets/mini_muv.csv.gz

0 → 100644

+19.3 KiB

File added.

No diff preview for this file type.

deepchem/featurizers/featurize.py

+0 −2

Original line number	Diff line number	Diff line
		@@ -322,8 +322,6 @@ class FeaturizedSamples(object):
		if dataset_files is not None:
		if not reload:
		save_to_disk(dataset_files, self._get_dataset_paths_filename())
		else:
		raise ValueError("Can't change dataset_files already stored on disk")
		else:
		save_to_disk(dataset_files, self._get_dataset_paths_filename())
		self.dataset_files = load_from_disk(self._get_dataset_paths_filename())

deepchem/models/test/init.py

+2 −0

Original line number	Diff line number	Diff line
		@@ -42,6 +42,7 @@ class TestAPI(unittest.TestCase):
		self.feature_dir = tempfile.mkdtemp()
		self.samples_dir = tempfile.mkdtemp()
		self.train_dir = tempfile.mkdtemp()
		self.valid_dir = tempfile.mkdtemp()
		self.test_dir = tempfile.mkdtemp()
		self.model_dir = tempfile.mkdtemp()
		if not os.path.exists(self.model_dir):
		@@ -51,6 +52,7 @@ class TestAPI(unittest.TestCase):
		shutil.rmtree(self.feature_dir)
		shutil.rmtree(self.samples_dir)
		shutil.rmtree(self.train_dir)
		shutil.rmtree(self.valid_dir)
		shutil.rmtree(self.test_dir)
		# TODO(rbharath): Removing this causes crashes for some reason. Need to
		# debug.

deepchem/models/test/test_reload.py

0 → 100644

+116 −0

Original line number	Diff line number	Diff line
		"""
		Testing reload.
		"""
		from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		__author__ = "Bharath Ramsundar"
		__copyright__ = "Copyright 2016, Stanford University"
		__license__ = "LGPL"

		import os
		import numpy as np
		from sklearn.linear_model import LogisticRegression
		from deepchem.models.test import TestAPI
		from deepchem.utils.save import load_from_disk
		from deepchem.datasets import Dataset
		from deepchem.featurizers.featurize import DataFeaturizer
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.splits import ScaffoldSplitter
		from deepchem.datasets import Dataset
		from deepchem.transformers import BalancingTransformer
		from deepchem.hyperparameters import HyperparamOpt
		from deepchem.models.multitask import SingletaskToMultitask
		from deepchem import metrics
		from deepchem.metrics import Metric
		from deepchem.models.sklearn_models import SklearnModel

		class TestReload(TestAPI):
		"""
		Test reload for datasets.
		"""

		def _run_muv_experiment(self, reload=False, verbosity=None):
		"""Loads or reloads a small version of MUV dataset."""
		# Load MUV dataset
		dataset_file = os.path.join(
		self.current_dir, "../../../datasets/mini_muv.csv.gz")
		dataset = load_from_disk(dataset_file)
		print("Number of examples in dataset: %s" % str(dataset.shape[0]))

		featurizers = [CircularFingerprint(size=1024)]
		MUV_tasks = ['MUV-692', 'MUV-689', 'MUV-846', 'MUV-859', 'MUV-644',
		'MUV-548', 'MUV-852', 'MUV-600', 'MUV-810', 'MUV-712',
		'MUV-737', 'MUV-858', 'MUV-713', 'MUV-733', 'MUV-652',
		'MUV-466', 'MUV-832']
		featurizer = DataFeaturizer(tasks=MUV_tasks,
		smiles_field="smiles",
		compound_featurizers=featurizers,
		verbosity="low")
		featurized_samples = featurizer.featurize(
		dataset_file, self.feature_dir,
		self.samples_dir, shard_size=4096,
		reload=reload)

		splitter = ScaffoldSplitter()
		train_samples, valid_samples, test_samples = \
		splitter.train_valid_test_split(
		featurized_samples, self.train_dir, self.valid_dir, self.test_dir,
		log_every_n=1000, reload=reload)
		len_train_samples, len_valid_samples, len_test_samples = \
		len(train_samples), len(valid_samples), len(test_samples)

		print("Creating train dataset.")
		train_dataset = Dataset(data_dir=self.train_dir, samples=train_samples,
		featurizers=featurizers, tasks=MUV_tasks,
		verbosity=verbosity, reload=reload)
		print("Creating valid dataset.")
		valid_dataset = Dataset(data_dir=self.valid_dir, samples=valid_samples,
		featurizers=featurizers, tasks=MUV_tasks,
		verbosity=verbosity, reload=reload)
		print("Creating test dataset")
		test_dataset = Dataset(data_dir=self.test_dir, samples=test_samples,
		featurizers=featurizers, tasks=MUV_tasks,
		verbosity=verbosity, reload=reload)
		len_train_dataset, len_valid_dataset, len_test_dataset = \
		len(train_dataset), len(valid_dataset), len(test_dataset)

		input_transformers = []
		output_transformers = []
		weight_transformers = [BalancingTransformer(transform_w=True,
		dataset=train_dataset)]
		transformers = input_transformers + output_transformers + weight_transformers
		print("Transforming train dataset")
		for transformer in transformers:
		transformer.transform(train_dataset)
		print("Transforming valid dataset")
		for transformer in transformers:
		transformer.transform(valid_dataset)
		print("Transforming test dataset")
		for transformer in transformers:
		transformer.transform(test_dataset)

		return (len_train_samples, len_valid_samples, len_test_samples,
		len_train_dataset, len_valid_dataset, len_test_dataset)

		def test_reload(self):
		"""Check num samples for loaded and reload datasets is equal."""
		reload = False
		verbosity = None
		print("Running experiment for first time w/o reload.")
		(len_train_samples, len_valid_samples, len_test_samples,
		len_train_dataset, len_valid_dataset, len_test_dataset) = \
		self._run_muv_experiment(reload, verbosity)

		print("Running experiment for second time with reload.")
		reload = True
		(len_reload_train_samples, len_reload_valid_samples, len_reload_test_samples,
		len_reload_train_dataset, len_reload_valid_dataset, len_reload_test_dataset) = \
		self._run_muv_experiment(reload, verbosity)
		assert len_train_samples == len_reload_train_samples
		assert len_valid_samples == len_reload_valid_samples
		assert len_test_samples == len_reload_valid_samples
		assert len_train_dataset == len_reload_train_dataset
		assert len_valid_dataset == len_reload_valid_dataset
		assert len_test_dataset == len_reload_valid_dataset

deepchem/splits/init.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -53,7 +53,7 @@ class Splitter(object):
		dataset_files=dataset_files,
		featurizers=samples.featurizers,
		verbosity=self.verbosity,
		reload=False)
		reload=reload)
		if not reload:
		train_samples._set_compound_df(samples.compounds_df.iloc[train_inds])
		if test_dir is not None:
		@@ -61,7 +61,7 @@ class Splitter(object):
		dataset_files=dataset_files,
		featurizers=samples.featurizers,
		verbosity=self.verbosity,
		reload=False)
		reload=reload)
		if not reload:
		test_samples._set_compound_df(samples.compounds_df.iloc[test_inds])
		if valid_dir is not None:
		@@ -69,7 +69,7 @@ class Splitter(object):
		dataset_files=dataset_files,
		featurizers=samples.featurizers,
		verbosity=self.verbosity,
		reload=False)
		reload=reload)
		if not reload:
		valid_samples._set_compound_df(samples.compounds_df.iloc[valid_inds])

Admin message