Converted more molnet loaders to new API (9f9dcd29) · Commits · 钟慕尧 / deepchem

deepchem/molnet/load_function/clearance_datasets.py

+56 −110

Original line number	Diff line number	Diff line
		@@ -2,115 +2,61 @@
		clearance dataset loader.
		"""
		import os
		import logging
		import deepchem
		import deepchem as dc
		from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader
		from deepchem.data import Dataset
		from typing import List, Optional, Tuple, Union

		logger = logging.getLogger(__name__)

		DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir()
		CLEARANCE_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/clearance.csv"
		CLEARANCE_TASKS = ['target']


		def load_clearance(featurizer='ECFP',
		split='random',
		reload=True,
		move_mean=True,
		data_dir=None,
		save_dir=None,
		**kwargs):
		"""Load clearance datasets."""
		# Featurize clearance dataset
		logger.info("About to featurize clearance dataset.")
		logger.info("About to load clearance dataset.")
		if data_dir is None:
		data_dir = DEFAULT_DIR
		if save_dir is None:
		save_dir = DEFAULT_DIR

		clearance_tasks = ['target']

		if reload:
		save_folder = os.path.join(save_dir, "clearance-featurized")
		if not move_mean:
		save_folder = os.path.join(save_folder, str(featurizer) + "_mean_unmoved")
		else:
		save_folder = os.path.join(save_folder, str(featurizer))

		if featurizer == "smiles2img":
		img_spec = kwargs.get("img_spec", "std")
		save_folder = os.path.join(save_folder, img_spec)
		save_folder = os.path.join(save_folder, str(split))

		loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk(
		save_folder)
		if loaded:
		return clearance_tasks, all_dataset, transformers
		class _ClearanceLoader(_MolnetLoader):

		dataset_file = os.path.join(data_dir, "clearance.csv")
		def create_dataset(self) -> Dataset:
		dataset_file = os.path.join(self.data_dir, "clearance.csv")
		if not os.path.exists(dataset_file):
		deepchem.utils.data_utils.download_url(url=CLEARANCE_URL, dest_dir=data_dir)

		if featurizer == 'ECFP':
		featurizer = deepchem.feat.CircularFingerprint(size=1024)
		elif featurizer == 'GraphConv':
		featurizer = deepchem.feat.ConvMolFeaturizer()
		elif featurizer == 'Weave':
		featurizer = deepchem.feat.WeaveFeaturizer()
		elif featurizer == 'Raw':
		featurizer = deepchem.feat.RawFeaturizer()
		elif featurizer == "smiles2img":
		img_spec = kwargs.get("img_spec", "std")
		img_size = kwargs.get("img_size", 80)
		featurizer = deepchem.feat.SmilesToImage(
		img_size=img_size, img_spec=img_spec)

		loader = deepchem.data.CSVLoader(
		tasks=clearance_tasks, smiles_field="smiles", featurizer=featurizer)
		dataset = loader.featurize(dataset_file, shard_size=8192)

		if split is None:
		# Initialize transformers
		transformers = [
		deepchem.trans.NormalizationTransformer(
		transform_y=True, dataset=dataset, move_mean=move_mean)
		]

		logger.info("Split is None, about to transform data")
		for transformer in transformers:
		dataset = transformer.transform(dataset)

		return clearance_tasks, (dataset, None, None), transformers

		splitters = {
		'index': deepchem.splits.IndexSplitter(),
		'random': deepchem.splits.RandomSplitter(),
		'scaffold': deepchem.splits.ScaffoldSplitter(),
		'stratified': deepchem.splits.SingletaskStratifiedSplitter()
		}
		splitter = splitters[split]
		logger.info("About to split data with {} splitter.".format(split))
		frac_train = kwargs.get("frac_train", 0.8)
		frac_valid = kwargs.get('frac_valid', 0.1)
		frac_test = kwargs.get('frac_test', 0.1)

		train, valid, test = splitter.train_valid_test_split(
		dataset,
		frac_train=frac_train,
		frac_valid=frac_valid,
		frac_test=frac_test)

		transformers = [
		deepchem.trans.NormalizationTransformer(
		transform_y=True, dataset=train, move_mean=move_mean)
		]

		logger.info("About to transform data")
		for transformer in transformers:
		train = transformer.transform(train)
		valid = transformer.transform(valid)
		test = transformer.transform(test)

		if reload:
		deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid,
		test, transformers)
		return clearance_tasks, (train, valid, test), transformers
		dc.utils.data_utils.download_url(
		url=CLEARANCE_URL, dest_dir=self.data_dir)
		loader = dc.data.CSVLoader(
		tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer)
		return loader.create_dataset(dataset_file, shard_size=8192)


		def load_clearance(
		featurizer: Union[dc.feat.Featurizer, str] = 'ECFP',
		splitter: Union[dc.splits.Splitter, str, None] = 'scaffold',
		transformers: List[Union[TransformerGenerator, str]] = ['normalization'],
		reload: bool = True,
		data_dir: Optional[str] = None,
		save_dir: Optional[str] = None,
		**kwargs
		) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]:
		"""
		Load clearance datasets.

		Parameters
		----------
		featurizer: Featurizer or str
		the featurizer to use for processing the data. Alternatively you can pass
		one of the names from dc.molnet.featurizers as a shortcut.
		splitter: Splitter or str
		the splitter to use for splitting the data into training, validation, and
		test sets. Alternatively you can pass one of the names from
		dc.molnet.splitters as a shortcut. If this is None, all the data
		will be included in a single dataset.
		transformers: list of TransformerGenerators or strings
		the Transformers to apply to the data. Each one is specified by a
		TransformerGenerator or, as a shortcut, one of the names from
		dc.molnet.transformers.
		reload: bool
		if True, the first call for a particular featurizer and splitter will cache
		the datasets to disk, and subsequent calls will reload the cached datasets.
		data_dir: str
		a directory to save the raw data in
		save_dir: str
		a directory to save the dataset in
		"""
		loader = _ClearanceLoader(featurizer, splitter, transformers, CLEARANCE_TASKS,
		data_dir, save_dir, **kwargs)
		return loader.load_dataset('clearance', reload)

deepchem/molnet/load_function/hiv_datasets.py

+51 −97

Original line number	Diff line number	Diff line
		@@ -2,21 +2,35 @@
		hiv dataset loader.
		"""
		import os
		import logging
		import deepchem

		logger = logging.getLogger(__name__)
		import deepchem as dc
		from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader
		from deepchem.data import Dataset
		from typing import List, Optional, Tuple, Union

		HIV_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/HIV.csv"
		DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir()
		HIV_TASKS = ["HIV_active"]


		class _HIVLoader(_MolnetLoader):

		def load_hiv(featurizer='ECFP',
		split='index',
		reload=True,
		data_dir=None,
		save_dir=None,
		**kwargs):
		def create_dataset(self) -> Dataset:
		dataset_file = os.path.join(self.data_dir, "HIV.csv")
		if not os.path.exists(dataset_file):
		dc.utils.data_utils.download_url(url=HIV_URL, dest_dir=self.data_dir)
		loader = dc.data.CSVLoader(
		tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer)
		return loader.create_dataset(dataset_file, shard_size=8192)


		def load_hiv(
		featurizer: Union[dc.feat.Featurizer, str] = 'ECFP',
		splitter: Union[dc.splits.Splitter, str, None] = 'scaffold',
		transformers: List[Union[TransformerGenerator, str]] = ['balancing'],
		reload: bool = True,
		data_dir: Optional[str] = None,
		save_dir: Optional[str] = None,
		**kwargs
		) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]:
		"""Load HIV dataset

		The HIV dataset was introduced by the Drug Therapeutics
		@@ -36,93 +50,33 @@ def load_hiv(featurizer='ECFP',
		- "activity": Three-class labels for screening results: CI/CM/CA
		- "HIV_active": Binary labels for screening results: 1 (CA/CM) and 0 (CI)

		Parameters
		----------
		featurizer: Featurizer or str
		the featurizer to use for processing the data. Alternatively you can pass
		one of the names from dc.molnet.featurizers as a shortcut.
		splitter: Splitter or str
		the splitter to use for splitting the data into training, validation, and
		test sets. Alternatively you can pass one of the names from
		dc.molnet.splitters as a shortcut. If this is None, all the data
		will be included in a single dataset.
		transformers: list of TransformerGenerators or strings
		the Transformers to apply to the data. Each one is specified by a
		TransformerGenerator or, as a shortcut, one of the names from
		dc.molnet.transformers.
		reload: bool
		if True, the first call for a particular featurizer and splitter will cache
		the datasets to disk, and subsequent calls will reload the cached datasets.
		data_dir: str
		a directory to save the raw data in
		save_dir: str
		a directory to save the dataset in

		References
		----------
		.. [1] AIDS Antiviral Screen Data.
		https://wiki.nci.nih.gov/display/NCIDTPdata/AIDS+Antiviral+Screen+Data
		"""
		# Featurize hiv dataset
		logger.info("About to featurize hiv dataset.")
		if data_dir is None:
		data_dir = DEFAULT_DIR
		if save_dir is None:
		save_dir = DEFAULT_DIR

		hiv_tasks = ["HIV_active"]

		if reload:
		save_folder = os.path.join(save_dir, "hiv-featurized", str(featurizer))
		if featurizer == "smiles2img":
		img_spec = kwargs.get("img_spec", "std")
		save_folder = os.path.join(save_folder, img_spec)
		save_folder = os.path.join(save_folder, str(split))

		if reload:
		loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk(
		save_folder)
		if loaded:
		return hiv_tasks, all_dataset, transformers

		dataset_file = os.path.join(data_dir, "HIV.csv")
		if not os.path.exists(dataset_file):
		deepchem.utils.data_utils.download_url(url=HIV_URL, dest_dir=data_dir)

		if featurizer == 'ECFP':
		featurizer = deepchem.feat.CircularFingerprint(size=1024)
		elif featurizer == 'GraphConv':
		featurizer = deepchem.feat.ConvMolFeaturizer()
		elif featurizer == 'Weave':
		featurizer = deepchem.feat.WeaveFeaturizer()
		elif featurizer == 'Raw':
		featurizer = deepchem.feat.RawFeaturizer()
		elif featurizer == "smiles2img":
		img_spec = kwargs.get("img_spec", "std")
		img_size = kwargs.get("img_size", 80)
		featurizer = deepchem.feat.SmilesToImage(
		img_size=img_size, img_spec=img_spec)

		loader = deepchem.data.CSVLoader(
		tasks=hiv_tasks, smiles_field="smiles", featurizer=featurizer)
		dataset = loader.featurize(dataset_file, shard_size=8192)

		if split is None:
		transformers = [deepchem.trans.BalancingTransformer(dataset=dataset)]

		logger.info("Split is None, about to transform data")
		for transformer in transformers:
		dataset = transformer.transform(dataset)

		return hiv_tasks, (dataset, None, None), transformers

		splitters = {
		'index': deepchem.splits.IndexSplitter(),
		'random': deepchem.splits.RandomSplitter(),
		'scaffold': deepchem.splits.ScaffoldSplitter(),
		'butina': deepchem.splits.ButinaSplitter(),
		'stratified': deepchem.splits.RandomStratifiedSplitter()
		}
		splitter = splitters[split]
		logger.info("About to split dataset with {} splitter.".format(split))
		frac_train = kwargs.get("frac_train", 0.8)
		frac_valid = kwargs.get('frac_valid', 0.1)
		frac_test = kwargs.get('frac_test', 0.1)

		train, valid, test = splitter.train_valid_test_split(
		dataset,
		frac_train=frac_train,
		frac_valid=frac_valid,
		frac_test=frac_test)
		train, valid, test = splitter.train_valid_test_split(dataset)

		transformers = [deepchem.trans.BalancingTransformer(dataset=train)]

		logger.info("About to transform data.")
		for transformer in transformers:
		train = transformer.transform(train)
		valid = transformer.transform(valid)
		test = transformer.transform(test)

		if reload:
		deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid,
		test, transformers)
		return hiv_tasks, (train, valid, test), transformers
		loader = _HIVLoader(featurizer, splitter, transformers, HIV_TASKS, data_dir,
		save_dir, **kwargs)
		return loader.load_dataset('hiv', reload)

deepchem/molnet/load_function/hopv_datasets.py

+57 −105

Original line number	Diff line number	Diff line
		@@ -2,21 +2,40 @@
		HOPV dataset loader.
		"""
		import os
		import logging
		import deepchem

		logger = logging.getLogger(__name__)
		import deepchem as dc
		from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader
		from deepchem.data import Dataset
		from typing import List, Optional, Tuple, Union

		HOPV_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/hopv.tar.gz"
		DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir()
		HOPV_TASKS = [
		'HOMO', 'LUMO', 'electrochemical_gap', 'optical_gap', 'PCE', 'V_OC', 'J_SC',
		'fill_factor'
		]


		class _HOPVLoader(_MolnetLoader):

		def load_hopv(featurizer='ECFP',
		split='index',
		reload=True,
		data_dir=None,
		save_dir=None,
		**kwargs):
		def create_dataset(self) -> Dataset:
		dataset_file = os.path.join(self.data_dir, "hopv.csv")
		if not os.path.exists(dataset_file):
		dc.utils.data_utils.download_url(url=HOPV_URL, dest_dir=self.data_dir)
		dc.utils.data_utils.untargz_file(
		os.path.join(self.data_dir, 'hopv.tar.gz'), self.data_dir)
		loader = dc.data.CSVLoader(
		tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer)
		return loader.create_dataset(dataset_file, shard_size=8192)


		def load_hopv(
		featurizer: Union[dc.feat.Featurizer, str] = 'ECFP',
		splitter: Union[dc.splits.Splitter, str, None] = 'scaffold',
		transformers: List[Union[TransformerGenerator, str]] = ['normalization'],
		reload: bool = True,
		data_dir: Optional[str] = None,
		save_dir: Optional[str] = None,
		**kwargs
		) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]:
		"""Load HOPV datasets. Does not do train/test split

		The HOPV datasets consist of the "Harvard Organic
		@@ -30,96 +49,29 @@ def load_hopv(featurizer='ECFP',
		removed (for now).

		Lopez, Steven A., et al. "The Harvard organic photovoltaic dataset." Scientific data 3.1 (2016): 1-7.
		"""
		# Featurize HOPV dataset
		logger.info("About to featurize HOPV dataset.")
		if data_dir is None:
		data_dir = DEFAULT_DIR
		if save_dir is None:
		save_dir = DEFAULT_DIR

		hopv_tasks = [
		'HOMO', 'LUMO', 'electrochemical_gap', 'optical_gap', 'PCE', 'V_OC',
		'J_SC', 'fill_factor'
		]

		if reload:
		save_folder = os.path.join(save_dir, "hopv-featurized", str(featurizer))
		if featurizer == "smiles2img":
		img_spec = kwargs.get("img_spec", "std")
		save_folder = os.path.join(save_folder, img_spec)
		save_folder = os.path.join(save_folder, str(split))

		loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk(
		save_folder)
		if loaded:
		return hopv_tasks, all_dataset, transformers

		dataset_file = os.path.join(data_dir, "hopv.csv")
		if not os.path.exists(dataset_file):
		deepchem.utils.data_utils.download_url(url=HOPV_URL, dest_dir=data_dir)
		deepchem.utils.data_utils.untargz_file(
		os.path.join(data_dir, 'hopv.tar.gz'), data_dir)

		if featurizer == 'ECFP':
		featurizer = deepchem.feat.CircularFingerprint(size=1024)
		elif featurizer == 'GraphConv':
		featurizer = deepchem.feat.ConvMolFeaturizer()
		elif featurizer == 'Weave':
		featurizer = deepchem.feat.WeaveFeaturizer()
		elif featurizer == 'Raw':
		featurizer = deepchem.feat.RawFeaturizer()
		elif featurizer == "smiles2img":
		img_spec = kwargs.get("img_spec", "std")
		img_size = kwargs.get("img_size", 80)
		featurizer = deepchem.feat.SmilesToImage(
		img_size=img_size, img_spec=img_spec)

		loader = deepchem.data.CSVLoader(
		tasks=hopv_tasks, smiles_field="smiles", featurizer=featurizer)
		dataset = loader.featurize(dataset_file, shard_size=8192)

		if split == None:
		transformers = [
		deepchem.trans.NormalizationTransformer(
		transform_y=True, dataset=dataset)
		]

		logger.info("Split is None, about to transform data")
		for transformer in transformers:
		dataset = transformer.transform(dataset)

		return hopv_tasks, (dataset, None, None), transformers

		splitters = {
		'index': deepchem.splits.IndexSplitter(),
		'random': deepchem.splits.RandomSplitter(),
		'scaffold': deepchem.splits.ScaffoldSplitter(),
		'butina': deepchem.splits.ButinaSplitter()
		}
		splitter = splitters[split]
		logger.info("About to split dataset with {} splitter.".format(split))
		frac_train = kwargs.get("frac_train", 0.8)
		frac_valid = kwargs.get('frac_valid', 0.1)
		frac_test = kwargs.get('frac_test', 0.1)

		train, valid, test = splitter.train_valid_test_split(
		dataset,
		frac_train=frac_train,
		frac_valid=frac_valid,
		frac_test=frac_test)

		transformers = [
		deepchem.trans.NormalizationTransformer(transform_y=True, dataset=train)
		]

		logger.info("About to transform data.")
		for transformer in transformers:
		train = transformer.transform(train)
		valid = transformer.transform(valid)
		test = transformer.transform(test)

		if reload:
		deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid,
		test, transformers)
		return hopv_tasks, (train, valid, test), transformers
		Parameters
		----------
		featurizer: Featurizer or str
		the featurizer to use for processing the data. Alternatively you can pass
		one of the names from dc.molnet.featurizers as a shortcut.
		splitter: Splitter or str
		the splitter to use for splitting the data into training, validation, and
		test sets. Alternatively you can pass one of the names from
		dc.molnet.splitters as a shortcut. If this is None, all the data
		will be included in a single dataset.
		transformers: list of TransformerGenerators or strings
		the Transformers to apply to the data. Each one is specified by a
		TransformerGenerator or, as a shortcut, one of the names from
		dc.molnet.transformers.
		reload: bool
		if True, the first call for a particular featurizer and splitter will cache
		the datasets to disk, and subsequent calls will reload the cached datasets.
		data_dir: str
		a directory to save the raw data in
		save_dir: str
		a directory to save the dataset in
		"""
		loader = _HOPVLoader(featurizer, splitter, transformers, HOPV_TASKS, data_dir,
		save_dir, **kwargs)
		return loader.load_dataset('hopv', reload)

deepchem/molnet/load_function/hppb_datasets.py

+54 −105

File changed.

Preview size limit exceeded, changes collapsed.

deepchem/molnet/load_function/lipo_datasets.py

+56 −113

File changed.

Preview size limit exceeded, changes collapsed.

Admin message