Loading deepchem/molnet/load_function/nci_datasets.py +64 −114 Original line number Diff line number Diff line Loading @@ -4,119 +4,69 @@ Original Author - Bharath Ramsundar Author - Aneesh Pappu """ import os import logging import deepchem import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union logger = logging.getLogger(__name__) DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() NCI_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/nci_unique.csv" def load_nci(featurizer='ECFP', shard_size=1000, split='random', reload=True, data_dir=None, save_dir=None, **kwargs): # Load nci dataset logger.info("About to load NCI dataset.") if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR all_nci_tasks = [ 'CCRF-CEM', 'HL-60(TB)', 'K-562', 'MOLT-4', 'RPMI-8226', 'SR', 'A549/ATCC', 'EKVX', 'HOP-62', 'HOP-92', 'NCI-H226', 'NCI-H23', 'NCI-H322M', 'NCI-H460', 'NCI-H522', 'COLO 205', 'HCC-2998', 'HCT-116', 'HCT-15', 'HT29', 'KM12', 'SW-620', 'SF-268', 'SF-295', 'SF-539', 'SNB-19', 'SNB-75', 'U251', 'LOX IMVI', 'MALME-3M', 'M14', 'MDA-MB-435', 'SK-MEL-2', 'SK-MEL-28', 'SK-MEL-5', 'UACC-257', 'UACC-62', 'IGR-OV1', 'OVCAR-3', 'OVCAR-4', 'OVCAR-5', 'OVCAR-8', 'NCI/ADR-RES', 'SK-OV-3', '786-0', 'A498', 'ACHN', 'CAKI-1', 'RXF 393', 'SN12C', 'TK-10', 'UO-31', 'PC-3', 'DU-145', 'MCF7', 'MDA-MB-231/ATCC', 'MDA-MB-468', 'HS 578T', 'BT-549', 'T-47D' NCI_TASKS = [ 'CCRF-CEM', 'HL-60(TB)', 'K-562', 'MOLT-4', 'RPMI-8226', 'SR', 'A549/ATCC', 'EKVX', 'HOP-62', 'HOP-92', 'NCI-H226', 'NCI-H23', 'NCI-H322M', 'NCI-H460', 'NCI-H522', 'COLO 205', 'HCC-2998', 'HCT-116', 'HCT-15', 'HT29', 'KM12', 'SW-620', 'SF-268', 'SF-295', 'SF-539', 'SNB-19', 'SNB-75', 'U251', 'LOX IMVI', 'MALME-3M', 'M14', 'MDA-MB-435', 'SK-MEL-2', 'SK-MEL-28', 'SK-MEL-5', 'UACC-257', 'UACC-62', 'IGR-OV1', 'OVCAR-3', 'OVCAR-4', 'OVCAR-5', 'OVCAR-8', 'NCI/ADR-RES', 'SK-OV-3', '786-0', 'A498', 'ACHN', 'CAKI-1', 'RXF 393', 'SN12C', 'TK-10', 'UO-31', 'PC-3', 'DU-145', 'MCF7', 'MDA-MB-231/ATCC', 'MDA-MB-468', 'HS 578T', 'BT-549', 'T-47D' ] if reload: save_folder = os.path.join(save_dir, "nci-featurized", featurizer) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return all_nci_tasks, all_dataset, transformers class _NCILoader(_MolnetLoader): dataset_file = os.path.join(data_dir, "nci_unique.csv") def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "nci_unique.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=NCI_URL, dest_dir=data_dir) # Featurize nci dataset logger.info("About to featurize nci dataset.") if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") img_size = kwargs.get("img_size", 80) featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=all_nci_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, shard_size=shard_size) if split == None: logger.info("Split is None, about to transform data") transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=dataset) ] for transformer in transformers: dataset = transformer.transform(dataset) return all_nci_tasks, (dataset, None, None), transformers splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'scaffold': deepchem.splits.ScaffoldSplitter() } splitter = splitters[split] logger.info("About to split data with {} splitter.".format(splitter)) frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train, valid, test = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer(transform_y=True, dataset=train) ] logger.info("About to transform dataset.") for transformer in transformers: train = transformer.transform(train) valid = transformer.transform(valid) test = transformer.transform(test) if reload: deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid, test, transformers) return all_nci_tasks, (train, valid, test), transformers dc.utils.data_utils.download_url(url=NCI_URL, dest_dir=self.data_dir) loader = dc.data.CSVLoader( tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_nci( featurizer: Union[dc.feat.Featurizer, str] = 'ECFP', splitter: Union[dc.splits.Splitter, str, None] = 'random', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load NCI dataset. Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in """ loader = _NCILoader(featurizer, splitter, transformers, NCI_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('nci', reload) deepchem/molnet/load_function/ppb_datasets.py +54 −103 Original line number Diff line number Diff line Loading @@ -2,108 +2,59 @@ PPB dataset loader. """ import os import logging import deepchem import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union logger = logging.getLogger(__name__) DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() PPB_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/PPB.csv" PPB_TASKS = ['exp'] def load_ppb(featurizer='ECFP', split='index', reload=True, data_dir=None, save_dir=None, **kwargs): """Load PPB datasets.""" # Featurize PPB dataset logger.info("About to featurize PPB dataset.") logger.info("About to load PPB dataset.") PPB_tasks = ['exp'] if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR if reload: save_folder = os.path.join(save_dir, "ppb-featurized", str(featurizer)) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) class _PPBLoader(_MolnetLoader): loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return PPB_tasks, all_dataset, transformers dataset_file = os.path.join(data_dir, "PPB.csv") def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "PPB.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=PPB_URL, dest_dir=data_dir) if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") img_size = kwargs.get("img_size", 80) featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=PPB_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, shard_size=8192) if split == None: transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=dataset) ] logger.info("Split is None, about to transform data") for transformer in transformers: dataset = transformer.transform(dataset) return PPB_tasks, (dataset, None, None), transformers splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'scaffold': deepchem.splits.ScaffoldSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter() } splitter = splitters[split] logger.info("About to split dataset with {} splitter.".format(split)) frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train, valid, test = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer(transform_y=True, dataset=train) ] logger.info("About to transform dataset.") for transformer in transformers: train = transformer.transform(train) valid = transformer.transform(valid) test = transformer.transform(test) if reload: deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid, test, transformers) return PPB_tasks, (train, valid, test), transformers dc.utils.data_utils.download_url(url=PPB_URL, dest_dir=self.data_dir) loader = dc.data.CSVLoader( tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_ppb( featurizer: Union[dc.feat.Featurizer, str] = 'ECFP', splitter: Union[dc.splits.Splitter, str, None] = 'scaffold', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load PPB datasets. Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in """ loader = _PPBLoader(featurizer, splitter, transformers, PPB_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('ppb', reload) deepchem/molnet/load_function/sampl_datasets.py +53 −114 Original line number Diff line number Diff line Loading @@ -2,22 +2,35 @@ SAMPL dataset loader. """ import os import logging import deepchem logger = logging.getLogger(__name__) import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union SAMPL_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/SAMPL.csv" DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() SAMPL_TASKS = ['expt'] class _SAMPLLoader(_MolnetLoader): def load_sampl(featurizer='ECFP', split='index', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "SAMPL.csv") if not os.path.exists(dataset_file): dc.utils.data_utils.download_url(url=SAMPL_URL, dest_dir=self.data_dir) loader = dc.data.CSVLoader( tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_sampl( featurizer: Union[dc.feat.Featurizer, str] = 'ECFP', splitter: Union[dc.splits.Splitter, str, None] = 'scaffold', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load SAMPL(FreeSolv) dataset The Free Solvation Database, FreeSolv(SAMPL), provides experimental and Loading @@ -36,6 +49,27 @@ def load_sampl(featurizer='ECFP', used as label - "calc" - Calculated solvation energy (unit: kcal/mol) of the compound Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in References ---------- Loading @@ -43,101 +77,6 @@ def load_sampl(featurizer='ECFP', experimental and calculated hydration free energies, with input files." Journal of computer-aided molecular design 28.7 (2014): 711-720. """ # Featurize SAMPL dataset logger.info("About to featurize SAMPL dataset.") logger.info("About to load SAMPL dataset.") if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR if reload: save_folder = os.path.join(save_dir, "sampl-featurized") if not move_mean: save_folder = os.path.join(save_folder, str(featurizer) + "_mean_unmoved") else: save_folder = os.path.join(save_folder, str(featurizer)) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) dataset_file = os.path.join(data_dir, "SAMPL.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=SAMPL_URL, dest_dir=data_dir) SAMPL_tasks = ['expt'] if reload: loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return SAMPL_tasks, all_dataset, transformers if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == 'smiles2img': img_size = kwargs.get("img_size", 80) img_spec = kwargs.get("img_spec", "std") featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=SAMPL_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, shard_size=8192) if split == None: transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=dataset, move_mean=move_mean) ] logger.info("Split is None, about to transform data") for transformer in transformers: dataset = transformer.transform(dataset) return SAMPL_tasks, (dataset, None, None), transformers splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'scaffold': deepchem.splits.ScaffoldSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] logger.info("About to split dataset with {} splitter.".format(split)) frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train, valid, test = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) train, valid, test = splitter.train_valid_test_split(dataset) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train, move_mean=move_mean) ] logger.info("About to transform dataset.") for transformer in transformers: train = transformer.transform(train) valid = transformer.transform(valid) test = transformer.transform(test) if reload: deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid, test, transformers) return SAMPL_tasks, (train, valid, test), transformers loader = _SAMPLLoader(featurizer, splitter, transformers, SAMPL_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('sampl', reload) deepchem/molnet/load_function/sider_datasets.py +71 −103 File changed.Preview size limit exceeded, changes collapsed. Show changes examples/sider/sider_rf.py +1 −2 Original line number Diff line number Diff line Loading @@ -10,10 +10,9 @@ import os import shutil import numpy as np import deepchem as dc from sider_datasets import load_sider from sklearn.ensemble import RandomForestClassifier sider_tasks, datasets, transformers = load_sider() sider_tasks, datasets, transformers = dc.molnet.load_sider() train_dataset, valid_dataset, test_dataset = datasets metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean, Loading Loading
deepchem/molnet/load_function/nci_datasets.py +64 −114 Original line number Diff line number Diff line Loading @@ -4,119 +4,69 @@ Original Author - Bharath Ramsundar Author - Aneesh Pappu """ import os import logging import deepchem import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union logger = logging.getLogger(__name__) DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() NCI_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/nci_unique.csv" def load_nci(featurizer='ECFP', shard_size=1000, split='random', reload=True, data_dir=None, save_dir=None, **kwargs): # Load nci dataset logger.info("About to load NCI dataset.") if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR all_nci_tasks = [ 'CCRF-CEM', 'HL-60(TB)', 'K-562', 'MOLT-4', 'RPMI-8226', 'SR', 'A549/ATCC', 'EKVX', 'HOP-62', 'HOP-92', 'NCI-H226', 'NCI-H23', 'NCI-H322M', 'NCI-H460', 'NCI-H522', 'COLO 205', 'HCC-2998', 'HCT-116', 'HCT-15', 'HT29', 'KM12', 'SW-620', 'SF-268', 'SF-295', 'SF-539', 'SNB-19', 'SNB-75', 'U251', 'LOX IMVI', 'MALME-3M', 'M14', 'MDA-MB-435', 'SK-MEL-2', 'SK-MEL-28', 'SK-MEL-5', 'UACC-257', 'UACC-62', 'IGR-OV1', 'OVCAR-3', 'OVCAR-4', 'OVCAR-5', 'OVCAR-8', 'NCI/ADR-RES', 'SK-OV-3', '786-0', 'A498', 'ACHN', 'CAKI-1', 'RXF 393', 'SN12C', 'TK-10', 'UO-31', 'PC-3', 'DU-145', 'MCF7', 'MDA-MB-231/ATCC', 'MDA-MB-468', 'HS 578T', 'BT-549', 'T-47D' NCI_TASKS = [ 'CCRF-CEM', 'HL-60(TB)', 'K-562', 'MOLT-4', 'RPMI-8226', 'SR', 'A549/ATCC', 'EKVX', 'HOP-62', 'HOP-92', 'NCI-H226', 'NCI-H23', 'NCI-H322M', 'NCI-H460', 'NCI-H522', 'COLO 205', 'HCC-2998', 'HCT-116', 'HCT-15', 'HT29', 'KM12', 'SW-620', 'SF-268', 'SF-295', 'SF-539', 'SNB-19', 'SNB-75', 'U251', 'LOX IMVI', 'MALME-3M', 'M14', 'MDA-MB-435', 'SK-MEL-2', 'SK-MEL-28', 'SK-MEL-5', 'UACC-257', 'UACC-62', 'IGR-OV1', 'OVCAR-3', 'OVCAR-4', 'OVCAR-5', 'OVCAR-8', 'NCI/ADR-RES', 'SK-OV-3', '786-0', 'A498', 'ACHN', 'CAKI-1', 'RXF 393', 'SN12C', 'TK-10', 'UO-31', 'PC-3', 'DU-145', 'MCF7', 'MDA-MB-231/ATCC', 'MDA-MB-468', 'HS 578T', 'BT-549', 'T-47D' ] if reload: save_folder = os.path.join(save_dir, "nci-featurized", featurizer) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return all_nci_tasks, all_dataset, transformers class _NCILoader(_MolnetLoader): dataset_file = os.path.join(data_dir, "nci_unique.csv") def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "nci_unique.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=NCI_URL, dest_dir=data_dir) # Featurize nci dataset logger.info("About to featurize nci dataset.") if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") img_size = kwargs.get("img_size", 80) featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=all_nci_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, shard_size=shard_size) if split == None: logger.info("Split is None, about to transform data") transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=dataset) ] for transformer in transformers: dataset = transformer.transform(dataset) return all_nci_tasks, (dataset, None, None), transformers splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'scaffold': deepchem.splits.ScaffoldSplitter() } splitter = splitters[split] logger.info("About to split data with {} splitter.".format(splitter)) frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train, valid, test = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer(transform_y=True, dataset=train) ] logger.info("About to transform dataset.") for transformer in transformers: train = transformer.transform(train) valid = transformer.transform(valid) test = transformer.transform(test) if reload: deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid, test, transformers) return all_nci_tasks, (train, valid, test), transformers dc.utils.data_utils.download_url(url=NCI_URL, dest_dir=self.data_dir) loader = dc.data.CSVLoader( tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_nci( featurizer: Union[dc.feat.Featurizer, str] = 'ECFP', splitter: Union[dc.splits.Splitter, str, None] = 'random', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load NCI dataset. Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in """ loader = _NCILoader(featurizer, splitter, transformers, NCI_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('nci', reload)
deepchem/molnet/load_function/ppb_datasets.py +54 −103 Original line number Diff line number Diff line Loading @@ -2,108 +2,59 @@ PPB dataset loader. """ import os import logging import deepchem import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union logger = logging.getLogger(__name__) DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() PPB_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/PPB.csv" PPB_TASKS = ['exp'] def load_ppb(featurizer='ECFP', split='index', reload=True, data_dir=None, save_dir=None, **kwargs): """Load PPB datasets.""" # Featurize PPB dataset logger.info("About to featurize PPB dataset.") logger.info("About to load PPB dataset.") PPB_tasks = ['exp'] if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR if reload: save_folder = os.path.join(save_dir, "ppb-featurized", str(featurizer)) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) class _PPBLoader(_MolnetLoader): loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return PPB_tasks, all_dataset, transformers dataset_file = os.path.join(data_dir, "PPB.csv") def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "PPB.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=PPB_URL, dest_dir=data_dir) if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") img_size = kwargs.get("img_size", 80) featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=PPB_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, shard_size=8192) if split == None: transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=dataset) ] logger.info("Split is None, about to transform data") for transformer in transformers: dataset = transformer.transform(dataset) return PPB_tasks, (dataset, None, None), transformers splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'scaffold': deepchem.splits.ScaffoldSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter() } splitter = splitters[split] logger.info("About to split dataset with {} splitter.".format(split)) frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train, valid, test = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer(transform_y=True, dataset=train) ] logger.info("About to transform dataset.") for transformer in transformers: train = transformer.transform(train) valid = transformer.transform(valid) test = transformer.transform(test) if reload: deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid, test, transformers) return PPB_tasks, (train, valid, test), transformers dc.utils.data_utils.download_url(url=PPB_URL, dest_dir=self.data_dir) loader = dc.data.CSVLoader( tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_ppb( featurizer: Union[dc.feat.Featurizer, str] = 'ECFP', splitter: Union[dc.splits.Splitter, str, None] = 'scaffold', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load PPB datasets. Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in """ loader = _PPBLoader(featurizer, splitter, transformers, PPB_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('ppb', reload)
deepchem/molnet/load_function/sampl_datasets.py +53 −114 Original line number Diff line number Diff line Loading @@ -2,22 +2,35 @@ SAMPL dataset loader. """ import os import logging import deepchem logger = logging.getLogger(__name__) import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union SAMPL_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/SAMPL.csv" DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() SAMPL_TASKS = ['expt'] class _SAMPLLoader(_MolnetLoader): def load_sampl(featurizer='ECFP', split='index', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "SAMPL.csv") if not os.path.exists(dataset_file): dc.utils.data_utils.download_url(url=SAMPL_URL, dest_dir=self.data_dir) loader = dc.data.CSVLoader( tasks=self.tasks, feature_field="smiles", featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_sampl( featurizer: Union[dc.feat.Featurizer, str] = 'ECFP', splitter: Union[dc.splits.Splitter, str, None] = 'scaffold', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load SAMPL(FreeSolv) dataset The Free Solvation Database, FreeSolv(SAMPL), provides experimental and Loading @@ -36,6 +49,27 @@ def load_sampl(featurizer='ECFP', used as label - "calc" - Calculated solvation energy (unit: kcal/mol) of the compound Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in References ---------- Loading @@ -43,101 +77,6 @@ def load_sampl(featurizer='ECFP', experimental and calculated hydration free energies, with input files." Journal of computer-aided molecular design 28.7 (2014): 711-720. """ # Featurize SAMPL dataset logger.info("About to featurize SAMPL dataset.") logger.info("About to load SAMPL dataset.") if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR if reload: save_folder = os.path.join(save_dir, "sampl-featurized") if not move_mean: save_folder = os.path.join(save_folder, str(featurizer) + "_mean_unmoved") else: save_folder = os.path.join(save_folder, str(featurizer)) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) dataset_file = os.path.join(data_dir, "SAMPL.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=SAMPL_URL, dest_dir=data_dir) SAMPL_tasks = ['expt'] if reload: loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return SAMPL_tasks, all_dataset, transformers if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == 'smiles2img': img_size = kwargs.get("img_size", 80) img_spec = kwargs.get("img_spec", "std") featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=SAMPL_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, shard_size=8192) if split == None: transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=dataset, move_mean=move_mean) ] logger.info("Split is None, about to transform data") for transformer in transformers: dataset = transformer.transform(dataset) return SAMPL_tasks, (dataset, None, None), transformers splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'scaffold': deepchem.splits.ScaffoldSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] logger.info("About to split dataset with {} splitter.".format(split)) frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train, valid, test = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) train, valid, test = splitter.train_valid_test_split(dataset) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train, move_mean=move_mean) ] logger.info("About to transform dataset.") for transformer in transformers: train = transformer.transform(train) valid = transformer.transform(valid) test = transformer.transform(test) if reload: deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid, test, transformers) return SAMPL_tasks, (train, valid, test), transformers loader = _SAMPLLoader(featurizer, splitter, transformers, SAMPL_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('sampl', reload)
deepchem/molnet/load_function/sider_datasets.py +71 −103 File changed.Preview size limit exceeded, changes collapsed. Show changes
examples/sider/sider_rf.py +1 −2 Original line number Diff line number Diff line Loading @@ -10,10 +10,9 @@ import os import shutil import numpy as np import deepchem as dc from sider_datasets import load_sider from sklearn.ensemble import RandomForestClassifier sider_tasks, datasets, transformers = load_sider() sider_tasks, datasets, transformers = dc.molnet.load_sider() train_dataset, valid_dataset, test_dataset = datasets metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean, Loading
