Loading deepchem/molnet/load_function/qm7_datasets.py +49 −267 Original line number Diff line number Diff line Loading @@ -2,233 +2,39 @@ qm7 dataset loader. """ import os import numpy as np import deepchem import scipy.io import logging import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union logger = logging.getLogger(__name__) DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() QM7_MAT_UTL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/qm7.mat" QM7_CSV_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/qm7.csv" QM7B_MAT_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/qm7b.mat" GDB7_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/gdb7.tar.gz" QM7_TASKS = ["u0_atom"] def load_qm7_from_mat(featurizer='CoulombMatrix', split='stratified', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): qm7_tasks = ["u0_atom"] if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR if reload: save_folder = os.path.join(save_dir, "qm7-featurized") if not move_mean: save_folder = os.path.join(save_folder, str(featurizer) + "_mean_unmoved") else: save_folder = os.path.join(save_folder, str(featurizer)) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return qm7_tasks, all_dataset, transformers if featurizer == 'CoulombMatrix': dataset_file = os.path.join(data_dir, "qm7.mat") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM7_MAT_URL, dest_dir=data_dir) dataset = scipy.io.loadmat(dataset_file) X = dataset['X'] y = dataset['T'].T w = np.ones_like(y) dataset = deepchem.data.DiskDataset.from_numpy(X, y, w, ids=None) elif featurizer == 'BPSymmetryFunctionInput': dataset_file = os.path.join(data_dir, "qm7.mat") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM7_MAT_URL, dest_dir=data_dir) dataset = scipy.io.loadmat(dataset_file) X = np.concatenate([np.expand_dims(dataset['Z'], 2), dataset['R']], axis=2) y = dataset['T'].reshape(-1, 1) # scipy.io.loadmat puts samples on axis 1 w = np.ones_like(y) dataset = deepchem.data.DiskDataset.from_numpy(X, y, w, ids=None) else: dataset_file = os.path.join(data_dir, "qm7.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM7_CSV_URL, dest_dir=data_dir) if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") img_size = kwargs.get("img_size", 80) featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=qm7_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file) if split == None: raise ValueError() else: splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train_dataset, move_mean=move_mean) ] for transformer in transformers: train_dataset = transformer.transform(train_dataset) valid_dataset = transformer.transform(valid_dataset) test_dataset = transformer.transform(test_dataset) if reload: deepchem.utils.data_utils.save_dataset_to_disk( save_folder, train_dataset, valid_dataset, test_dataset, transformers) return qm7_tasks, (train_dataset, valid_dataset, test_dataset), transformers def load_qm7b_from_mat(featurizer='CoulombMatrix', split='stratified', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): """Load QM7B dataset QM7b is an extension for the QM7 dataset with additional properties predicted at different levels (ZINDO, SCS, PBE0, GW). In total 14 tasks are included for 7211 molecules with up to 7 heavy atoms. Random splitting is recommended for this dataset. The data file (.mat format, we recommend using `scipy.io.loadmat` for python users to load this original data) contains two arrays: - "X" - (7211 x 23 x 23), Coulomb matrices - "T" - (7211 x 14), properties: #. Atomization energies E (PBE0, unit: kcal/mol) #. Excitation of maximal optimal absorption E_max (ZINDO, unit: eV) #. Absorption Intensity at maximal absorption I_max (ZINDO) #. Highest occupied molecular orbital HOMO (ZINDO, unit: eV) #. Lowest unoccupied molecular orbital LUMO (ZINDO, unit: eV) #. First excitation energy E_1st (ZINDO, unit: eV) #. Ionization potential IP (ZINDO, unit: eV) #. Electron affinity EA (ZINDO, unit: eV) #. Highest occupied molecular orbital HOMO (PBE0, unit: eV) #. Lowest unoccupied molecular orbital LUMO (PBE0, unit: eV) #. Highest occupied molecular orbital HOMO (GW, unit: eV) #. Lowest unoccupied molecular orbital LUMO (GW, unit: eV) #. Polarizabilities α (PBE0, unit: Å^3) #. Polarizabilities α (SCS, unit: Å^3) References ---------- .. [1] Blum, Lorenz C., and Jean-Louis Reymond. "970 million druglike small molecules for virtual screening in the chemical universe database GDB-13." Journal of the American Chemical Society 131.25 (2009): 8732-8733. .. [2] Montavon, Grégoire, et al. "Machine learning of molecular electronic properties in chemical compound space." New Journal of Physics 15.9 (2013): 095003. """ if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR dataset_file = os.path.join(data_dir, "qm7b.mat") class _QM7Loader(_MolnetLoader): def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "gdb7.sdf") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM7B_MAT_URL, dest_dir=data_dir) dataset = scipy.io.loadmat(dataset_file) X = dataset['X'] y = dataset['T'] w = np.ones_like(y) dataset = deepchem.data.DiskDataset.from_numpy(X, y, w, ids=None) if split == None: raise ValueError() else: splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train_dataset, move_mean=move_mean) ] for transformer in transformers: train_dataset = transformer.transform(train_dataset) valid_dataset = transformer.transform(valid_dataset) test_dataset = transformer.transform(test_dataset) qm7_tasks = np.arange(y.shape[1]) return qm7_tasks, (train_dataset, valid_dataset, test_dataset), transformers def load_qm7(featurizer='CoulombMatrix', split='random', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): dc.utils.data_utils.download_url(url=GDB7_URL, dest_dir=self.data_dir) dc.utils.data_utils.untargz_file( os.path.join(self.data_dir, "gdb7.tar.gz"), self.data_dir) loader = dc.data.SDFLoader(tasks=self.tasks, featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_qm7( featurizer: Union[dc.feat.Featurizer, str] = dc.feat.CoulombMatrix(23), splitter: Union[dc.splits.Splitter, str, None] = 'random', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load QM7 dataset QM7 is a subset of GDB-13 (a database of nearly 1 billion Loading @@ -253,6 +59,28 @@ def load_qm7(featurizer='CoulombMatrix', - "R" - (7165 x 23 x 3), cartesian coordinate (unit: Bohr) of each atom in the molecules Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in References ---------- .. [1] Rupp, Matthias, et al. "Fast and accurate modeling of molecular Loading @@ -262,52 +90,6 @@ def load_qm7(featurizer='CoulombMatrix', molecules for atomization energy prediction." Advances in Neural Information Proccessing Systems. 2012. """ # Featurize qm7 dataset logger.info("About to featurize qm7 dataset.") if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR dataset_file = os.path.join(data_dir, "gdb7.sdf") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=GDB7_URL, dest_dir=data_dir) deepchem.utils.data_utils.untargz_file( os.path.join(data_dir, 'gdb7.tar.gz'), data_dir) qm7_tasks = ["u0_atom"] if featurizer == 'CoulombMatrix': featurizer = deepchem.feat.CoulombMatrixEig(23) loader = deepchem.data.SDFLoader(tasks=qm7_tasks, featurizer=featurizer) dataset = loader.featurize(dataset_file) if split == None: raise ValueError() splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train_dataset, move_mean=move_mean) ] for transformer in transformers: train_dataset = transformer.transform(train_dataset) valid_dataset = transformer.transform(valid_dataset) test_dataset = transformer.transform(test_dataset) return qm7_tasks, (train_dataset, valid_dataset, test_dataset), transformers loader = _QM7Loader(featurizer, splitter, transformers, QM7_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('qm7', reload) deepchem/molnet/load_function/qm8_datasets.py +55 −108 Original line number Diff line number Diff line Loading @@ -2,23 +2,41 @@ qm8 dataset loader. """ import os import deepchem import logging import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union logger = logging.getLogger(__name__) DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() GDB8_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/gdb8.tar.gz" QM8_CSV_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/qm8.csv" QM8_TASKS = [ "E1-CC2", "E2-CC2", "f1-CC2", "f2-CC2", "E1-PBE0", "E2-PBE0", "f1-PBE0", "f2-PBE0", "E1-PBE0", "E2-PBE0", "f1-PBE0", "f2-PBE0", "E1-CAM", "E2-CAM", "f1-CAM", "f2-CAM" ] def load_qm8(featurizer='CoulombMatrix', split='random', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): class _QM8Loader(_MolnetLoader): def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "qm8.sdf") if not os.path.exists(dataset_file): dc.utils.data_utils.download_url(url=GDB8_URL, dest_dir=self.data_dir) dc.utils.data_utils.untargz_file( os.path.join(self.data_dir, "gdb8.tar.gz"), self.data_dir) loader = dc.data.SDFLoader(tasks=self.tasks, featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_qm8( featurizer: Union[dc.feat.Featurizer, str] = dc.feat.CoulombMatrix(26), splitter: Union[dc.splits.Splitter, str, None] = 'random', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load QM8 dataset QM8 is the dataset used in a study on modeling quantum Loading Loading @@ -50,6 +68,28 @@ def load_qm8(featurizer='CoulombMatrix', - Columns 10-13: LR-TDPBE0/def2TZVP - Columns 14-17: LR-TDCAM-B3LYP/def2TZVP Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in References ---------- .. [1] Blum, Lorenz C., and Jean-Louis Reymond. "970 million druglike Loading @@ -60,99 +100,6 @@ def load_qm8(featurizer='CoulombMatrix', and machine learning in chemical space." The Journal of chemical physics 143.8 (2015): 084111. """ qm8_tasks = [ "E1-CC2", "E2-CC2", "f1-CC2", "f2-CC2", "E1-PBE0", "E2-PBE0", "f1-PBE0", "f2-PBE0", "E1-PBE0", "E2-PBE0", "f1-PBE0", "f2-PBE0", "E1-CAM", "E2-CAM", "f1-CAM", "f2-CAM" ] if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR if reload: save_folder = os.path.join(save_dir, "qm8-featurized") if not move_mean: save_folder = os.path.join(save_folder, str(featurizer) + "_mean_unmoved") else: save_folder = os.path.join(save_folder, str(featurizer)) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return qm8_tasks, all_dataset, transformers if featurizer in ['CoulombMatrix', 'BPSymmetryFunctionInput', 'MP', 'Raw']: dataset_file = os.path.join(data_dir, "qm8.sdf") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=GDB8_URL, dest_dir=data_dir) deepchem.utils.data_utils.untargz_file( os.path.join(data_dir, 'gdb8.tar.gz'), data_dir) else: dataset_file = os.path.join(data_dir, "qm8.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM8_CSV_URL, dest_dir=data_dir) if featurizer in ['CoulombMatrix', 'BPSymmetryFunctionInput', 'MP', 'Raw']: if featurizer == 'CoulombMatrix': featurizer = deepchem.feat.CoulombMatrix(26) elif featurizer == 'BPSymmetryFunctionInput': featurizer = deepchem.feat.BPSymmetryFunctionInput(26) elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == 'MP': featurizer = deepchem.feat.WeaveFeaturizer( graph_distance=False, explicit_H=True) loader = deepchem.data.SDFLoader(tasks=qm8_tasks, featurizer=featurizer) else: if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") img_size = kwargs.get("img_size", 80) featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=qm8_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file) if split == None: raise ValueError() splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train_dataset, move_mean=move_mean) ] for transformer in transformers: train_dataset = transformer.transform(train_dataset) valid_dataset = transformer.transform(valid_dataset) test_dataset = transformer.transform(test_dataset) if reload: deepchem.utils.data_utils.save_dataset_to_disk( save_folder, train_dataset, valid_dataset, test_dataset, transformers) return qm8_tasks, (train_dataset, valid_dataset, test_dataset), transformers loader = _QM8Loader(featurizer, splitter, transformers, QM8_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('qm8', reload) deepchem/molnet/load_function/qm9_datasets.py +54 −111 File changed.Preview size limit exceeded, changes collapsed. Show changes Loading
deepchem/molnet/load_function/qm7_datasets.py +49 −267 Original line number Diff line number Diff line Loading @@ -2,233 +2,39 @@ qm7 dataset loader. """ import os import numpy as np import deepchem import scipy.io import logging import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union logger = logging.getLogger(__name__) DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() QM7_MAT_UTL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/qm7.mat" QM7_CSV_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/qm7.csv" QM7B_MAT_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/qm7b.mat" GDB7_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/gdb7.tar.gz" QM7_TASKS = ["u0_atom"] def load_qm7_from_mat(featurizer='CoulombMatrix', split='stratified', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): qm7_tasks = ["u0_atom"] if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR if reload: save_folder = os.path.join(save_dir, "qm7-featurized") if not move_mean: save_folder = os.path.join(save_folder, str(featurizer) + "_mean_unmoved") else: save_folder = os.path.join(save_folder, str(featurizer)) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return qm7_tasks, all_dataset, transformers if featurizer == 'CoulombMatrix': dataset_file = os.path.join(data_dir, "qm7.mat") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM7_MAT_URL, dest_dir=data_dir) dataset = scipy.io.loadmat(dataset_file) X = dataset['X'] y = dataset['T'].T w = np.ones_like(y) dataset = deepchem.data.DiskDataset.from_numpy(X, y, w, ids=None) elif featurizer == 'BPSymmetryFunctionInput': dataset_file = os.path.join(data_dir, "qm7.mat") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM7_MAT_URL, dest_dir=data_dir) dataset = scipy.io.loadmat(dataset_file) X = np.concatenate([np.expand_dims(dataset['Z'], 2), dataset['R']], axis=2) y = dataset['T'].reshape(-1, 1) # scipy.io.loadmat puts samples on axis 1 w = np.ones_like(y) dataset = deepchem.data.DiskDataset.from_numpy(X, y, w, ids=None) else: dataset_file = os.path.join(data_dir, "qm7.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM7_CSV_URL, dest_dir=data_dir) if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") img_size = kwargs.get("img_size", 80) featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=qm7_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file) if split == None: raise ValueError() else: splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train_dataset, move_mean=move_mean) ] for transformer in transformers: train_dataset = transformer.transform(train_dataset) valid_dataset = transformer.transform(valid_dataset) test_dataset = transformer.transform(test_dataset) if reload: deepchem.utils.data_utils.save_dataset_to_disk( save_folder, train_dataset, valid_dataset, test_dataset, transformers) return qm7_tasks, (train_dataset, valid_dataset, test_dataset), transformers def load_qm7b_from_mat(featurizer='CoulombMatrix', split='stratified', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): """Load QM7B dataset QM7b is an extension for the QM7 dataset with additional properties predicted at different levels (ZINDO, SCS, PBE0, GW). In total 14 tasks are included for 7211 molecules with up to 7 heavy atoms. Random splitting is recommended for this dataset. The data file (.mat format, we recommend using `scipy.io.loadmat` for python users to load this original data) contains two arrays: - "X" - (7211 x 23 x 23), Coulomb matrices - "T" - (7211 x 14), properties: #. Atomization energies E (PBE0, unit: kcal/mol) #. Excitation of maximal optimal absorption E_max (ZINDO, unit: eV) #. Absorption Intensity at maximal absorption I_max (ZINDO) #. Highest occupied molecular orbital HOMO (ZINDO, unit: eV) #. Lowest unoccupied molecular orbital LUMO (ZINDO, unit: eV) #. First excitation energy E_1st (ZINDO, unit: eV) #. Ionization potential IP (ZINDO, unit: eV) #. Electron affinity EA (ZINDO, unit: eV) #. Highest occupied molecular orbital HOMO (PBE0, unit: eV) #. Lowest unoccupied molecular orbital LUMO (PBE0, unit: eV) #. Highest occupied molecular orbital HOMO (GW, unit: eV) #. Lowest unoccupied molecular orbital LUMO (GW, unit: eV) #. Polarizabilities α (PBE0, unit: Å^3) #. Polarizabilities α (SCS, unit: Å^3) References ---------- .. [1] Blum, Lorenz C., and Jean-Louis Reymond. "970 million druglike small molecules for virtual screening in the chemical universe database GDB-13." Journal of the American Chemical Society 131.25 (2009): 8732-8733. .. [2] Montavon, Grégoire, et al. "Machine learning of molecular electronic properties in chemical compound space." New Journal of Physics 15.9 (2013): 095003. """ if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR dataset_file = os.path.join(data_dir, "qm7b.mat") class _QM7Loader(_MolnetLoader): def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "gdb7.sdf") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM7B_MAT_URL, dest_dir=data_dir) dataset = scipy.io.loadmat(dataset_file) X = dataset['X'] y = dataset['T'] w = np.ones_like(y) dataset = deepchem.data.DiskDataset.from_numpy(X, y, w, ids=None) if split == None: raise ValueError() else: splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train_dataset, move_mean=move_mean) ] for transformer in transformers: train_dataset = transformer.transform(train_dataset) valid_dataset = transformer.transform(valid_dataset) test_dataset = transformer.transform(test_dataset) qm7_tasks = np.arange(y.shape[1]) return qm7_tasks, (train_dataset, valid_dataset, test_dataset), transformers def load_qm7(featurizer='CoulombMatrix', split='random', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): dc.utils.data_utils.download_url(url=GDB7_URL, dest_dir=self.data_dir) dc.utils.data_utils.untargz_file( os.path.join(self.data_dir, "gdb7.tar.gz"), self.data_dir) loader = dc.data.SDFLoader(tasks=self.tasks, featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_qm7( featurizer: Union[dc.feat.Featurizer, str] = dc.feat.CoulombMatrix(23), splitter: Union[dc.splits.Splitter, str, None] = 'random', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load QM7 dataset QM7 is a subset of GDB-13 (a database of nearly 1 billion Loading @@ -253,6 +59,28 @@ def load_qm7(featurizer='CoulombMatrix', - "R" - (7165 x 23 x 3), cartesian coordinate (unit: Bohr) of each atom in the molecules Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in References ---------- .. [1] Rupp, Matthias, et al. "Fast and accurate modeling of molecular Loading @@ -262,52 +90,6 @@ def load_qm7(featurizer='CoulombMatrix', molecules for atomization energy prediction." Advances in Neural Information Proccessing Systems. 2012. """ # Featurize qm7 dataset logger.info("About to featurize qm7 dataset.") if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR dataset_file = os.path.join(data_dir, "gdb7.sdf") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=GDB7_URL, dest_dir=data_dir) deepchem.utils.data_utils.untargz_file( os.path.join(data_dir, 'gdb7.tar.gz'), data_dir) qm7_tasks = ["u0_atom"] if featurizer == 'CoulombMatrix': featurizer = deepchem.feat.CoulombMatrixEig(23) loader = deepchem.data.SDFLoader(tasks=qm7_tasks, featurizer=featurizer) dataset = loader.featurize(dataset_file) if split == None: raise ValueError() splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train_dataset, move_mean=move_mean) ] for transformer in transformers: train_dataset = transformer.transform(train_dataset) valid_dataset = transformer.transform(valid_dataset) test_dataset = transformer.transform(test_dataset) return qm7_tasks, (train_dataset, valid_dataset, test_dataset), transformers loader = _QM7Loader(featurizer, splitter, transformers, QM7_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('qm7', reload)
deepchem/molnet/load_function/qm8_datasets.py +55 −108 Original line number Diff line number Diff line Loading @@ -2,23 +2,41 @@ qm8 dataset loader. """ import os import deepchem import logging import deepchem as dc from deepchem.molnet.load_function.molnet_loader import TransformerGenerator, _MolnetLoader from deepchem.data import Dataset from typing import List, Optional, Tuple, Union logger = logging.getLogger(__name__) DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir() GDB8_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/gdb8.tar.gz" QM8_CSV_URL = "https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/qm8.csv" QM8_TASKS = [ "E1-CC2", "E2-CC2", "f1-CC2", "f2-CC2", "E1-PBE0", "E2-PBE0", "f1-PBE0", "f2-PBE0", "E1-PBE0", "E2-PBE0", "f1-PBE0", "f2-PBE0", "E1-CAM", "E2-CAM", "f1-CAM", "f2-CAM" ] def load_qm8(featurizer='CoulombMatrix', split='random', reload=True, move_mean=True, data_dir=None, save_dir=None, **kwargs): class _QM8Loader(_MolnetLoader): def create_dataset(self) -> Dataset: dataset_file = os.path.join(self.data_dir, "qm8.sdf") if not os.path.exists(dataset_file): dc.utils.data_utils.download_url(url=GDB8_URL, dest_dir=self.data_dir) dc.utils.data_utils.untargz_file( os.path.join(self.data_dir, "gdb8.tar.gz"), self.data_dir) loader = dc.data.SDFLoader(tasks=self.tasks, featurizer=self.featurizer) return loader.create_dataset(dataset_file, shard_size=8192) def load_qm8( featurizer: Union[dc.feat.Featurizer, str] = dc.feat.CoulombMatrix(26), splitter: Union[dc.splits.Splitter, str, None] = 'random', transformers: List[Union[TransformerGenerator, str]] = ['normalization'], reload: bool = True, data_dir: Optional[str] = None, save_dir: Optional[str] = None, **kwargs ) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]: """Load QM8 dataset QM8 is the dataset used in a study on modeling quantum Loading Loading @@ -50,6 +68,28 @@ def load_qm8(featurizer='CoulombMatrix', - Columns 10-13: LR-TDPBE0/def2TZVP - Columns 14-17: LR-TDCAM-B3LYP/def2TZVP Parameters ---------- featurizer: Featurizer or str the featurizer to use for processing the data. Alternatively you can pass one of the names from dc.molnet.featurizers as a shortcut. splitter: Splitter or str the splitter to use for splitting the data into training, validation, and test sets. Alternatively you can pass one of the names from dc.molnet.splitters as a shortcut. If this is None, all the data will be included in a single dataset. transformers: list of TransformerGenerators or strings the Transformers to apply to the data. Each one is specified by a TransformerGenerator or, as a shortcut, one of the names from dc.molnet.transformers. reload: bool if True, the first call for a particular featurizer and splitter will cache the datasets to disk, and subsequent calls will reload the cached datasets. data_dir: str a directory to save the raw data in save_dir: str a directory to save the dataset in References ---------- .. [1] Blum, Lorenz C., and Jean-Louis Reymond. "970 million druglike Loading @@ -60,99 +100,6 @@ def load_qm8(featurizer='CoulombMatrix', and machine learning in chemical space." The Journal of chemical physics 143.8 (2015): 084111. """ qm8_tasks = [ "E1-CC2", "E2-CC2", "f1-CC2", "f2-CC2", "E1-PBE0", "E2-PBE0", "f1-PBE0", "f2-PBE0", "E1-PBE0", "E2-PBE0", "f1-PBE0", "f2-PBE0", "E1-CAM", "E2-CAM", "f1-CAM", "f2-CAM" ] if data_dir is None: data_dir = DEFAULT_DIR if save_dir is None: save_dir = DEFAULT_DIR if reload: save_folder = os.path.join(save_dir, "qm8-featurized") if not move_mean: save_folder = os.path.join(save_folder, str(featurizer) + "_mean_unmoved") else: save_folder = os.path.join(save_folder, str(featurizer)) if featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") save_folder = os.path.join(save_folder, img_spec) save_folder = os.path.join(save_folder, str(split)) loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk( save_folder) if loaded: return qm8_tasks, all_dataset, transformers if featurizer in ['CoulombMatrix', 'BPSymmetryFunctionInput', 'MP', 'Raw']: dataset_file = os.path.join(data_dir, "qm8.sdf") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=GDB8_URL, dest_dir=data_dir) deepchem.utils.data_utils.untargz_file( os.path.join(data_dir, 'gdb8.tar.gz'), data_dir) else: dataset_file = os.path.join(data_dir, "qm8.csv") if not os.path.exists(dataset_file): deepchem.utils.data_utils.download_url(url=QM8_CSV_URL, dest_dir=data_dir) if featurizer in ['CoulombMatrix', 'BPSymmetryFunctionInput', 'MP', 'Raw']: if featurizer == 'CoulombMatrix': featurizer = deepchem.feat.CoulombMatrix(26) elif featurizer == 'BPSymmetryFunctionInput': featurizer = deepchem.feat.BPSymmetryFunctionInput(26) elif featurizer == 'Raw': featurizer = deepchem.feat.RawFeaturizer() elif featurizer == 'MP': featurizer = deepchem.feat.WeaveFeaturizer( graph_distance=False, explicit_H=True) loader = deepchem.data.SDFLoader(tasks=qm8_tasks, featurizer=featurizer) else: if featurizer == 'ECFP': featurizer = deepchem.feat.CircularFingerprint(size=1024) elif featurizer == 'GraphConv': featurizer = deepchem.feat.ConvMolFeaturizer() elif featurizer == 'Weave': featurizer = deepchem.feat.WeaveFeaturizer() elif featurizer == "smiles2img": img_spec = kwargs.get("img_spec", "std") img_size = kwargs.get("img_size", 80) featurizer = deepchem.feat.SmilesToImage( img_size=img_size, img_spec=img_spec) loader = deepchem.data.CSVLoader( tasks=qm8_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file) if split == None: raise ValueError() splitters = { 'index': deepchem.splits.IndexSplitter(), 'random': deepchem.splits.RandomSplitter(), 'stratified': deepchem.splits.SingletaskStratifiedSplitter(task_number=0) } splitter = splitters[split] frac_train = kwargs.get("frac_train", 0.8) frac_valid = kwargs.get('frac_valid', 0.1) frac_test = kwargs.get('frac_test', 0.1) train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( dataset, frac_train=frac_train, frac_valid=frac_valid, frac_test=frac_test) transformers = [ deepchem.trans.NormalizationTransformer( transform_y=True, dataset=train_dataset, move_mean=move_mean) ] for transformer in transformers: train_dataset = transformer.transform(train_dataset) valid_dataset = transformer.transform(valid_dataset) test_dataset = transformer.transform(test_dataset) if reload: deepchem.utils.data_utils.save_dataset_to_disk( save_folder, train_dataset, valid_dataset, test_dataset, transformers) return qm8_tasks, (train_dataset, valid_dataset, test_dataset), transformers loader = _QM8Loader(featurizer, splitter, transformers, QM8_TASKS, data_dir, save_dir, **kwargs) return loader.load_dataset('qm8', reload)
deepchem/molnet/load_function/qm9_datasets.py +54 −111 File changed.Preview size limit exceeded, changes collapsed. Show changes
