Loading deepchem/data/data_loader.py +19 −6 Original line number Diff line number Diff line Loading @@ -64,8 +64,13 @@ def featurize_smiles_df(df, featurizer, field, log_every_N=1000, verbose=True): mol = Chem.MolFromSmiles(elem) if ind % log_every_N == 0: log("Featurizing sample %d" % ind, verbose) features.append(np.squeeze(featurizer.featurize([mol]))) return np.array(features) features.append(featurizer.featurize([mol])) ########################################################## DEBUG valid_inds = np.array([1 if elt.size > 0 else 0 for elt in features], dtype=bool) features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid] return np.squeeze(np.array(features)), valid_inds ########################################################### DEBUG def get_user_specified_features(df, featurizer, verbose=True): """Extract and merge user specified features. Loading Loading @@ -105,7 +110,12 @@ def featurize_mol_df(df, featurizer, field, verbose=True, log_every_N=1000): if ind % log_every_N == 0: log("Featurizing sample %d" % ind, verbose) features.append(featurizer.featurize([mol])) return np.array(features) ########################################################## DEBUG valid_inds = np.array([1 if elt.size > 0 else 0 for elt in features], dtype=bool) features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid] return np.squeeze(np.array(features)), valid_inds ########################################################### DEBUG class DataLoader(object): """ Loading Loading @@ -145,8 +155,10 @@ class DataLoader(object): def shard_generator(): for shard_num, shard in enumerate(self.get_shards(input_files, shard_size)): time1 = time.time() X = self.featurize_shard(shard) X, valid_inds = self.featurize_shard(shard) ids, y, w = convert_df_to_numpy(shard, self.tasks, self.id_field) # Filter out examples where featurization failed. ids, y, w = (ids[valid_inds], y[valid_inds], w[valid_inds]) assert len(X) == len(ids) == len(y) == len(w) time2 = time.time() log("TIMING: featurizing shard %d took %0.3f s" % (shard_num, time2-time1), Loading Loading @@ -185,13 +197,14 @@ class UserCSVLoader(DataLoader): def featurize_shard(self, shard): """Featurizes a shard of an input dataframe.""" assert isinstance(self.featurizer, UserDefinedFeaturizer) return get_user_specified_features(shard, self.featurizer) X = get_user_specified_features(shard, self.featurizer) return (X, np.ones(len(X))) class SDFLoader(DataLoader): """ Handles loading of SDF files. """ def get_shard(self, input_files, shard_size): def get_shards(self, input_files, shard_size): """Defines a generator which returns data for each shard""" return load_sdf_files(input_files) Loading deepchem/data/datasets.py +28 −7 Original line number Diff line number Diff line Loading @@ -347,7 +347,7 @@ class DiskDataset(Dataset): """ A Dataset that is stored as a set of files on disk. """ def __init__(self, shard_generator=None, data_dir=None, tasks=[], def __init__(self, shard_generator=[], data_dir=None, tasks=[], reload=False, verbose=True): """ Turns featurized dataframes into numpy files, writes them & metadata to disk. Loading @@ -363,8 +363,15 @@ class DiskDataset(Dataset): if reload: log("Loading pre-existing dataset.", self.verbose) if os.path.exists(self._get_metadata_filename()): self.metadata_df = load_from_disk(self._get_metadata_filename()) (self.tasks, self.metadata_df) = load_from_disk( self._get_metadata_filename()) else: ################################################ DEBUG print("self.data_dir") print(self.data_dir) print("os.listdir(self.data_dir)") print(os.listdir(self.data_dir)) ################################################ DEBUG raise ValueError("No metadata found.") return Loading @@ -375,6 +382,7 @@ class DiskDataset(Dataset): metadata_rows.append( DiskDataset.write_data_to_disk( self.data_dir, basename, tasks, X, y, w, ids)) self.tasks = tasks self.metadata_df = DiskDataset.construct_metadata(metadata_rows) self.save_to_disk() time2 = time.time() Loading Loading @@ -414,15 +422,16 @@ class DiskDataset(Dataset): def save_to_disk(self): """Save dataset to disk.""" save_to_disk( self.metadata_df, self._get_metadata_filename()) (self.tasks, self.metadata_df), self._get_metadata_filename()) def get_task_names(self): """ Gets learning tasks associated with this dataset. """ if not len(self.metadata_df): raise ValueError("No data in dataset.") return next(self.metadata_df.iterrows())[1]['task_names'] return self.tasks #if not len(self.metadata_df): # raise ValueError("No data in dataset.") #return next(self.metadata_df.iterrows())[1]['task_names'] def reshard(self, shard_size): """Reshards data to have specified shard size.""" Loading Loading @@ -728,6 +737,18 @@ class DiskDataset(Dataset): os.path.join(self.data_dir, row['ids'])), dtype=object) return (X, y, w, ids) def add_shard(self, X, y, w, ids): """Adds a data shard.""" metadata_rows = self.metadata_df.values.tolist() shard_num = len(metadata_rows) basename = "shard-%d" % shard_num tasks = self.get_task_names() metadata_rows.append( DiskDataset.write_data_to_disk( self.data_dir, basename, tasks, X, y, w, ids)) self.metadata_df = DiskDataset.construct_metadata(metadata_rows) self.save_to_disk() def set_shard(self, shard_num, X, y, w, ids): """Writes data shard to disk""" basename = "shard-%d" % shard_num Loading Loading @@ -777,7 +798,7 @@ class DiskDataset(Dataset): # Break when all indices have been used up already if indices_count >= len(indices): return return DiskDataset(generator(), data_dir=select_dir) return DiskDataset(generator(), data_dir=select_dir, tasks=tasks) @property def ids(self): Loading deepchem/data/tests/__init__.py +6 −6 Original line number Diff line number Diff line Loading @@ -23,7 +23,7 @@ def load_solubility_data(): tasks = ["log-solubility"] task_type = "regression" input_file = os.path.join(current_dir, "../../models/tests/example.csv") featurizer = dc.data.DataLoader( featurizer = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) return featurizer.featurize(input_file) Loading @@ -36,7 +36,7 @@ def load_multitask_data(): "task13", "task14", "task15", "task16"] input_file = os.path.join( current_dir, "../../models/tests/multitask_example.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) return loader.featurize(input_file) Loading @@ -48,7 +48,7 @@ def load_classification_data(): task_type = "classification" input_file = os.path.join( current_dir, "../../models/tests/example_classification.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) return loader.featurize(input_file) Loading @@ -61,7 +61,7 @@ def load_sparse_multitask_dataset(): "task7", "task8", "task9"] input_file = os.path.join( current_dir, "../../models/tests/sparse_multitask_example.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) return loader.featurize(input_file) Loading @@ -73,7 +73,7 @@ def load_feat_multitask_data(): tasks = ["task0", "task1", "task2", "task3", "task4", "task5"] input_file = os.path.join( current_dir, "../../models/tests/feat_multitask_example.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, featurizer=featurizer, id_field="id") return loader.featurize(input_file) Loading @@ -87,6 +87,6 @@ def load_gaussian_cdf_data(): tasks = ["task0","task1"] input_file = os.path.join( current_dir, "../../models/tests/gaussian_cdf_example.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, featurizer=featurizer, id_field="id") return loader.featurize(input_file) deepchem/data/tests/test_drop.py +4 −3 Original line number Diff line number Diff line Loading @@ -29,12 +29,13 @@ class TestDrop(unittest.TestCase): featurizer = dc.feat.CircularFingerprint(size=1024) emols_tasks = ['activity'] loader = dc.data.DataLoader( tasks=emols_tasks, smiles_field="smiles", featurizer=featurizer) loader = dc.data.CSVLoader( tasks=emols_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file) X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids) ############################################################ DEBUG print("ids.shape, X.shape, y.shape, w.shape") print(ids.shape, X.shape, y.shape, w.shape) ############################################################ DEBUG assert len(X) == len(y) == len(w) == len(ids) deepchem/data/tests/test_load.py +10 −6 Original line number Diff line number Diff line Loading @@ -32,15 +32,19 @@ class TestLoad(unittest.TestCase): featurizer = dc.feat.CircularFingerprint(size=1024) tasks = ["log-solubility"] loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, data_dir) X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids) shutil.move(data_dir, moved_data_dir) ################################################ DEBUG print("data_dir, moved_data_dir") print(data_dir, moved_data_dir) ################################################ DEBUG moved_dataset = dc.data.DiskDataset( moved_data_dir, reload=True) data_dir=moved_data_dir, reload=True) X_moved, y_moved, w_moved, ids_moved = (moved_dataset.X, moved_dataset.y, moved_dataset.w, moved_dataset.ids) Loading Loading @@ -74,7 +78,7 @@ class TestLoad(unittest.TestCase): all_tasks = ["task%d"%i for i in range(17)] ####### Do featurization loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=all_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, data_dir) Loading @@ -87,7 +91,7 @@ class TestLoad(unittest.TestCase): y_tasks, w_tasks, = [], [] for ind, task in enumerate(all_tasks): print("Processing task %s" % task) dataset = dc.data.DiskDataset(data_dir, reload=reload) dataset = dc.data.DiskDataset(data_dir=data_dir, reload=reload) X_task, y_task, w_task, ids_task = (dataset.X, dataset.y, dataset.w, dataset.ids) Loading Loading @@ -131,7 +135,7 @@ class TestLoad(unittest.TestCase): tasks = all_tasks[0:n_tasks] ####### Do multitask load loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, data_dir) Loading @@ -145,7 +149,7 @@ class TestLoad(unittest.TestCase): print("Processing task %s" % task) if os.path.exists(data_dir): shutil.rmtree(data_dir) loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=[task], smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, data_dir) Loading Loading
deepchem/data/data_loader.py +19 −6 Original line number Diff line number Diff line Loading @@ -64,8 +64,13 @@ def featurize_smiles_df(df, featurizer, field, log_every_N=1000, verbose=True): mol = Chem.MolFromSmiles(elem) if ind % log_every_N == 0: log("Featurizing sample %d" % ind, verbose) features.append(np.squeeze(featurizer.featurize([mol]))) return np.array(features) features.append(featurizer.featurize([mol])) ########################################################## DEBUG valid_inds = np.array([1 if elt.size > 0 else 0 for elt in features], dtype=bool) features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid] return np.squeeze(np.array(features)), valid_inds ########################################################### DEBUG def get_user_specified_features(df, featurizer, verbose=True): """Extract and merge user specified features. Loading Loading @@ -105,7 +110,12 @@ def featurize_mol_df(df, featurizer, field, verbose=True, log_every_N=1000): if ind % log_every_N == 0: log("Featurizing sample %d" % ind, verbose) features.append(featurizer.featurize([mol])) return np.array(features) ########################################################## DEBUG valid_inds = np.array([1 if elt.size > 0 else 0 for elt in features], dtype=bool) features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid] return np.squeeze(np.array(features)), valid_inds ########################################################### DEBUG class DataLoader(object): """ Loading Loading @@ -145,8 +155,10 @@ class DataLoader(object): def shard_generator(): for shard_num, shard in enumerate(self.get_shards(input_files, shard_size)): time1 = time.time() X = self.featurize_shard(shard) X, valid_inds = self.featurize_shard(shard) ids, y, w = convert_df_to_numpy(shard, self.tasks, self.id_field) # Filter out examples where featurization failed. ids, y, w = (ids[valid_inds], y[valid_inds], w[valid_inds]) assert len(X) == len(ids) == len(y) == len(w) time2 = time.time() log("TIMING: featurizing shard %d took %0.3f s" % (shard_num, time2-time1), Loading Loading @@ -185,13 +197,14 @@ class UserCSVLoader(DataLoader): def featurize_shard(self, shard): """Featurizes a shard of an input dataframe.""" assert isinstance(self.featurizer, UserDefinedFeaturizer) return get_user_specified_features(shard, self.featurizer) X = get_user_specified_features(shard, self.featurizer) return (X, np.ones(len(X))) class SDFLoader(DataLoader): """ Handles loading of SDF files. """ def get_shard(self, input_files, shard_size): def get_shards(self, input_files, shard_size): """Defines a generator which returns data for each shard""" return load_sdf_files(input_files) Loading
deepchem/data/datasets.py +28 −7 Original line number Diff line number Diff line Loading @@ -347,7 +347,7 @@ class DiskDataset(Dataset): """ A Dataset that is stored as a set of files on disk. """ def __init__(self, shard_generator=None, data_dir=None, tasks=[], def __init__(self, shard_generator=[], data_dir=None, tasks=[], reload=False, verbose=True): """ Turns featurized dataframes into numpy files, writes them & metadata to disk. Loading @@ -363,8 +363,15 @@ class DiskDataset(Dataset): if reload: log("Loading pre-existing dataset.", self.verbose) if os.path.exists(self._get_metadata_filename()): self.metadata_df = load_from_disk(self._get_metadata_filename()) (self.tasks, self.metadata_df) = load_from_disk( self._get_metadata_filename()) else: ################################################ DEBUG print("self.data_dir") print(self.data_dir) print("os.listdir(self.data_dir)") print(os.listdir(self.data_dir)) ################################################ DEBUG raise ValueError("No metadata found.") return Loading @@ -375,6 +382,7 @@ class DiskDataset(Dataset): metadata_rows.append( DiskDataset.write_data_to_disk( self.data_dir, basename, tasks, X, y, w, ids)) self.tasks = tasks self.metadata_df = DiskDataset.construct_metadata(metadata_rows) self.save_to_disk() time2 = time.time() Loading Loading @@ -414,15 +422,16 @@ class DiskDataset(Dataset): def save_to_disk(self): """Save dataset to disk.""" save_to_disk( self.metadata_df, self._get_metadata_filename()) (self.tasks, self.metadata_df), self._get_metadata_filename()) def get_task_names(self): """ Gets learning tasks associated with this dataset. """ if not len(self.metadata_df): raise ValueError("No data in dataset.") return next(self.metadata_df.iterrows())[1]['task_names'] return self.tasks #if not len(self.metadata_df): # raise ValueError("No data in dataset.") #return next(self.metadata_df.iterrows())[1]['task_names'] def reshard(self, shard_size): """Reshards data to have specified shard size.""" Loading Loading @@ -728,6 +737,18 @@ class DiskDataset(Dataset): os.path.join(self.data_dir, row['ids'])), dtype=object) return (X, y, w, ids) def add_shard(self, X, y, w, ids): """Adds a data shard.""" metadata_rows = self.metadata_df.values.tolist() shard_num = len(metadata_rows) basename = "shard-%d" % shard_num tasks = self.get_task_names() metadata_rows.append( DiskDataset.write_data_to_disk( self.data_dir, basename, tasks, X, y, w, ids)) self.metadata_df = DiskDataset.construct_metadata(metadata_rows) self.save_to_disk() def set_shard(self, shard_num, X, y, w, ids): """Writes data shard to disk""" basename = "shard-%d" % shard_num Loading Loading @@ -777,7 +798,7 @@ class DiskDataset(Dataset): # Break when all indices have been used up already if indices_count >= len(indices): return return DiskDataset(generator(), data_dir=select_dir) return DiskDataset(generator(), data_dir=select_dir, tasks=tasks) @property def ids(self): Loading
deepchem/data/tests/__init__.py +6 −6 Original line number Diff line number Diff line Loading @@ -23,7 +23,7 @@ def load_solubility_data(): tasks = ["log-solubility"] task_type = "regression" input_file = os.path.join(current_dir, "../../models/tests/example.csv") featurizer = dc.data.DataLoader( featurizer = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) return featurizer.featurize(input_file) Loading @@ -36,7 +36,7 @@ def load_multitask_data(): "task13", "task14", "task15", "task16"] input_file = os.path.join( current_dir, "../../models/tests/multitask_example.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) return loader.featurize(input_file) Loading @@ -48,7 +48,7 @@ def load_classification_data(): task_type = "classification" input_file = os.path.join( current_dir, "../../models/tests/example_classification.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) return loader.featurize(input_file) Loading @@ -61,7 +61,7 @@ def load_sparse_multitask_dataset(): "task7", "task8", "task9"] input_file = os.path.join( current_dir, "../../models/tests/sparse_multitask_example.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) return loader.featurize(input_file) Loading @@ -73,7 +73,7 @@ def load_feat_multitask_data(): tasks = ["task0", "task1", "task2", "task3", "task4", "task5"] input_file = os.path.join( current_dir, "../../models/tests/feat_multitask_example.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, featurizer=featurizer, id_field="id") return loader.featurize(input_file) Loading @@ -87,6 +87,6 @@ def load_gaussian_cdf_data(): tasks = ["task0","task1"] input_file = os.path.join( current_dir, "../../models/tests/gaussian_cdf_example.csv") loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, featurizer=featurizer, id_field="id") return loader.featurize(input_file)
deepchem/data/tests/test_drop.py +4 −3 Original line number Diff line number Diff line Loading @@ -29,12 +29,13 @@ class TestDrop(unittest.TestCase): featurizer = dc.feat.CircularFingerprint(size=1024) emols_tasks = ['activity'] loader = dc.data.DataLoader( tasks=emols_tasks, smiles_field="smiles", featurizer=featurizer) loader = dc.data.CSVLoader( tasks=emols_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file) X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids) ############################################################ DEBUG print("ids.shape, X.shape, y.shape, w.shape") print(ids.shape, X.shape, y.shape, w.shape) ############################################################ DEBUG assert len(X) == len(y) == len(w) == len(ids)
deepchem/data/tests/test_load.py +10 −6 Original line number Diff line number Diff line Loading @@ -32,15 +32,19 @@ class TestLoad(unittest.TestCase): featurizer = dc.feat.CircularFingerprint(size=1024) tasks = ["log-solubility"] loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, data_dir) X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids) shutil.move(data_dir, moved_data_dir) ################################################ DEBUG print("data_dir, moved_data_dir") print(data_dir, moved_data_dir) ################################################ DEBUG moved_dataset = dc.data.DiskDataset( moved_data_dir, reload=True) data_dir=moved_data_dir, reload=True) X_moved, y_moved, w_moved, ids_moved = (moved_dataset.X, moved_dataset.y, moved_dataset.w, moved_dataset.ids) Loading Loading @@ -74,7 +78,7 @@ class TestLoad(unittest.TestCase): all_tasks = ["task%d"%i for i in range(17)] ####### Do featurization loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=all_tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, data_dir) Loading @@ -87,7 +91,7 @@ class TestLoad(unittest.TestCase): y_tasks, w_tasks, = [], [] for ind, task in enumerate(all_tasks): print("Processing task %s" % task) dataset = dc.data.DiskDataset(data_dir, reload=reload) dataset = dc.data.DiskDataset(data_dir=data_dir, reload=reload) X_task, y_task, w_task, ids_task = (dataset.X, dataset.y, dataset.w, dataset.ids) Loading Loading @@ -131,7 +135,7 @@ class TestLoad(unittest.TestCase): tasks = all_tasks[0:n_tasks] ####### Do multitask load loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=tasks, smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, data_dir) Loading @@ -145,7 +149,7 @@ class TestLoad(unittest.TestCase): print("Processing task %s" % task) if os.path.exists(data_dir): shutil.rmtree(data_dir) loader = dc.data.DataLoader( loader = dc.data.CSVLoader( tasks=[task], smiles_field="smiles", featurizer=featurizer) dataset = loader.featurize(dataset_file, data_dir) Loading
