yapf (7cc13d53) · Commits · 钟慕尧 / deepchem

deepchem/data/data_loader.py

+20 −14

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ from deepchem.feat.molecule_featurizers import OneHotFeaturizer

		logger = logging.getLogger(__name__)


		def _convert_df_to_numpy(df: pd.DataFrame,
		tasks: List[str]) -> Tuple[np.ndarray, np.ndarray]:
		"""Transforms a dataframe containing deepchem input into numpy arrays
		@@ -488,8 +489,8 @@ class UserCSVLoader(CSVLoader):
		shard[feature_fields] = shard[feature_fields].apply(pd.to_numeric)
		X_shard = shard[feature_fields].to_numpy()
		time2 = time.time()
		logger.info(
		"TIMING: user specified processing took %0.3f s" % (time2 - time1))
		logger.info("TIMING: user specified processing took %0.3f s" %
		(time2 - time1))
		return (X_shard, np.ones(len(X_shard), dtype=bool))


		@@ -831,8 +832,7 @@ class SDFLoader(DataLoader):
		Iterator[pd.DataFrame]
		Iterator over shards
		"""
		return load_sdf_files(
		input_files=input_files,
		return load_sdf_files(input_files=input_files,
		clean_mols=self.sanitize,
		tasks=self.tasks,
		shard_size=shard_size)
		@@ -875,7 +875,9 @@ class FASTALoader(DataLoader):
		learning tasks.
		"""

		def __init__(self, featurizer: Featurizer = OneHotFeaturizer, charset: str = "ATCGN"):
		def __init__(self,
		featurizer: Featurizer = OneHotFeaturizer,
		charset: str = "ATCGN"):
		"""Initialize FASTALoader.

		Parameters
		@@ -898,8 +900,8 @@ class FASTALoader(DataLoader):
		"protein": ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L',
		'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
		'Y', 'Z', '*', '-'),
		"nucleic": ('A', 'C', 'G', 'T', 'U', '(i)', 'R', 'Y', 'K', 'M', 'S', 'W',
		'B', 'D', 'H', 'V', 'N', '-'),
		"nucleic": ('A', 'C', 'G', 'T', 'U', '(i)', 'R', 'Y', 'K', 'M', 'S',
		'W', 'B', 'D', 'H', 'V', 'N', '-'),
		"ATCGN": ('A', 'T', 'C', 'G', 'N')
		}

		@@ -963,6 +965,7 @@ class FASTALoader(DataLoader):
		"""
		Convert the FASTA file to a numpy array of FASTA-format strings.
		"""

		def _generate_sequences(fasta_file, header_mark=">") -> np.array:
		"""
		Uses a fasta_file to create a numpy array of annotated FASTA-format strings
		@@ -986,7 +989,8 @@ class FASTALoader(DataLoader):
		def _add_sequence(sequences: np.array, sequence: list) -> np.array:
		# Handle empty sequence
		if sequence == None or len(sequence) <= 0:
		logger.warning("Attempting to add empty sequence, returning empty array...")
		logger.warning(
		"Attempting to add empty sequence, returning empty array...")
		return np.array([])
		# Annotate start/stop of sequence
		if auto_add_annotations:
		@@ -1000,6 +1004,7 @@ class FASTALoader(DataLoader):

		return DiskDataset.create_dataset(shard_generator(), data_dir)


		class ImageLoader(DataLoader):
		"""Handles loading of image files.

		@@ -1115,11 +1120,12 @@ class ImageLoader(DataLoader):

		if in_memory:
		if data_dir is None:
		return NumpyDataset(
		load_image_files(image_files), y=labels, w=weights, ids=image_files)
		return NumpyDataset(load_image_files(image_files),
		y=labels,
		w=weights,
		ids=image_files)
		else:
		dataset = DiskDataset.from_numpy(
		load_image_files(image_files),
		dataset = DiskDataset.from_numpy(load_image_files(image_files),
		y=labels,
		w=weights,
		ids=image_files,
		@@ -1268,8 +1274,8 @@ class InMemoryLoader(DataLoader):

		# FIXME: Signature of "_featurize_shard" incompatible with supertype "DataLoader"
		def _featurize_shard( # type: ignore[override]
		self, shard: List, global_index: int
		) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
		self, shard: List, global_index: int) -> Tuple[np.ndarray, np.ndarray,
		np.ndarray, np.ndarray]:
		"""Featurizes a shard of an input data.

		Parameters

Admin message