yapf fixes to SmilesTokenizer (854f8c31) · Commits · 钟慕尧 / deepchem

deepchem/feat/smiles_tokenizer.py

+23 −32

Original line number	Diff line number	Diff line
		@@ -17,9 +17,6 @@ try:
		except ModuleNotFoundError:
		logger.warning(
		"HuggingFace transformers is not available. Please install using 'pip install transformers' to use the SmilesTokenizer")



		"""
		SMI_REGEX_PATTERN: str
		SMILES regex pattern for tokenization. Designed by Schwaller et. al.
		@@ -37,14 +34,11 @@ SMI_REGEX_PATTERN = r"""(\[[^\]]+]\|Br?\|Cl?\|N\|O\|S\|P\|F\|I\|b\|c\|n\|o\|s\|p\|\(\|\)\|\.\|=\|


		def get_default_tokenizer():
		default_vocab_path = (
		pkg_resources.resource_filename(
		"deepchem",
		"feat/tests/vocab.txt"
		)
		)
		default_vocab_path = (pkg_resources.resource_filename("deepchem",
		"feat/tests/vocab.txt"))
		return SmilesTokenizer(default_vocab_path)


		class SmilesTokenizer(BertTokenizer):
		"""
		Creates the SmilesTokenizer class. The tokenizer heavily inherits from the BERT
		@@ -85,9 +79,7 @@ class SmilesTokenizer(BertTokenizer):
		# pad_token="[PAD]",
		# cls_token="[CLS]",
		# mask_token="[MASK]",
		**kwargs
		):

		**kwargs):
		"""Constructs a SmilesTokenizer.

		Parameters
		@@ -103,19 +95,14 @@ class SmilesTokenizer(BertTokenizer):
		self.max_len_sentences_pair = self.max_len - 3

		if not os.path.isfile(vocab_file):
		raise ValueError(
		"Can't find a vocab file at path '{}'.".format(vocab_file)
		)
		raise ValueError("Can't find a vocab file at path '{}'.".format(
		vocab_file))
		self.vocab = load_vocab(vocab_file)
		self.highest_unused_index = max(
		[
		i for i, v in enumerate(self.vocab.keys())
		if v.startswith("[unused")
		]
		)
		self.highest_unused_index = max([
		i for i, v in enumerate(self.vocab.keys()) if v.startswith("[unused")
		])
		self.ids_to_tokens = collections.OrderedDict(
		[(ids, tok) for tok, ids in self.vocab.items()]
		)
		[(ids, tok) for tok, ids in self.vocab.items()])
		self.basic_tokenizer = BasicSmilesTokenizer()
		self.init_kwargs["max_len"] = self.max_len

		@@ -234,7 +221,8 @@ class SmilesTokenizer(BertTokenizer):

		return sequence_pair

		def add_special_tokens_ids_sequence_pair(self, token_ids_0 : List[int], token_ids_1: List[int]) -> List[int]:
		def add_special_tokens_ids_sequence_pair(self, token_ids_0: List[int],
		token_ids_1: List[int]) -> List[int]:
		"""
		Adds special tokens to a sequence pair for sequence classification tasks.
		A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP]
		@@ -253,7 +241,10 @@ class SmilesTokenizer(BertTokenizer):

		return cls + token_ids_0 + sep + token_ids_1 + sep

		def add_padding_tokens(self, token_ids: List[int], length: int, right: bool=True) -> List[int]:
		def add_padding_tokens(self,
		token_ids: List[int],
		length: int,
		right: bool=True) -> List[int]:
		"""
		Adds padding tokens to return a sequence of length max_length.
		By default padding tokens are added to the right of the sequence.
		@@ -283,7 +274,9 @@ class SmilesTokenizer(BertTokenizer):
		else:
		return padding + token_ids

		def save_vocabulary(self, vocab_path: str): # -> tuple[str]: doctest issue raised with this return type annotation
		def save_vocabulary(
		self, vocab_path: str
		): # -> tuple[str]: doctest issue raised with this return type annotation
		"""
		Save the tokenizer vocabulary to a file.

		@@ -305,21 +298,19 @@ class SmilesTokenizer(BertTokenizer):
		vocab_file = vocab_path
		with open(vocab_file, "w", encoding="utf-8") as writer:
		for token, token_index in sorted(
		self.vocab.items(), key=lambda kv: kv[1]
		):
		self.vocab.items(), key=lambda kv: kv[1]):
		if index != token_index:
		logger.warning(
		"Saving vocabulary to {}: vocabulary indices are not consecutive."
		" Please check that the vocabulary is not corrupted!".
		format(vocab_file)
		)
		" Please check that the vocabulary is not corrupted!".format(
		vocab_file))
		index = token_index
		writer.write(token + u"\n")
		index += 1
		return (vocab_file,)

		class BasicSmilesTokenizer(object):

		class BasicSmilesTokenizer(object):
		"""

		Run basic SMILES tokenization using a regex pattern developed by Schwaller et. al. This tokenizer is to be used

Admin message