Commit 3fd77316 authored by seyonechithrananda's avatar seyonechithrananda
Browse files

use os to get vocab path

parent 2df051c6
Loading
Loading
Loading
Loading
+5 −3
Original line number Diff line number Diff line
# Requriments - transformers, tokenizers

# Requirements - transformers, tokenizers
import os
from unittest import TestCase
from deepchem.feat.smiles_tokenizer import SmilesTokenizer
from transformers import RobertaForMaskedLM
@@ -10,9 +10,11 @@ class TestSmilesTokenizer(TestCase):


  def test_tokenize(self):
      vocab_path = os.path.join(os.path.dirname(__file__), "data", "vocab.txt")

      model = RobertaForMaskedLM.from_pretrained('seyonec/SMILES_tokenized_PubChem_shard00_50k')
      model.num_parameters()

      tokenizer = SmilesTokenizer('deepchem/feat/tests/data/vocab.txt', max_len=model.config.max_position_embeddings)
      tokenizer = SmilesTokenizer(vocab_path, max_len=model.config.max_position_embeddings)
      print(tokenizer.encode("CCC(CC)COC(=O)[C@H](C)N[P@](=O)(OC[C@H]1O[C@](C#N)([C@H](O)[C@@H]1O)C1=CC=C2N1N=CN=C2N)OC1=CC=CC=C1"))