add load_vocab method

        """

        tokens = [token for token in self.regex.findall(text)]

        return tokens

def load_vocab(vocab_file):

    """Loads a vocabulary file into a dictionary."""

    vocab = collections.OrderedDict()

    with open(vocab_file, "r", encoding="utf-8") as reader:

        tokens = reader.readlines()

    for index, token in enumerate(tokens):

        token = token.rstrip("\n")

        vocab[token] = index

    return vocab

 No newline at end of file