mirror of
https://github.com/QData/TextAttack.git
synced 2021-10-13 00:05:06 +03:00
12 lines
339 B
Python
12 lines
339 B
Python
from textattack.models.tokenizers import AutoTokenizer
|
|
|
|
|
|
class BERTTokenizer(AutoTokenizer):
|
|
"""
|
|
A generic class that convert text to tokens and tokens to IDs. Intended
|
|
for fine-tuned BERT models.
|
|
"""
|
|
|
|
def __init__(self, name="bert-base-uncased", max_length=256):
|
|
super().__init__(name, max_length=max_length)
|