Update README.md

README.md

@@ -22,6 +22,11 @@ To run:
     tok = AutoTokenizer.from_pretrained('alex2awesome/stance-detection-t5')
     model = T5ForConditionalGeneration.from_pretrained('alex2awesome/stance-detection-t5')
 
+    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
+    tokenizer_with_prefix_space = AutoTokenizer.from_pretrained(model_name_or_path, add_prefix_space=True)
+    def get_tokens_as_tuple(word):
+        return tuple(tokenizer_with_prefix_space([word], add_special_tokens=False).input_ids[0])
+
     input_ids = tokenizer(text, return_tensors="pt").input_ids
     y_pred_gen_output = model.generate(
         input_ids,