always manually add special tokens when loading relation extraction dataset

Aethor · Aethor · commit 2d295edc8319 · 2026-01-07T23:43:27.000+08:00
diff --git a/renard/pipeline/relation_extraction.py b/renard/pipeline/relation_extraction.py
@@ -39,6 +39,7 @@ def format_rel(rel: dict) -> str:
     batch = tokenizer(
         tokenizer.bos_token + GenerativeRelationExtractor.task_prompt(text),
         text_target=labels + tokenizer.eos_token,
+        add_special_tokens=False,
     )
     batch["relations"] = relations
 

Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,7 @@ def format_rel(rel: dict) -> str:`
`39`	`39`	`batch = tokenizer(`
`40`	`40`	`tokenizer.bos_token + GenerativeRelationExtractor.task_prompt(text),`
`41`	`41`	`text_target=labels + tokenizer.eos_token,`
	`42`	`+ add_special_tokens=False,`
`42`	`43`	`)`
`43`	`44`	`batch["relations"] = relations`
`44`	`45`