Fix Issue #39030: AutoTokenizer.from_pretrained does not propagate token (#40956)

* fix merge conflicts

* change token typing

---------

Co-authored-by: Ubuntu <ubuntu@ip-172-31-27-253.ec2.internal>

Author: brandenkmurray

src/transformers/processing_utils.py

@@ -963,6 +963,7 @@ def get_processor_dict(
                         local_files_only=local_files_only,
                         revision=revision,
                         cache_dir=cache_dir,
+                        token=token,
                     ):
                         additional_chat_template_files[template] = f"{CHAT_TEMPLATE_DIR}/{template}.jinja"
                 except EntryNotFoundError:

src/transformers/tokenization_utils_base.py

@@ -2042,6 +2042,7 @@ def from_pretrained(
                             local_files_only=local_files_only,
                             revision=revision,
                             cache_dir=cache_dir,
+                            token=token,
                         ):
                             template = template.removesuffix(".jinja")
                             vocab_files[f"chat_template_{template}"] = f"{CHAT_TEMPLATE_DIR}/{template}.jinja"

src/transformers/utils/hub.py

@@ -155,6 +155,7 @@ def list_repo_templates(
     local_files_only: bool,
     revision: Optional[str] = None,
     cache_dir: Optional[str] = None,
+    token: Union[bool, str, None] = None,
 ) -> list[str]:
     """List template files from a repo.
 
@@ -171,6 +172,7 @@ def list_repo_templates(
                     revision=revision,
                     path_in_repo=CHAT_TEMPLATE_DIR,
                     recursive=False,
+                    token=token,
                 )
                 if entry.path.endswith(".jinja")
             ]