diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 73d77de5..73117c3b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,6 +20,12 @@ jobs: tests: runs-on: ubuntu-latest + environment: + name: tests + + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + steps: - name: Checkout uses: actions/checkout@v4 diff --git a/tests/data/test_template.py b/tests/data/test_template.py index fa82973b..ceb8acc4 100644 --- a/tests/data/test_template.py +++ b/tests/data/test_template.py @@ -44,7 +44,7 @@ def _check_tokenization( def _check_single_template(model_id: str, template_name: str, prompt_str: str, answer_str: str, use_fast: bool): - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=use_fast) + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=use_fast, token=os.environ.get("HF_TOKEN", None)) content_str = tokenizer.apply_chat_template(MESSAGES, tokenize=False).rstrip("\n") # avoid extra newline content_ids = tokenizer.encode(content_str, add_special_tokens=False) template = get_template_and_fix_tokenizer(tokenizer, name=template_name)