p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix test

This commit is contained in:
hiyouga 2024-07-14 15:44:30 +08:00
parent 88a20ba797
commit 173921419d
2 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/tests.yml vendored
View File

@ -20,6 +20,12 @@ jobs:
tests: tests:
runs-on: ubuntu-latest runs-on: ubuntu-latest
environment:
name: tests
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4

2
tests/data/test_template.py
View File

@ -44,7 +44,7 @@ def _check_tokenization(
def _check_single_template(model_id: str, template_name: str, prompt_str: str, answer_str: str, use_fast: bool): def _check_single_template(model_id: str, template_name: str, prompt_str: str, answer_str: str, use_fast: bool):
tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=use_fast) tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=use_fast, token=os.environ.get("HF_TOKEN", None))
content_str = tokenizer.apply_chat_template(MESSAGES, tokenize=False).rstrip("\n") # avoid extra newline content_str = tokenizer.apply_chat_template(MESSAGES, tokenize=False).rstrip("\n") # avoid extra newline
content_ids = tokenizer.encode(content_str, add_special_tokens=False) content_ids = tokenizer.encode(content_str, add_special_tokens=False)
template = get_template_and_fix_tokenizer(tokenizer, name=template_name) template = get_template_and_fix_tokenizer(tokenizer, name=template_name)