【備忘録】Gradioから llm-jp-3-13b-instruct3 を使おうとしたら ['token_type_ids'] に関するエラーがでたのでその解決策を書きます。

エラー内容

    raise ValueError(
ValueError: The following `model_kwargs` are not used by the model: ['token_type_ids'] (note: typos in the generate arguments will also show up in this list)

解決策

# token_type_idsを削除
if "token_type_ids" in input_tensors:
    del input_tensors["token_type_ids"]

最終的なスクリプト

import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
from threading import Thread
import torch

model_name = "llm-jp/llm-jp-3-13b-instruct3"

system_prompt_text = "以下は、タスクを説明する指示です。要求を適切に満たす応答を書きなさい。"
init = {
    "role": "system",
    "content": system_prompt_text
}

quantization_config = BitsAndBytesConfig(load_in_4bit=True)

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    quantization_config=quantization_config,
    device_map="auto"
)

tokenizer = AutoTokenizer.from_pretrained(model_name)
streamer = TextIteratorStreamer(
    tokenizer,
    skip_prompt=True,
    skip_special_tokens=True
)

def user(
    message: str,
    history: list[dict]
):
    if len(history) == 0:
        history.insert(0, init)
    history.append(
        {
            "role": "user", 
            "content": message
        }
    )
    return "", history

def bot(
    history: list[dict]
):
    input_tensors = tokenizer.apply_chat_template(
        history,
        add_generation_prompt=True,
        return_tensors="pt",
        return_dict=True,
    ).to(model.device)

    # token_type_idsを削除
    if "token_type_ids" in input_tensors:
        del input_tensors["token_type_ids"]

    generation_kwargs = dict(
        **input_tensors,
        streamer=streamer,
        max_new_tokens=512,
        do_sample=True,
        temperature=0.7,
        top_p=0.95,
        repetition_penalty=1.05,
        pad_token_id=tokenizer.eos_token_id,
    )
    history.append({"role": "assistant", "content": ""})

    thread = Thread(target=model.generate, kwargs=generation_kwargs)
    thread.start()

    for new_text in streamer:
        history[-1]["content"] += new_text
        yield history

with gr.Blocks() as demo:
    gr.Markdown("# llm-jp/llm-jp-3-13b-instruct3")
    chatbot = gr.Chatbot(
        type="messages",
        latex_delimiters = [
            {"left": "[", "right": "\\]", "display": True }
        ]
    )
    msg = gr.Textbox()
    clear = gr.ClearButton([msg, chatbot], value="新しいチャットを開始")
    
    msg.submit(
        user, [msg, chatbot], [msg, chatbot], queue=False
    ).then(
        bot, chatbot, chatbot
    )

demo.launch()