MMRC: 실제 대화 환경에서 Multimodal Large Language Model을 평가하는 대규모 Benchmark

# NOTE-TAKING 전략 구현 예시

SYSTEM_PROMPT_NOTETAKER = """
You are an intelligent chatbot designed to accurately record information from conversations in JSON format.
- Focus on the user's preferences and the facts within the conversation.
- If the user presents a new fact, it should overwrite the outdated fact.
- REMEMBER NOT TO RESPOND TO THE USER'S INPUT!
"""

USER_PROMPT_NOTETAKER = """
Extract key information from the input, including the user's preferences, events, and facts.
If you detect a change in information, overwrite the outdated content.

Here is the user input: {user_input}

Provide the Note in JSON format, for example:
{{
  "user_info": {{
    "health_conditions": ["High blood pressure", "Dairy allergy"]
  }},
  "preferences": {{
    "food_preference": "Italian cuisine"
  }},
  "purpose": "Seeking meal suggestions."
}}
"""

def take_note(client, user_input, current_note):
    """유저 메시지에서 핵심 정보를 추출해 노트 업데이트"""
    prompt = USER_PROMPT_NOTETAKER.format(user_input=user_input)
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT_NOTETAKER},
            {"role": "user", "content": f"Current note: {current_note}\n\n{prompt}"}
        ]
    )
    return response.choices[0].message.content

def chat_with_note(client, history, question, note):
    """노트를 컨텍스트로 포함해 응답 생성"""
    augmented_history = history + [
        {"role": "system", "content": f"[Conversation Notes]\n{note}"}
    ]
    augmented_history.append({"role": "user", "content": question})
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=augmented_history
    )
    return response.choices[0].message.content

# 사용 예시
note = {}
history = []
for user_message in conversation_turns:
    # 1. 노트 업데이트
    note = take_note(client, user_message, note)
    # 2. 응답 생성
    response = chat_with_note(client, history, user_message, note)
    history.append({"role": "user", "content": user_message})
    history.append({"role": "assistant", "content": response})