MADQA: Multimodal Agent가 PDF 문서 컬렉션을 탐색하는 방식 — 전략적 추론인가, 무작위 검색인가?

# BM25 MLLM Agent 핵심 루프 (논문 Algorithm 1 기반 Python 슈도코드)

from whoosh import index, qparser
from PIL import Image
import base64

SYSTEM_PROMPT = """
You are a document QA assistant with access to a search tool.
The answer is definitely in the documents.
If search returns no results, try different terms (synonyms, abbreviations, rephrasing).

Once relevant pages are found, provide:
1. answer: List of short answer values (exact document words preferred)
2. citations: List of {file, page} dicts
"""

def bm25_agent(question: str, search_index, vlm_client, max_steps=10, top_k=5):
    messages = [{"role": "system", "content": SYSTEM_PROMPT},
                {"role": "user", "content": question}]
    
    tools = [{
        "name": "search_documents",
        "description": "Search document collection. Supports boolean ops (AND/OR/NOT), phrases in quotes, wildcards (*). Example: '\"annual report\" AND revenue'",
        "parameters": {"query": {"type": "string"}}
    }]
    
    for step in range(max_steps):
        force_answer = (step == max_steps - 1)
        
        response = vlm_client.chat(
            messages=messages,
            tools=None if force_answer else tools,
            # force structured answer output on last step
        )
        
        if response.type == "answer":
            return response.answer, response.citations
        
        elif response.type == "tool_call" and response.tool == "search_documents":
            query = response.args["query"]
            
            # BM25 검색 후 페이지 이미지로 변환
            results = search_index.search(query, limit=top_k)  # (file, page) 튜플 반환
            page_images = [render_page_as_image(file, page) for file, page in results]
            
            # 이미지를 메시지에 추가 (VLM이 시각적으로 분석)
            tool_result = {"role": "tool", "content": [
                {"type": "image", "data": img_to_base64(img)} for img in page_images
            ]}
            messages.append(tool_result)
    
    return fallback_answer(messages)

# 핵심 포인트: 검색 실패 시 과감하게 다른 쿼리 사용
# Claude Sonnet 4.5의 평균 쿼리 드리프트 0.38이 성능의 핵심
# GPT-4.1 Nano처럼 비슷한 쿼리만 반복하면(drift 0.10) 성능 저하