최종 답변을 넘어서: 투명한 Multimodal Reasoning 평가를 위한 CRYSTAL Benchmark

# CRYSTAL 평가 프롬프트 (모델에 이 형식으로 출력 요청)
system_prompt = """
You are a vision-language model. Analyze the provided image(s) and user text silently.
Return ONLY a valid JSON object with this schema:
{"reasoning_steps": [], "answer": ""}

Rules for "reasoning_steps":
- Include enough steps to make the answer evident without filler.
- Write single-clause sentences, each adding a new, directly checkable fact.
- No multi-sentence items. No internal monologue.

Rules for "answer":
- Ground strictly in visible content and given text.
- Multiple-choice: return only the best LETTER (e.g., "B").
- Numeric: include units.
"""

# Match F1 계산 예시 (sentence-transformers 사용)
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

model = SentenceTransformer('all-distilroberta-v1')
THRESHOLD = 0.35

def compute_match_f1(predicted_steps, reference_steps):
    if not predicted_steps and not reference_steps:
        return 1.0
    if not predicted_steps or not reference_steps:
        return 0.0
    
    pred_emb = model.encode(predicted_steps)
    ref_emb = model.encode(reference_steps)
    sim_matrix = cosine_similarity(pred_emb, ref_emb)
    
    # Greedy 1:1 매칭
    matched_pred, matched_ref = set(), set()
    pairs = [(sim_matrix[i,j], i, j) 
             for i in range(len(predicted_steps)) 
             for j in range(len(reference_steps)) 
             if sim_matrix[i,j] >= THRESHOLD]
    pairs.sort(reverse=True)
    
    for score, i, j in pairs:
        if i not in matched_pred and j not in matched_ref:
            matched_pred.add(i)
            matched_ref.add(j)
    
    tp = len(matched_pred)
    precision = tp / max(len(predicted_steps), 1)
    recall = tp / max(len(reference_steps), 1)
    
    if precision + recall == 0:
        return 0.0
    return 2 * precision * recall / (precision + recall)

# CPR (Causal Process Reward) 계산
def compute_cpr(answer_correct, f1_step, aw=0.65, sw=0.35, lambda_penalty=0.3):
    if answer_correct:
        return aw * 1.0 + sw * f1_step
    else:
        return sw * f1_step * lambda_penalty