LLMRouterBench: LLM 라우팅을 위한 대규모 벤치마크 및 통합 프레임워크

# Avengers-Pro 스타일 클러스터링 라우터 (간략 구현)
from sklearn.cluster import KMeans
from sentence_transformers import SentenceTransformer
import numpy as np

class ClusteringRouter:
    def __init__(self, model_pool, k=64, embed_model='all-MiniLM-L6-v2'):
        self.model_pool = model_pool  # {'gpt-5': client_gpt5, 'qwen3-235b': client_qwen, ...}
        self.k = k
        self.embedder = SentenceTransformer(embed_model)  # 임베딩 모델 품질은 큰 영향 없음
        self.kmeans = None
        self.cluster_to_model = {}  # 클러스터별 최적 모델 매핑

    def fit(self, train_queries, train_labels):
        """train_labels: {query_i: {'gpt-5': correct, 'qwen3-235b': correct, ...}}"""
        embeddings = self.embedder.encode(train_queries)
        self.kmeans = KMeans(n_clusters=self.k, random_state=42)
        cluster_ids = self.kmeans.fit_predict(embeddings)

        # 각 클러스터에서 정확도 가장 높은 모델 선택
        for c in range(self.k):
            idxs = np.where(cluster_ids == c)[0]
            scores = {m: np.mean([train_labels[i].get(m, 0) for i in idxs])
                      for m in self.model_pool}
            self.cluster_to_model[c] = max(scores, key=scores.get)

    def route(self, query, alpha=0.7):
        """alpha: 성능 계수 (1.0=성능 최우선, 0.0=비용 최우선)"""
        emb = self.embedder.encode([query])
        cluster = self.kmeans.predict(emb)[0]
        return self.cluster_to_model[cluster]

# 사용 예시
# router = ClusteringRouter(model_pool={'gpt-5': ..., 'qwen3-235b': ..., 'gemini-flash': ...})
# router.fit(train_queries, train_labels)
# best_model = router.route('Prove that sqrt(2) is irrational')