On Calibration of Large Language Models: From Response To Capability

# Verbalized Confidence prompt (for API-only models)
prompt = """
Question: {question}

How likely are you to answer the question correctly?
You may refer to the following probabilities P:
- 0.0-0.1: "Almost no chance"
- 0.1-0.2: "Highly unlikely"
- 0.2-0.3: "Chances are slight"
- 0.3-0.4: "Unlikely"
- 0.4-0.5: "Less than even"
- 0.5-0.6: "Better than even"
- 0.6-0.7: "Likely"
- 0.7-0.8: "Very good chance"
- 0.8-0.9: "Highly likely"
- 0.9-1.0: "Almost certain"

Reason about your uncertainty and confidence, then provide a probability P between 0.0 and 1.0 in the format of \\boxed{P}.
"""

# Simulate pass@k with capability-calibrated confidence
def simulate_pass_at_k(confidence_scores: list[float], k: int) -> float:
    """Estimate pass@k using capability-calibrated confidence p"""
    pass_at_k_per_instance = [1 - (1 - p) ** k for p in confidence_scores]
    return sum(pass_at_k_per_instance) / len(pass_at_k_per_instance)

# Example: estimated success rate for 10 questions with given confidences at k=5
confidences = [0.9, 0.3, 0.7, 0.5, 0.8, 0.2, 0.6, 0.4, 0.95, 0.1]
print(f"Estimated pass@5: {simulate_pass_at_k(confidences, k=5):.3f}")

# Inference Budget Greedy allocation (Damani et al. 2024 approach)
def greedy_budget_allocation(confidences: list[float], total_budget: int) -> list[int]:
    """Greedy budget allocation based on capability confidence"""
    import heapq
    n = len(confidences)
    allocations = [1] * n  # minimum 1 each
    remaining = total_budget - n
    
    # gain = p * (1-p)^k — expected gain when adding 1 more to current allocation
    heap = []
    for i, p in enumerate(confidences):
        gain = p * (1 - p) ** 1  # initial gain based on k=1
        heapq.heappush(heap, (-gain, i))
    
    for _ in range(remaining):
        neg_gain, i = heapq.heappop(heap)
        allocations[i] += 1
        k = allocations[i]
        p = confidences[i]
        new_gain = p * (1 - p) ** k
        heapq.heappush(heap, (-new_gain, i))
    
    return allocations