Just-In-Time Reinforcement Learning: Gradient 업데이트 없이 LLM 에이전트의 지속 학습

# JitRL 핵심 로직 스니펫 (Python 의사코드)
import numpy as np
from collections import defaultdict

class JitRLMemory:
    def __init__(self):
        self.memory = []  # List of (state, action, G_t)
    
    def jaccard_similarity(self, s1_tokens: set, s2_tokens: set) -> float:
        if not s1_tokens and not s2_tokens:
            return 1.0
        return len(s1_tokens & s2_tokens) / len(s1_tokens | s2_tokens)
    
    def retrieve_neighbors(self, state: str, k: int = 10) -> list:
        state_tokens = set(state.split())
        scored = [
            (self.jaccard_similarity(state_tokens, set(s.split())), s, a, g)
            for s, a, g in self.memory
        ]
        scored.sort(reverse=True)
        return scored[:k]
    
    def update_logits(
        self,
        state: str,
        candidate_actions: list,
        base_logits: dict,  # {action: logit}
        beta: float = 1.0,
        k: int = 10
    ) -> dict:
        neighbors = self.retrieve_neighbors(state, k)
        
        # State value (baseline)
        V_s = np.mean([g for _, _, _, g in neighbors]) if neighbors else 0.0
        
        updated_logits = {}
        for action in candidate_actions:
            # Action value from matching neighbors
            matching = [g for _, _, a, g in neighbors if a == action]
            if matching:
                Q_sa = np.mean(matching)
            else:
                # Exploration bonus for unseen actions
                Q_sa = V_s + 5.0 / (len(neighbors) + 1e-8)
            
            advantage = Q_sa - V_s
            base = base_logits.get(action, 0.0)
            updated_logits[action] = base + beta * advantage
        
        return updated_logits
    
    def add_trajectory(self, trajectory: list, gamma: float = 0.9):
        # trajectory: [(state, action, reward), ...]
        T = len(trajectory)
        for t, (s, a, r) in enumerate(trajectory):
            G_t = sum(gamma**(u-t) * trajectory[u][2] for u in range(t, T))
            self.memory.append((s, a, G_t))

# 사용 예시
memory = JitRLMemory()

# 추론 시
state = "page:shopping/cart filter:price_asc"
candidates = ["click(checkout)", "click(continue_shopping)", "click(apply_coupon)"]
base_logits = {"click(checkout)": 1.2, "click(continue_shopping)": 0.8, "click(apply_coupon)": 0.5}

updated = memory.update_logits(state, candidates, base_logits)
best_action = max(updated, key=updated.get)
print(f"Best action: {best_action}")

# 에피소드 후 메모리 업데이트
traj = [(state, best_action, 1.0)]  # (state, action, reward)
memory.add_trajectory(traj)