Add SHAP Explainability Endpoint

# Add shap to requirements.txt and understand SHAP TreeExplainer

# requirements.txt addition
requirements_addition = "shap==0.45.0\n"

with open("requirements.txt", "a") as f:
    f.write(requirements_addition)
print("shap added to requirements.txt")

# Quick explainer primer — run outside Docker to verify shap is working
import shap
import xgboost as xgb
import numpy as np

# Minimal XGBoost model for illustration
X_demo = np.random.randn(100, 7)
y_demo = X_demo[:, 0] * 10 + X_demo[:, 2] * 5 + np.random.randn(100)
xgb_demo = xgb.XGBRegressor(n_estimators=50, max_depth=3, random_state=42)
xgb_demo.fit(X_demo, y_demo)

# TreeExplainer works directly with the XGBoost Booster
explainer_demo = shap.TreeExplainer(xgb_demo)

# SHAP values for one instance
single_instance = X_demo[[0]]
shap_values_demo = explainer_demo.shap_values(single_instance)

print(f"Base value (expected output): {explainer_demo.expected_value:.4f}")
print(f"SHAP values shape           : {np.array(shap_values_demo).shape}")
print(f"Sum of SHAP values + base   : {explainer_demo.expected_value + shap_values_demo[0].sum():.4f}")
print(f"Model direct prediction     : {xgb_demo.predict(single_instance)[0]:.4f}")
print("Sanity check: SHAP sum + base ≈ model prediction (should be True)")

# api/explainer.py
from typing import List, Dict, Any
import numpy as np
import shap
import mlflow.xgboost

FEATURE_COLS = [
    "batting_average",
    "innings_count",
    "boundary_pct",
    "dot_ball_pct",
    "opponent_bowling_quality",
    "venue_encoded",
    "is_knockout",
]

class IPLStrikeRateExplainer:
    """SHAP TreeExplainer wrapper for the IPL strike-rate XGBoost model."""

    def __init__(self, model_uri: str):
        # Load as native XGBoost model to access the Booster for SHAP
        self._xgb_model = mlflow.xgboost.load_model(model_uri)
        self._explainer = shap.TreeExplainer(self._xgb_model)
        self._base_value = float(self._explainer.expected_value)
        print(f"[explainer] SHAP TreeExplainer initialised. Base value: {self._base_value:.4f}")

    def explain(self, feature_vector: np.ndarray) -> Dict[str, Any]:
        """
        Compute SHAP attributions for a single prediction.
        Args:
            feature_vector: shape (1, n_features) NumPy array
        Returns:
            dict with base_value, predicted_strike_rate, and feature_attributions
        """
        if feature_vector.shape[0] != 1:
            raise ValueError("explain() expects a single-row feature vector.")

        # Compute SHAP values — shape (1, n_features) for regression
        ipl_shap_values = self._explainer.shap_values(feature_vector)
        shap_row = ipl_shap_values[0]  # (n_features,)

        # Reconstruct prediction from SHAP decomposition
        predicted_strike_rate = self._base_value + float(shap_row.sum())

        # Build attribution list sorted by absolute SHAP value (most impactful first)
        attributions = sorted(
            [
                {
                    "feature": FEATURE_COLS[i],
                    "shap_value": round(float(shap_row[i]), 4),
                    "abs_impact": abs(float(shap_row[i])),
                }
                for i in range(len(FEATURE_COLS))
            ],
            key=lambda x: x["abs_impact"],
            reverse=True,
        )
        # Remove helper field before returning
        for attr in attributions:
            del attr["abs_impact"]

        return {
            "base_value": round(self._base_value, 4),
            "predicted_strike_rate": round(predicted_strike_rate, 2),
            "feature_attributions": attributions,
        }

# Updated api/schemas.py — add explanation response models
from pydantic import BaseModel, Field
from typing import List, Optional

# (existing schemas omitted for brevity — keep CricketPlayerInput, PredictionResponse, HealthResponse)

class FeatureAttribution(BaseModel):
    """SHAP attribution for a single feature."""
    feature: str = Field(..., description="Feature name")
    shap_value: float = Field(
        ...,
        description="SHAP value: positive means feature pushed prediction up, negative means it pushed it down"
    )

class ExplanationResponse(BaseModel):
    """SHAP explanation response for a single IPL player prediction."""
    base_value: float = Field(
        ...,
        description="Expected model output (average strike rate across training data)"
    )
    predicted_strike_rate: float = Field(
        ...,
        description="Final predicted strike rate (base_value + sum of all SHAP values)"
    )
    feature_attributions: List[FeatureAttribution] = Field(
        ...,
        description="Per-feature SHAP attributions sorted by absolute impact (highest first)"
    )
    model_version: str


# Updated api/main.py — /explain route addition (patch to existing file)
explain_route_patch = '''
from api.explainer import IPLStrikeRateExplainer
from api.schemas import ExplanationResponse, FeatureAttribution

# Add to app_state dict in lifespan:
#   app_state["explainer"] = IPLStrikeRateExplainer(model_uri)

@app.post("/explain", response_model=ExplanationResponse)
async def explain(player_input: CricketPlayerInput) -> ExplanationResponse:
    if app_state.get("explainer") is None:
        raise HTTPException(status_code=503, detail="Explainer not loaded")

    feature_vector = np.array([[
        getattr(player_input, col) for col in FEATURE_COLS
    ]])

    explanation = app_state["explainer"].explain(feature_vector)

    return ExplanationResponse(
        base_value=explanation["base_value"],
        predicted_strike_rate=explanation["predicted_strike_rate"],
        feature_attributions=[
            FeatureAttribution(**attr)
            for attr in explanation["feature_attributions"]
        ],
        model_version=app_state["model_version"],
    )
'''
print(explain_route_patch)

# test_explain.py — verify SHAP endpoint correctness
import httpx
import pytest

# Rohit Sharma — high boundary%, low dot% — expect high strike rate
rohit_ipl_features = {
    "batting_average": 31.17,
    "innings_count": 243,
    "boundary_pct": 0.60,   # high — positive driver
    "dot_ball_pct": 0.25,   # low — less negative drag
    "opponent_bowling_quality": 6.5,
    "venue_encoded": 5,
    "is_knockout": 0,
}

# MS Dhoni finisher profile — lower boundary%, higher dot%
dhoni_ipl_features = {
    "batting_average": 28.5,
    "innings_count": 210,
    "boundary_pct": 0.41,
    "dot_ball_pct": 0.44,
    "opponent_bowling_quality": 8.1,
    "venue_encoded": 2,
    "is_knockout": 1,
}

def test_explain_shap_consistency():
    """SHAP sum + base_value must equal predicted_strike_rate."""
    with httpx.Client(base_url="http://localhost:8000") as client:
        resp = client.post("/explain", json=rohit_ipl_features)
    assert resp.status_code == 200
    body = resp.json()

    base_value   = body["base_value"]
    predicted_sr = body["predicted_strike_rate"]
    shap_sum     = sum(attr["shap_value"] for attr in body["feature_attributions"])

    # Allow 0.01 tolerance for float rounding
    assert abs(base_value + shap_sum - predicted_sr) < 0.01, (
        f"SHAP consistency failed: {base_value} + {shap_sum} != {predicted_sr}"
    )
    print(f"[PASS] SHAP consistency: {base_value:.4f} + {shap_sum:.4f} = {predicted_sr:.4f}")

def test_explain_boundary_pct_positive_attribution():
    """High boundary_pct should have positive SHAP value for Rohit's input."""
    with httpx.Client(base_url="http://localhost:8000") as client:
        resp = client.post("/explain", json=rohit_ipl_features)
    body = resp.json()
    attributions = {a["feature"]: a["shap_value"] for a in body["feature_attributions"]}
    assert attributions["boundary_pct"] > 0, (
        f"Expected positive SHAP for boundary_pct, got {attributions['boundary_pct']}"
    )
    print(f"[PASS] boundary_pct SHAP = {attributions['boundary_pct']:.4f} (positive as expected)")

def test_explain_returns_all_features():
    """Response must contain SHAP values for all 7 features."""
    with httpx.Client(base_url="http://localhost:8000") as client:
        resp = client.post("/explain", json=rohit_ipl_features)
    body = resp.json()
    feature_names = [a["feature"] for a in body["feature_attributions"]]
    expected_features = [
        "batting_average", "innings_count", "boundary_pct",
        "dot_ball_pct", "opponent_bowling_quality", "venue_encoded", "is_knockout"
    ]
    for f in expected_features:
        assert f in feature_names, f"Missing feature '{f}' in SHAP response"
    print(f"[PASS] All 7 features present in SHAP response")

if __name__ == "__main__":
    test_explain_shap_consistency()
    test_explain_boundary_pct_positive_attribution()
    test_explain_returns_all_features()
    print("\nAll SHAP endpoint tests PASSED.")

# curl spot-check for /explain
spot_check_cmd = """
# Rebuild container with SHAP dependency
docker compose up --build -d

# Test /explain endpoint with Rohit Sharma's profile
curl -s -X POST http://localhost:8000/explain \\
  -H 'Content-Type: application/json' \\
  -d '{
    "batting_average": 31.17,
    "innings_count": 243,
    "boundary_pct": 0.60,
    "dot_ball_pct": 0.25,
    "opponent_bowling_quality": 6.5,
    "venue_encoded": 5,
    "is_knockout": 0
  }' | python3 -m json.tool

# Expected response structure:
# {
#   "base_value": 132.18,
#   "predicted_strike_rate": 154.32,
#   "feature_attributions": [
#     {"feature": "boundary_pct",  "shap_value": 17.84},
#     {"feature": "dot_ball_pct",  "shap_value": -9.12},
#     {"feature": "batting_average","shap_value": 4.67},
#     ... (remaining features sorted by |shap_value|)
#   ],
#   "model_version": "1"
# }

# Run pytest suite
pytest test_explain.py -v
"""
print(spot_check_cmd)