#!/usr/bin/env python3
# Source: q4-ml-coding.html
import numpy as np


def log_softmax(z: np.ndarray, axis: int = -1) -> np.ndarray:
    """log softmax with log-sum-exp. Subtracting the max makes exp safe."""
    m = z.max(axis=axis, keepdims=True)
    lse = m + np.log(np.exp(z - m).sum(axis=axis, keepdims=True))
    return z - lse


def softmax(z: np.ndarray, axis: int = -1) -> np.ndarray:
    e = np.exp(z - z.max(axis=axis, keepdims=True))
    return e / e.sum(axis=axis, keepdims=True)


def cross_entropy(logits: np.ndarray, y: np.ndarray) -> tuple[float, np.ndarray]:
    """Mean CE over a batch and its gradient w.r.t. the logits.

    logits: (n, C). y: (n,) integer class ids. Returns (loss, dlogits (n, C)).
    """
    n = logits.shape[0]
    logp = log_softmax(logits, axis=1)
    loss = -logp[np.arange(n), y].mean()
    grad = np.exp(logp)                      # softmax probabilities
    grad[np.arange(n), y] -= 1.0             # softmax - onehot
    return float(loss), grad / n             # divide by n because loss is a mean


if __name__ == "__main__":
    z = np.array([[1000.0, 1001.0, 1002.0], [-1000.0, 0.0, 1000.0]])
    p = softmax(z)
    assert np.all(np.isfinite(p)) and np.allclose(p.sum(1), 1.0)
    rng = np.random.default_rng(3)
    L = rng.normal(size=(4, 5))
    y = np.array([0, 3, 1, 4])
    loss, g = cross_entropy(L, y)
    eps, num = 1e-6, np.zeros_like(L)
    for i in range(L.shape[0]):
        for j in range(L.shape[1]):
            Lp, Lm = L.copy(), L.copy()
            Lp[i, j] += eps
            Lm[i, j] -= eps
            num[i, j] = (cross_entropy(Lp, y)[0] - cross_entropy(Lm, y)[0]) / (2 * eps)
    assert np.allclose(g, num, atol=1e-7)
    print("ok loss", round(loss, 4))
