#!/usr/bin/env python3
# Source: t3-probability-information.html
import numpy as np


def kl(p: np.ndarray, q: np.ndarray) -> float:
    """KL(p || q) in nats. Assumes q > 0 wherever p > 0."""
    m = p > 0
    return float(np.sum(p[m] * np.log(p[m] / q[m])))


def entropy(p: np.ndarray) -> float:
    m = p > 0
    return float(-np.sum(p[m] * np.log(p[m])))


p = np.array([0.70, 0.20, 0.10])
q = np.array([0.40, 0.40, 0.20])
cross = float(-np.sum(p * np.log(q)))
print(f"H(p)={entropy(p):.4f}  H(p,q)={cross:.4f}  KL(p||q)={kl(p, q):.4f}")
print(f"H(p,q) - H(p) = {cross - entropy(p):.4f}")
print(f"KL(q||p)={kl(q, p):.4f}  (not equal: KL is asymmetric)")
# H(p)=0.8018  H(p,q)=0.9856  KL(p||q)=0.1838
# H(p,q) - H(p) = 0.1838
# KL(q||p)=0.1920  (not equal: KL is asymmetric)
