#!/usr/bin/env python3
# Source: t1-learning-theory.html
import numpy as np

rng = np.random.default_rng(2)
f = lambda x: np.sin(x)
# Source: x ~ N(0, 1). Target: x ~ N(1.5, 0.5^2). Same p(y|x).
xs = rng.normal(0, 1, 5000)
ys = f(xs) + 0.1 * rng.standard_normal(xs.size)
xt = rng.normal(1.5, 0.5, 5000)
yt = f(xt) + 0.1 * rng.standard_normal(xt.size)

def pdf(x, m, s):
    return np.exp(-0.5 * ((x - m) / s) ** 2) / (s * np.sqrt(2 * np.pi))

w = pdf(xs, 1.5, 0.5) / pdf(xs, 0, 1)        # importance weights q(x)/p(x)

def fit_line(x, y, sw):
    A = np.c_[np.ones_like(x), x] * np.sqrt(sw)[:, None]
    return np.linalg.lstsq(A, y * np.sqrt(sw), rcond=None)[0]

for name, sw in [("unweighted", np.ones_like(xs)), ("importance-weighted", w)]:
    b = fit_line(xs, ys, sw)
    mse = np.mean((b[0] + b[1] * xt - yt) ** 2)
    print(f"{name:20s} target MSE = {mse:.4f}")
ess = w.sum() ** 2 / (w ** 2).sum()
print(f"effective sample size = {ess:.0f} of {xs.size}")

# unweighted           target MSE = 0.1127
# importance-weighted  target MSE = 0.0332
# effective sample size = 925 of 5000
