#!/usr/bin/env python3
# Source: t4-optimization.html
import numpy as np

rng = np.random.default_rng(0)
n, d = 500, 5
X = rng.normal(size=(n, d)); w_true = rng.normal(size=d)
y = (rng.random(n) < 1 / (1 + np.exp(-X @ w_true))).astype(float)
lam = 1e-2

def grad_hess(w):
    p = 1 / (1 + np.exp(-X @ w))
    g = X.T @ (p - y) / n + lam * w
    H = (X * (p * (1 - p))[:, None]).T @ X / n + lam * np.eye(d)
    return g, H

w = np.zeros(d)
for k in range(8):
    g, H = grad_hess(w)
    print(k, "%.2e" % np.linalg.norm(g))
    w = w - np.linalg.solve(H, g)       # Newton step
# gradient norm: 3.3e-01, 7.6e-02, 1.6e-02, 1.1e-03, 6.3e-06, 2.1e-10, 3.0e-17
