#!/usr/bin/env python3
# Source: q3-deep-learning.html
import torch
from torch import nn
from torch.utils.data import DataLoader, TensorDataset
from torch.utils.checkpoint import checkpoint

torch.manual_seed(7)
torch.set_num_threads(1)
device = "cuda" if torch.cuda.is_available() else "cpu"
use_cuda = device == "cuda"
X = torch.randn(256, 2)
y = (X[:, 0] + X[:, 1] > 0).float().reshape(-1, 1)
loader = DataLoader(TensorDataset(X, y), batch_size=32, shuffle=True)
block = nn.Sequential(nn.Linear(2, 16), nn.ReLU()).to(device)
head = nn.Linear(16, 1).to(device)
parameters = list(block.parameters()) + list(head.parameters())
opt = torch.optim.AdamW(parameters, lr=0.02)
loss_fn = nn.BCEWithLogitsLoss()
scaler = torch.amp.GradScaler("cuda", enabled=use_cuda)

for epoch in range(20):
    for x, labels in loader:
        x, labels = x.to(device), labels.to(device)
        opt.zero_grad(set_to_none=True)
        with torch.autocast(device_type=device, dtype=torch.float16, enabled=use_cuda):
            h = checkpoint(block, x, use_reentrant=False)
            loss = loss_fn(head(h), labels)
        scaler.scale(loss).backward()
        scaler.unscale_(opt)
        nn.utils.clip_grad_norm_(parameters, 1.0)
        scaler.step(opt)
        scaler.update()

with torch.no_grad():
    predicted = head(block(X.to(device))).cpu() > 0
    accuracy = (predicted == y.bool()).float().mean().item()
print(f"device={device}, training accuracy={accuracy:.3f}")
# Training accuracy checks this toy run. It is not held-out model quality.
