#!/usr/bin/env python3
# Source: t5-linear-algebra.html
import numpy as np

rng = np.random.default_rng(5)
B, H, T, D = 2, 4, 5, 8
q = rng.normal(size=(B, H, T, D))
k = rng.normal(size=(B, H, T, D))
v = rng.normal(size=(B, H, T, D))

scores = np.einsum("bhid,bhjd->bhij", q, k) / np.sqrt(D)
w = np.exp(scores - scores.max(-1, keepdims=True))
w /= w.sum(-1, keepdims=True)                 # softmax over j
out = np.einsum("bhij,bhjd->bhid", w, v)

print(scores.shape, out.shape)
print(np.allclose(scores, q @ k.swapaxes(-1, -2) / np.sqrt(D)))
# (2, 4, 5, 5) (2, 4, 5, 8)
# True
