#!/usr/bin/env python3
# Source: q3-deep-learning.html
import numpy as np

def attention(Q, K, V, causal=False):
    d_k = Q.shape[-1]
    scores = Q @ K.swapaxes(-1, -2) / np.sqrt(d_k)       # (..., n, n)
    if causal:
        n = scores.shape[-1]
        mask = np.triu(np.ones((n, n), dtype=bool), k=1)  # True above diagonal
        scores = np.where(mask, -np.inf, scores)
    scores = scores - scores.max(axis=-1, keepdims=True)  # stable softmax
    w = np.exp(scores)
    w = w / w.sum(axis=-1, keepdims=True)
    return w @ V
