"""Single-head causal attention with fixed weights; not a pretrained language model.""" from dataclasses import dataclass, field import hashlib import json import math import platform import random import statistics import time D=8 rng=random.Random(17) EMBED=[[rng.uniform(-1,1) for _ in range(D)] for _ in range(12)] WEIGHTS=[[[rng.uniform(-.3,.3) for _ in range(D)] for _ in range(D)] for _ in range(3)] MODEL_ID=hashlib.sha256(json.dumps([EMBED,WEIGHTS]).encode()).hexdigest() def dot(a,b): return math.fsum(x*y for x,y in zip(a,b)) def project(x,w): return [dot(row,x) for row in w] def qkv(token,position): if type(token) is not int or not 0<=token