GELECEĞİ SADECE İZLEYENLERDEN Mİ OLACAKSIN,
YOKSA KENDİ İMPARATORLUĞUNU KURANLARDAN MI?

   1: class SelfAttention(nn.Module):
   2:     def __init__(self, n_embd, head_size):
   3:         super().__init__()
   4:         self.key = nn.Linear(n_embd, head_size, bias=False)
   5:         self.query = nn.Linear(n_embd, head_size, bias=False)
   6:         self.value = nn.Linear(n_embd, head_size, bias=False)
   7:         self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))
   8: 
   9:     def forward(self, x):
  10:         B, T, C = x.shape
  11:         k = self.key(x)   # [B, T, head_size]
  12:         q = self.query(x) # [B, T, head_size]
  13:         wei = q @ k.transpose(-2, -1) * head_size**-0.5 # [B, T, T]
  14:         wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))
  15:         wei = F.softmax(wei, dim=-1)
  16:         v = self.value(x) # [B, T, head_size]
  17:         out = wei @ v    # [B, T, head_size]
  18:         return out

   1: class BasicTokenizer:
   2:     def __init__(self, text):
   3:         self.chars = sorted(list(set(text)))
   4:         self.vocab_size = len(self.chars)
   5:         self.stoi = { ch:i for i,ch in enumerate(self.chars) }
   6:         self.itos = { i:ch for i,ch in enumerate(self.chars) }
   7: 
   8:     def encode(self, s):
   9:         return [self.stoi[c] for c in s]
  10: 
  11:     def decode(self, l):
  12:         return "".join([self.itos[i] for i in l])

   1: class BigramLM(nn.Module):
   2:     def __init__(self, vocab_size):
   3:         super().__init__()
   4:         self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)
   5: 
   6:     def forward(self, idx, targets=None):
   7:         logits = self.token_embedding_table(idx) # [B, T, V]
   8:         if targets is None:
   9:             loss = None
  10:         else:
  11:             B, T, C = logits.shape
  12:             logits = logits.view(B*T, C)
  13:             targets = targets.view(B*T)
  14:             loss = F.cross_entropy(logits, targets)
  15:         return logits, loss