85_注意力分数.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 注意力分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import torch\n",
    "from torch import nn\n",
    "from d2l import torch as d2l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 遮蔽softmax操作\n",
    "# 也即只针对句子的前valid_lens长度的单词来做softmax，后面的单词都忽略掉\n",
    "\n",
    "def masked_softmax(X, valid_lens):\n",
    "    \"\"\"通过在最后一个轴上遮蔽元素来执行softmax操作\"\"\"\n",
    "    \n",
    "    # 如果valid_lens，那么不需要做掩码，就是正常的softmax就可以了\n",
    "    if valid_lens is None:\n",
    "        return nn.functional.softmax(X, dim=-1)\n",
    "    else:\n",
    "        shape = X.shape\n",
    "        if valid_lens.dim() == 1:\n",
    "            # 将valid_lens重复shape[1]遍\n",
    "            valid_lens = torch.repeat_interleave(valid_lens, shape[1])\n",
    "        else:\n",
    "            valid_lens = valid_lens.reshape(-1)\n",
    "        \n",
    "        # 用很小的值 value = -1e6 来填充后面的部分（因为e^(-1e6)几乎为0了）\n",
    "        X = d2l.sequence_mask(X.reshape(-1, shape[-1]), valid_lens, value=-1e6)\n",
    "        \n",
    "        return nn.functional.softmax(X.reshape(shape), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[0.0819, 0.6884, 0.3716, 0.5099],\n",
      "         [0.1043, 0.7568, 0.4756, 0.7085]],\n",
      "\n",
      "        [[0.9847, 0.0046, 0.7059, 0.9903],\n",
      "         [0.6759, 0.3683, 0.1386, 0.6152]]])\n",
      "\n",
      "tensor([[[0.3529, 0.6471, 0.0000, 0.0000],\n",
      "         [0.3424, 0.6576, 0.0000, 0.0000]],\n",
      "\n",
      "        [[0.4691, 0.1760, 0.3549, 0.0000],\n",
      "         [0.4311, 0.3170, 0.2519, 0.0000]]])\n"
     ]
    }
   ],
   "source": [
    "# 演示此函数是如何工作\n",
    "X = torch.rand(2,2,4)\n",
    "print(X)\n",
    "print()\n",
    "\n",
    "output = masked_softmax(X, torch.tensor([2,3]))\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[0.8657, 0.3603, 0.8583, 0.4407],\n",
      "         [0.9273, 0.2020, 0.4907, 0.0074]],\n",
      "\n",
      "        [[0.4174, 0.3513, 0.1362, 0.2864],\n",
      "         [0.4780, 0.0143, 0.0262, 0.6546]]])\n",
      "\n",
      "tensor([[[1.0000, 0.0000, 0.0000, 0.0000],\n",
      "         [0.4694, 0.2273, 0.3033, 0.0000]],\n",
      "\n",
      "        [[0.5165, 0.4835, 0.0000, 0.0000],\n",
      "         [0.2891, 0.1819, 0.1840, 0.3450]]])\n"
     ]
    }
   ],
   "source": [
    "X = torch.rand(2,2,4)\n",
    "print(X)\n",
    "print()\n",
    "\n",
    "output = masked_softmax(X, torch.tensor([[1,3],[2,4]]))\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加性注意力\n",
    "# 这里是向量版本的query、key和value\n",
    "class AdditiveAttention(nn.Module):\n",
    "    \"\"\"加性注意力\"\"\"\n",
    "    \n",
    "    # 有W_k、W_q和w_v三个可以学习的参数\n",
    "    def __init__(self, key_size, query_size, num_hiddens, dropout, **kwargs):\n",
    "        super(AdditiveAttention, self).__init__(**kwargs)\n",
    "        self.W_k = nn.Linear(key_size, num_hiddens, bias=False)\n",
    "        self.W_q = nn.Linear(query_size, num_hiddens, bias=False)\n",
    "        self.w_v = nn.Linear(num_hiddens, 1, bias=False)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    # valid_lens：表示针对每个query，应该考虑前多少个key-value对\n",
    "    # 因为有一部分key-value对可能是pad出来的，所以这里是用来屏蔽被pad的这些部分的\n",
    "    def forward(self, queries, keys, values, valid_lens):\n",
    "        # 做乘法后的queries和keys\n",
    "        queries = self.W_q(queries)  # queries的维度为\"batch_size * number of queries * h\"\n",
    "        keys = self.W_k(keys)  # 而keys的维度为\"batch_size * number of keys * h\"\n",
    "        \n",
    "        print(f\"queries' shape is {queries.shape}\")\n",
    "        print(f\"keys' shape is {keys.shape}\")\n",
    "        # print(f\"queries is {queries}\")\n",
    "        # print(f\"keys is {keys}\")\n",
    "        \n",
    "        # 因为这里queries和keys的中间维度不同，不能直接相加；所以在这里要做一个广播\n",
    "        # unsqueeze之后，queries维度变成\"batch_size * number of queries * 1 * h\"\n",
    "        # 而keys的维度则变成了\"batch_size * 1 * number of keys * h\"\n",
    "        # 从而\"bs * nq * 1 * h\"和\"bs * 1 * nk * h\"就可以利用广播机制相加了，得到的结果为\"bs * nq * nk * h\"\n",
    "        features = queries.unsqueeze(2) + keys.unsqueeze(1)\n",
    "        features = torch.tanh(features)  # 对每个值都做一个tanh\n",
    "        \n",
    "        print(f\"features' shape is {features.shape}\")\n",
    "        \n",
    "        # 然后再和w_v做矩阵乘法，会得到\"bs * nq * nk * 1\"的结果；所以再用squeeze(-1)去掉最后一维\n",
    "        scores = self.w_v(features).squeeze(-1)\n",
    "        \n",
    "        print(f\"scores' shape is {scores.shape}\")\n",
    "        print(f\"scores is {scores}\")\n",
    "        \n",
    "        # 然后再对每个注意力分数再做一次masked_softmax，过滤掉不需要的key-value对们\n",
    "        self.attention_weights = masked_softmax(scores, valid_lens)\n",
    "        \n",
    "        print(f\"self.attention_weights' shape is {self.attention_weights.shape}\")\n",
    "        print(f\"self.attention_weights is {self.attention_weights}\")\n",
    "        \n",
    "        # 再是先对权重做一个dropout，再将这个注意力权重和values放在一起做batch的矩阵乘法，得到结果\n",
    "        # 返回结果output的维度为\"batch_size * number of queries * dim(v)\"\n",
    "        # 也即“批量数 * q的个数 * v的长度”\n",
    "        output = torch.bmm(self.dropout(self.attention_weights), values)\n",
    "        \n",
    "        print(f\"output's shape is {output.shape}\")\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "queries is tensor([[[ 0.3877,  0.1854, -0.0660,  0.6484,  1.4467, -1.0182,  1.2323,\n",
      "           1.8302,  0.0167,  0.3803,  0.1692, -1.0165,  0.5766,  0.5532,\n",
      "           1.2317, -0.1063,  1.1871, -0.2813,  1.8884, -1.0535]],\n",
      "\n",
      "        [[-1.1299, -1.1643, -0.6529,  0.8682, -1.1065, -2.1782, -0.7174,\n",
      "          -0.7151, -2.6513, -0.4739, -0.1660,  0.6910,  0.4890,  1.2006,\n",
      "           0.8941, -0.1994, -0.2832,  0.1053,  0.6804, -0.7624]]])\n",
      "------------------------------\n",
      "queries' shape is torch.Size([2, 1, 8])\n",
      "keys' shape is torch.Size([2, 10, 8])\n",
      "features' shape is torch.Size([2, 1, 10, 8])\n",
      "scores' shape is torch.Size([2, 1, 10])\n",
      "scores is tensor([[[ 0.0679,  0.0679,  0.0679,  0.0679,  0.0679,  0.0679,  0.0679,\n",
      "           0.0679,  0.0679,  0.0679]],\n",
      "\n",
      "        [[-0.4908, -0.4908, -0.4908, -0.4908, -0.4908, -0.4908, -0.4908,\n",
      "          -0.4908, -0.4908, -0.4908]]], grad_fn=<SqueezeBackward1>)\n",
      "self.attention_weights' shape is torch.Size([2, 1, 10])\n",
      "self.attention_weights is tensor([[[0.5000, 0.5000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
      "          0.0000, 0.0000]],\n",
      "\n",
      "        [[0.1667, 0.1667, 0.1667, 0.1667, 0.1667, 0.1667, 0.0000, 0.0000,\n",
      "          0.0000, 0.0000]]], grad_fn=<SoftmaxBackward0>)\n",
      "output's shape is torch.Size([2, 1, 4])\n",
      "------------------------------\n",
      "tensor([[[ 2.0000,  3.0000,  4.0000,  5.0000]],\n",
      "\n",
      "        [[10.0000, 11.0000, 12.0000, 13.0000]]], grad_fn=<BmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# 演示上面的AdditiveAttention类\n",
    "\n",
    "# torch.normal函数用法：https://blog.csdn.net/qq_34690929/article/details/80029974\n",
    "queries = torch.normal(0, 1, (2,1,20))  # 表示生成的queries的batch_size为2，每个batch包含1个query，每个query的长度为20\n",
    "keys = torch.ones((2,10,2))  # 表示生成的keys的batch_size为2，每个batch包含10个key，每个key的长度为2\n",
    "\n",
    "# 将1-40共40个数字分到1*10*4的tensor中，然后再把第一维repeat一遍，所以values的维度为2*10*4\n",
    "# 表示values的batch_size为2，每个batch包含10个value，每个value的长度为4\n",
    "values = torch.arange(40, dtype=torch.float32).reshape(1,10,4).repeat(2,1,1)  \n",
    "valid_lens = torch.tensor([2,6])\n",
    "\n",
    "print(f\"queries is {queries}\")\n",
    "# print(f\"keys is {keys}\")\n",
    "# print(f\"values is {values}\")\n",
    "print(\"-\"*30)\n",
    "\n",
    "attention = AdditiveAttention(key_size=2, query_size=20, num_hiddens=8, dropout=0.1)  \n",
    "attention.eval()\n",
    "\n",
    "output = attention(queries, keys, values, valid_lens)\n",
    "\n",
    "print(\"-\"*30)\n",
    "print(output)  # 返回结果为“批量数 * q的个数 * v的长度”"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"187.07675pt\" height=\"103.438906pt\" viewBox=\"0 0 187.07675 103.438906\" xmlns=\"http://www.w3.org/2000/svg\" version=\"1.1\">\n",
       " <metadata>\n",
       "  <rdf:RDF xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n",
       "   <cc:Work>\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n",
       "    <dc:date>2022-11-24T01:34:26.669407</dc:date>\n",
       "    <dc:format>image/svg+xml</dc:format>\n",
       "    <dc:creator>\n",
       "     <cc:Agent>\n",
       "      <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>\n",
       "     </cc:Agent>\n",
       "    </dc:creator>\n",
       "   </cc:Work>\n",
       "  </rdf:RDF>\n",
       " </metadata>\n",
       " <defs>\n",
       "  <style type=\"text/css\">*{stroke-linejoin: round; stroke-linecap: butt}</style>\n",
       " </defs>\n",
       " <g id=\"figure_1\">\n",
       "  <g id=\"patch_1\">\n",
       "   <path d=\"M -0 103.438906 \n",
       "L 187.07675 103.438906 \n",
       "L 187.07675 0 \n",
       "L -0 0 \n",
       "z\n",
       "\" style=\"fill: #ffffff\"/>\n",
       "  </g>\n",
       "  <g id=\"axes_1\">\n",
       "   <g id=\"patch_2\">\n",
       "    <path d=\"M 34.240625 59.94 \n",
       "L 145.840625 59.94 \n",
       "L 145.840625 37.62 \n",
       "L 34.240625 37.62 \n",
       "z\n",
       "\" style=\"fill: #ffffff\"/>\n",
       "   </g>\n",
       "   <g clip-path=\"url(#p80de84f851)\">\n",
       "    <image xlink:href=\"data:image/png;base64,\n",
       "iVBORw0KGgoAAAANSUhEUgAAAJsAAAAfCAYAAADwQL9CAAAAjUlEQVR4nO3aMRHCAAxA0Zau1YcFxmqoCzbMYYGNO+riZ+A9A8nwL1PW7+v8LaS2+zG9wojb9AL8D7GRERsZsZERGxmxkREbGbGRERsZsZERGxmxkREbGbGRERuZ9bHsY/9sz897ajQDXDYyYiMjNjJiIyM2MmIjIzYyYiMjNjJiIyM2MmIjIzYyYiNzAasBCAMqMzMRAAAAAElFTkSuQmCC\" id=\"imagea21c5e101b\" transform=\"scale(1 -1)translate(0 -22.32)\" x=\"34.240625\" y=\"-37.62\" width=\"111.6\" height=\"22.32\"/>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_1\">\n",
       "    <g id=\"xtick_1\">\n",
       "     <g id=\"line2d_1\">\n",
       "      <defs>\n",
       "       <path id=\"mb0d3392e23\" d=\"M 0 0 \n",
       "L 0 3.5 \n",
       "\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use xlink:href=\"#mb0d3392e23\" x=\"39.820625\" y=\"59.94\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_1\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(36.639375 74.538438)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-30\" d=\"M 2034 4250 \n",
       "Q 1547 4250 1301 3770 \n",
       "Q 1056 3291 1056 2328 \n",
       "Q 1056 1369 1301 889 \n",
       "Q 1547 409 2034 409 \n",
       "Q 2525 409 2770 889 \n",
       "Q 3016 1369 3016 2328 \n",
       "Q 3016 3291 2770 3770 \n",
       "Q 2525 4250 2034 4250 \n",
       "z\n",
       "M 2034 4750 \n",
       "Q 2819 4750 3233 4129 \n",
       "Q 3647 3509 3647 2328 \n",
       "Q 3647 1150 3233 529 \n",
       "Q 2819 -91 2034 -91 \n",
       "Q 1250 -91 836 529 \n",
       "Q 422 1150 422 2328 \n",
       "Q 422 3509 836 4129 \n",
       "Q 1250 4750 2034 4750 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_2\">\n",
       "     <g id=\"line2d_2\">\n",
       "      <g>\n",
       "       <use xlink:href=\"#mb0d3392e23\" x=\"95.620625\" y=\"59.94\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_2\">\n",
       "      <!-- 5 -->\n",
       "      <g transform=\"translate(92.439375 74.538438)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-35\" d=\"M 691 4666 \n",
       "L 3169 4666 \n",
       "L 3169 4134 \n",
       "L 1269 4134 \n",
       "L 1269 2991 \n",
       "Q 1406 3038 1543 3061 \n",
       "Q 1681 3084 1819 3084 \n",
       "Q 2600 3084 3056 2656 \n",
       "Q 3513 2228 3513 1497 \n",
       "Q 3513 744 3044 326 \n",
       "Q 2575 -91 1722 -91 \n",
       "Q 1428 -91 1123 -41 \n",
       "Q 819 9 494 109 \n",
       "L 494 744 \n",
       "Q 775 591 1075 516 \n",
       "Q 1375 441 1709 441 \n",
       "Q 2250 441 2565 725 \n",
       "Q 2881 1009 2881 1497 \n",
       "Q 2881 1984 2565 2268 \n",
       "Q 2250 2553 1709 2553 \n",
       "Q 1456 2553 1204 2497 \n",
       "Q 953 2441 691 2322 \n",
       "L 691 4666 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_3\">\n",
       "     <!-- Keys -->\n",
       "     <g transform=\"translate(78.371094 88.216563)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path id=\"DejaVuSans-4b\" d=\"M 628 4666 \n",
       "L 1259 4666 \n",
       "L 1259 2694 \n",
       "L 3353 4666 \n",
       "L 4166 4666 \n",
       "L 1850 2491 \n",
       "L 4331 0 \n",
       "L 3500 0 \n",
       "L 1259 2247 \n",
       "L 1259 0 \n",
       "L 628 0 \n",
       "L 628 4666 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-65\" d=\"M 3597 1894 \n",
       "L 3597 1613 \n",
       "L 953 1613 \n",
       "Q 991 1019 1311 708 \n",
       "Q 1631 397 2203 397 \n",
       "Q 2534 397 2845 478 \n",
       "Q 3156 559 3463 722 \n",
       "L 3463 178 \n",
       "Q 3153 47 2828 -22 \n",
       "Q 2503 -91 2169 -91 \n",
       "Q 1331 -91 842 396 \n",
       "Q 353 884 353 1716 \n",
       "Q 353 2575 817 3079 \n",
       "Q 1281 3584 2069 3584 \n",
       "Q 2775 3584 3186 3129 \n",
       "Q 3597 2675 3597 1894 \n",
       "z\n",
       "M 3022 2063 \n",
       "Q 3016 2534 2758 2815 \n",
       "Q 2500 3097 2075 3097 \n",
       "Q 1594 3097 1305 2825 \n",
       "Q 1016 2553 972 2059 \n",
       "L 3022 2063 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-79\" d=\"M 2059 -325 \n",
       "Q 1816 -950 1584 -1140 \n",
       "Q 1353 -1331 966 -1331 \n",
       "L 506 -1331 \n",
       "L 506 -850 \n",
       "L 844 -850 \n",
       "Q 1081 -850 1212 -737 \n",
       "Q 1344 -625 1503 -206 \n",
       "L 1606 56 \n",
       "L 191 3500 \n",
       "L 800 3500 \n",
       "L 1894 763 \n",
       "L 2988 3500 \n",
       "L 3597 3500 \n",
       "L 2059 -325 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-73\" d=\"M 2834 3397 \n",
       "L 2834 2853 \n",
       "Q 2591 2978 2328 3040 \n",
       "Q 2066 3103 1784 3103 \n",
       "Q 1356 3103 1142 2972 \n",
       "Q 928 2841 928 2578 \n",
       "Q 928 2378 1081 2264 \n",
       "Q 1234 2150 1697 2047 \n",
       "L 1894 2003 \n",
       "Q 2506 1872 2764 1633 \n",
       "Q 3022 1394 3022 966 \n",
       "Q 3022 478 2636 193 \n",
       "Q 2250 -91 1575 -91 \n",
       "Q 1294 -91 989 -36 \n",
       "Q 684 19 347 128 \n",
       "L 347 722 \n",
       "Q 666 556 975 473 \n",
       "Q 1284 391 1588 391 \n",
       "Q 1994 391 2212 530 \n",
       "Q 2431 669 2431 922 \n",
       "Q 2431 1156 2273 1281 \n",
       "Q 2116 1406 1581 1522 \n",
       "L 1381 1569 \n",
       "Q 847 1681 609 1914 \n",
       "Q 372 2147 372 2553 \n",
       "Q 372 3047 722 3315 \n",
       "Q 1072 3584 1716 3584 \n",
       "Q 2034 3584 2315 3537 \n",
       "Q 2597 3491 2834 3397 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-4b\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-65\" x=\"60.576172\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-79\" x=\"122.099609\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-73\" x=\"181.279297\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_2\">\n",
       "    <g id=\"ytick_1\">\n",
       "     <g id=\"line2d_3\">\n",
       "      <defs>\n",
       "       <path id=\"mb35117badd\" d=\"M 0 0 \n",
       "L -3.5 0 \n",
       "\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use xlink:href=\"#mb35117badd\" x=\"34.240625\" y=\"43.2\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_4\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(20.878125 46.999219)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_2\">\n",
       "     <g id=\"line2d_4\">\n",
       "      <g>\n",
       "       <use xlink:href=\"#mb35117badd\" x=\"34.240625\" y=\"54.36\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_5\">\n",
       "      <!-- 1 -->\n",
       "      <g transform=\"translate(20.878125 58.159219)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-31\" d=\"M 794 531 \n",
       "L 1825 531 \n",
       "L 1825 4091 \n",
       "L 703 3866 \n",
       "L 703 4441 \n",
       "L 1819 4666 \n",
       "L 2450 4666 \n",
       "L 2450 531 \n",
       "L 3481 531 \n",
       "L 3481 0 \n",
       "L 794 0 \n",
       "L 794 531 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_6\">\n",
       "     <!-- Queries -->\n",
       "     <g transform=\"translate(14.798437 68.087031)rotate(-90)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path id=\"DejaVuSans-51\" d=\"M 2522 4238 \n",
       "Q 1834 4238 1429 3725 \n",
       "Q 1025 3213 1025 2328 \n",
       "Q 1025 1447 1429 934 \n",
       "Q 1834 422 2522 422 \n",
       "Q 3209 422 3611 934 \n",
       "Q 4013 1447 4013 2328 \n",
       "Q 4013 3213 3611 3725 \n",
       "Q 3209 4238 2522 4238 \n",
       "z\n",
       "M 3406 84 \n",
       "L 4238 -825 \n",
       "L 3475 -825 \n",
       "L 2784 -78 \n",
       "Q 2681 -84 2626 -87 \n",
       "Q 2572 -91 2522 -91 \n",
       "Q 1538 -91 948 567 \n",
       "Q 359 1225 359 2328 \n",
       "Q 359 3434 948 4092 \n",
       "Q 1538 4750 2522 4750 \n",
       "Q 3503 4750 4090 4092 \n",
       "Q 4678 3434 4678 2328 \n",
       "Q 4678 1516 4351 937 \n",
       "Q 4025 359 3406 84 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-75\" d=\"M 544 1381 \n",
       "L 544 3500 \n",
       "L 1119 3500 \n",
       "L 1119 1403 \n",
       "Q 1119 906 1312 657 \n",
       "Q 1506 409 1894 409 \n",
       "Q 2359 409 2629 706 \n",
       "Q 2900 1003 2900 1516 \n",
       "L 2900 3500 \n",
       "L 3475 3500 \n",
       "L 3475 0 \n",
       "L 2900 0 \n",
       "L 2900 538 \n",
       "Q 2691 219 2414 64 \n",
       "Q 2138 -91 1772 -91 \n",
       "Q 1169 -91 856 284 \n",
       "Q 544 659 544 1381 \n",
       "z\n",
       "M 1991 3584 \n",
       "L 1991 3584 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-72\" d=\"M 2631 2963 \n",
       "Q 2534 3019 2420 3045 \n",
       "Q 2306 3072 2169 3072 \n",
       "Q 1681 3072 1420 2755 \n",
       "Q 1159 2438 1159 1844 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2956 \n",
       "Q 1341 3275 1631 3429 \n",
       "Q 1922 3584 2338 3584 \n",
       "Q 2397 3584 2469 3576 \n",
       "Q 2541 3569 2628 3553 \n",
       "L 2631 2963 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-69\" d=\"M 603 3500 \n",
       "L 1178 3500 \n",
       "L 1178 0 \n",
       "L 603 0 \n",
       "L 603 3500 \n",
       "z\n",
       "M 603 4863 \n",
       "L 1178 4863 \n",
       "L 1178 4134 \n",
       "L 603 4134 \n",
       "L 603 4863 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-51\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-75\" x=\"78.710938\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-65\" x=\"142.089844\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-72\" x=\"203.613281\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-69\" x=\"244.726562\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-65\" x=\"272.509766\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-73\" x=\"334.033203\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"patch_3\">\n",
       "    <path d=\"M 34.240625 59.94 \n",
       "L 34.240625 37.62 \n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_4\">\n",
       "    <path d=\"M 145.840625 59.94 \n",
       "L 145.840625 37.62 \n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_5\">\n",
       "    <path d=\"M 34.240625 59.94 \n",
       "L 145.840625 59.94 \n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_6\">\n",
       "    <path d=\"M 34.240625 37.62 \n",
       "L 145.840625 37.62 \n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "  </g>\n",
       "  <g id=\"axes_2\">\n",
       "   <g id=\"patch_7\">\n",
       "    <path d=\"M 152.815625 90.36 \n",
       "L 156.973625 90.36 \n",
       "L 156.973625 7.2 \n",
       "L 152.815625 7.2 \n",
       "z\n",
       "\" style=\"fill: #ffffff\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_8\">\n",
       "    <path clip-path=\"url(#p538806c331)\" style=\"fill: #ffffff; stroke: #ffffff; stroke-width: 0.01; stroke-linejoin: miter\"/>\n",
       "   </g>\n",
       "   <image xlink:href=\"data:image/png;base64,\n",
       "iVBORw0KGgoAAAANSUhEUgAAAAYAAAB0CAYAAACmJkOCAAAAyElEQVR4nM2WSwrEMAxDXcj9zzqbblp/egG/gEII02WEJMt2Qq+6f2XNN6yyO7dhGQQQo1qLFY9ij30AB5Rz6B7uIqNQKkiKAUwuAyRV8e6SmjTxgAcy5An+qTkD+vrQNVgw15sI5qWXi909kLwCpTCgDPgB83dfuXpVFDAD3vakR18H3KEqWJKJVCBAyTEgSukBw2WP9thsJP0zOAEoFcjoz22EUQ5kyFVhwH5BZx4MkBQ2Ee7mhAEjX2E88sxRCluyYg4MquoD0XoQUGMmzEcAAAAASUVORK5CYII=\" id=\"image2858388973\" transform=\"scale(1 -1)translate(0 -83.52)\" x=\"152.64\" y=\"-6.48\" width=\"4.32\" height=\"83.52\"/>\n",
       "   <g id=\"matplotlib.axis_3\">\n",
       "    <g id=\"ytick_3\">\n",
       "     <g id=\"line2d_5\">\n",
       "      <defs>\n",
       "       <path id=\"m71377fa5fe\" d=\"M 0 0 \n",
       "L 3.5 0 \n",
       "\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use xlink:href=\"#m71377fa5fe\" x=\"156.973625\" y=\"90.36\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_7\">\n",
       "      <!-- 0.0 -->\n",
       "      <g transform=\"translate(163.973625 94.159219)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-2e\" d=\"M 684 794 \n",
       "L 1344 794 \n",
       "L 1344 0 \n",
       "L 684 0 \n",
       "L 684 794 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_4\">\n",
       "     <g id=\"line2d_6\">\n",
       "      <g>\n",
       "       <use xlink:href=\"#m71377fa5fe\" x=\"156.973625\" y=\"57.096\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_8\">\n",
       "      <!-- 0.2 -->\n",
       "      <g transform=\"translate(163.973625 60.895219)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-32\" d=\"M 1228 531 \n",
       "L 3431 531 \n",
       "L 3431 0 \n",
       "L 469 0 \n",
       "L 469 531 \n",
       "Q 828 903 1448 1529 \n",
       "Q 2069 2156 2228 2338 \n",
       "Q 2531 2678 2651 2914 \n",
       "Q 2772 3150 2772 3378 \n",
       "Q 2772 3750 2511 3984 \n",
       "Q 2250 4219 1831 4219 \n",
       "Q 1534 4219 1204 4116 \n",
       "Q 875 4013 500 3803 \n",
       "L 500 4441 \n",
       "Q 881 4594 1212 4672 \n",
       "Q 1544 4750 1819 4750 \n",
       "Q 2544 4750 2975 4387 \n",
       "Q 3406 4025 3406 3419 \n",
       "Q 3406 3131 3298 2873 \n",
       "Q 3191 2616 2906 2266 \n",
       "Q 2828 2175 2409 1742 \n",
       "Q 1991 1309 1228 531 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-32\" x=\"95.410156\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_5\">\n",
       "     <g id=\"line2d_7\">\n",
       "      <g>\n",
       "       <use xlink:href=\"#m71377fa5fe\" x=\"156.973625\" y=\"23.832\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_9\">\n",
       "      <!-- 0.4 -->\n",
       "      <g transform=\"translate(163.973625 27.631219)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-34\" d=\"M 2419 4116 \n",
       "L 825 1625 \n",
       "L 2419 1625 \n",
       "L 2419 4116 \n",
       "z\n",
       "M 2253 4666 \n",
       "L 3047 4666 \n",
       "L 3047 1625 \n",
       "L 3713 1625 \n",
       "L 3713 1100 \n",
       "L 3047 1100 \n",
       "L 3047 0 \n",
       "L 2419 0 \n",
       "L 2419 1100 \n",
       "L 313 1100 \n",
       "L 313 1709 \n",
       "L 2253 4666 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-34\" x=\"95.410156\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"LineCollection_1\"/>\n",
       "   <g id=\"patch_9\">\n",
       "    <path d=\"M 152.815625 90.36 \n",
       "L 154.894625 90.36 \n",
       "L 156.973625 90.36 \n",
       "L 156.973625 7.2 \n",
       "L 154.894625 7.2 \n",
       "L 152.815625 7.2 \n",
       "L 152.815625 90.36 \n",
       "z\n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "  </g>\n",
       " </g>\n",
       " <defs>\n",
       "  <clipPath id=\"p80de84f851\">\n",
       "   <rect x=\"34.240625\" y=\"37.62\" width=\"111.6\" height=\"22.32\"/>\n",
       "  </clipPath>\n",
       "  <clipPath id=\"p538806c331\">\n",
       "   <rect x=\"152.815625\" y=\"7.2\" width=\"4.158\" height=\"83.16\"/>\n",
       "  </clipPath>\n",
       " </defs>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<Figure size 250x250 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 注意力权重\n",
    "d2l.show_heatmaps(attention.attention_weights.reshape((1,1,2,10)),\n",
    "                 xlabel='Keys', ylabel='Queries')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 缩放点积注意力\n",
    "# 在query和key的维度相同时方可使用\n",
    "\n",
    "class DotProductAttention(nn.Module):\n",
    "    \"\"\"缩放点积注意力\"\"\"\n",
    "    \n",
    "    def __init__(self, dropout, **kwargs):\n",
    "        super(DotProductAttention, self).__init__(**kwargs)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, queries, keys, values, valid_lens=None):\n",
    "        # 获取query或key的维度d\n",
    "        d = queries.shape[-1]\n",
    "        \n",
    "        # 这里keys要做一个转置（因为计算公式即为\"Q*K^T / sqrt(d)\"）\n",
    "        scores = torch.bmm(queries, keys.transpose(1,2)) / math.sqrt(d)\n",
    "        \n",
    "        self.attention_weights = masked_softmax(scores, valid_lens)\n",
    "        \n",
    "        output = torch.bmm(self.dropout(self.attention_weights), values)\n",
    "        \n",
    "        # 返回的结果应该也是“批量数 * q的个数 * v的长度”？？？\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 2.0000,  3.0000,  4.0000,  5.0000]],\n",
       "\n",
       "        [[10.0000, 11.0000, 12.0000, 13.0000]]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 演示上述的DotProductAttention类\n",
    "\n",
    "queries = torch.normal(0, 1, (2,1,2))\n",
    "\n",
    "attention = DotProductAttention(dropout=0.5)\n",
    "attention.eval()\n",
    "\n",
    "attention(queries, keys, values, valid_lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"187.07675pt\" height=\"103.438906pt\" viewBox=\"0 0 187.07675 103.438906\" xmlns=\"http://www.w3.org/2000/svg\" version=\"1.1\">\n",
       " <metadata>\n",
       "  <rdf:RDF xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n",
       "   <cc:Work>\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n",
       "    <dc:date>2022-11-24T01:35:15.779834</dc:date>\n",
       "    <dc:format>image/svg+xml</dc:format>\n",
       "    <dc:creator>\n",
       "     <cc:Agent>\n",
       "      <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>\n",
       "     </cc:Agent>\n",
       "    </dc:creator>\n",
       "   </cc:Work>\n",
       "  </rdf:RDF>\n",
       " </metadata>\n",
       " <defs>\n",
       "  <style type=\"text/css\">*{stroke-linejoin: round; stroke-linecap: butt}</style>\n",
       " </defs>\n",
       " <g id=\"figure_1\">\n",
       "  <g id=\"patch_1\">\n",
       "   <path d=\"M -0 103.438906 \n",
       "L 187.07675 103.438906 \n",
       "L 187.07675 0 \n",
       "L -0 0 \n",
       "z\n",
       "\" style=\"fill: #ffffff\"/>\n",
       "  </g>\n",
       "  <g id=\"axes_1\">\n",
       "   <g id=\"patch_2\">\n",
       "    <path d=\"M 34.240625 59.94 \n",
       "L 145.840625 59.94 \n",
       "L 145.840625 37.62 \n",
       "L 34.240625 37.62 \n",
       "z\n",
       "\" style=\"fill: #ffffff\"/>\n",
       "   </g>\n",
       "   <g clip-path=\"url(#p772bf68c7c)\">\n",
       "    <image xlink:href=\"data:image/png;base64,\n",
       "iVBORw0KGgoAAAANSUhEUgAAAJsAAAAfCAYAAADwQL9CAAAAjUlEQVR4nO3aMRHCAAxA0Zau1YcFxmqoCzbMYYGNO+riZ+A9A8nwL1PW7+v8LaS2+zG9wojb9AL8D7GRERsZsZERGxmxkREbGbGRERsZsZERGxmxkREbGbGRERuZ9bHsY/9sz897ajQDXDYyYiMjNjJiIyM2MmIjIzYyYiMjNjJiIyM2MmIjIzYyYiNzAasBCAMqMzMRAAAAAElFTkSuQmCC\" id=\"imagee41f0bb42a\" transform=\"scale(1 -1)translate(0 -22.32)\" x=\"34.240625\" y=\"-37.62\" width=\"111.6\" height=\"22.32\"/>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_1\">\n",
       "    <g id=\"xtick_1\">\n",
       "     <g id=\"line2d_1\">\n",
       "      <defs>\n",
       "       <path id=\"m96d17227de\" d=\"M 0 0 \n",
       "L 0 3.5 \n",
       "\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use xlink:href=\"#m96d17227de\" x=\"39.820625\" y=\"59.94\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_1\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(36.639375 74.538438)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-30\" d=\"M 2034 4250 \n",
       "Q 1547 4250 1301 3770 \n",
       "Q 1056 3291 1056 2328 \n",
       "Q 1056 1369 1301 889 \n",
       "Q 1547 409 2034 409 \n",
       "Q 2525 409 2770 889 \n",
       "Q 3016 1369 3016 2328 \n",
       "Q 3016 3291 2770 3770 \n",
       "Q 2525 4250 2034 4250 \n",
       "z\n",
       "M 2034 4750 \n",
       "Q 2819 4750 3233 4129 \n",
       "Q 3647 3509 3647 2328 \n",
       "Q 3647 1150 3233 529 \n",
       "Q 2819 -91 2034 -91 \n",
       "Q 1250 -91 836 529 \n",
       "Q 422 1150 422 2328 \n",
       "Q 422 3509 836 4129 \n",
       "Q 1250 4750 2034 4750 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_2\">\n",
       "     <g id=\"line2d_2\">\n",
       "      <g>\n",
       "       <use xlink:href=\"#m96d17227de\" x=\"95.620625\" y=\"59.94\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_2\">\n",
       "      <!-- 5 -->\n",
       "      <g transform=\"translate(92.439375 74.538438)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-35\" d=\"M 691 4666 \n",
       "L 3169 4666 \n",
       "L 3169 4134 \n",
       "L 1269 4134 \n",
       "L 1269 2991 \n",
       "Q 1406 3038 1543 3061 \n",
       "Q 1681 3084 1819 3084 \n",
       "Q 2600 3084 3056 2656 \n",
       "Q 3513 2228 3513 1497 \n",
       "Q 3513 744 3044 326 \n",
       "Q 2575 -91 1722 -91 \n",
       "Q 1428 -91 1123 -41 \n",
       "Q 819 9 494 109 \n",
       "L 494 744 \n",
       "Q 775 591 1075 516 \n",
       "Q 1375 441 1709 441 \n",
       "Q 2250 441 2565 725 \n",
       "Q 2881 1009 2881 1497 \n",
       "Q 2881 1984 2565 2268 \n",
       "Q 2250 2553 1709 2553 \n",
       "Q 1456 2553 1204 2497 \n",
       "Q 953 2441 691 2322 \n",
       "L 691 4666 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_3\">\n",
       "     <!-- Keys -->\n",
       "     <g transform=\"translate(78.371094 88.216563)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path id=\"DejaVuSans-4b\" d=\"M 628 4666 \n",
       "L 1259 4666 \n",
       "L 1259 2694 \n",
       "L 3353 4666 \n",
       "L 4166 4666 \n",
       "L 1850 2491 \n",
       "L 4331 0 \n",
       "L 3500 0 \n",
       "L 1259 2247 \n",
       "L 1259 0 \n",
       "L 628 0 \n",
       "L 628 4666 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-65\" d=\"M 3597 1894 \n",
       "L 3597 1613 \n",
       "L 953 1613 \n",
       "Q 991 1019 1311 708 \n",
       "Q 1631 397 2203 397 \n",
       "Q 2534 397 2845 478 \n",
       "Q 3156 559 3463 722 \n",
       "L 3463 178 \n",
       "Q 3153 47 2828 -22 \n",
       "Q 2503 -91 2169 -91 \n",
       "Q 1331 -91 842 396 \n",
       "Q 353 884 353 1716 \n",
       "Q 353 2575 817 3079 \n",
       "Q 1281 3584 2069 3584 \n",
       "Q 2775 3584 3186 3129 \n",
       "Q 3597 2675 3597 1894 \n",
       "z\n",
       "M 3022 2063 \n",
       "Q 3016 2534 2758 2815 \n",
       "Q 2500 3097 2075 3097 \n",
       "Q 1594 3097 1305 2825 \n",
       "Q 1016 2553 972 2059 \n",
       "L 3022 2063 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-79\" d=\"M 2059 -325 \n",
       "Q 1816 -950 1584 -1140 \n",
       "Q 1353 -1331 966 -1331 \n",
       "L 506 -1331 \n",
       "L 506 -850 \n",
       "L 844 -850 \n",
       "Q 1081 -850 1212 -737 \n",
       "Q 1344 -625 1503 -206 \n",
       "L 1606 56 \n",
       "L 191 3500 \n",
       "L 800 3500 \n",
       "L 1894 763 \n",
       "L 2988 3500 \n",
       "L 3597 3500 \n",
       "L 2059 -325 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-73\" d=\"M 2834 3397 \n",
       "L 2834 2853 \n",
       "Q 2591 2978 2328 3040 \n",
       "Q 2066 3103 1784 3103 \n",
       "Q 1356 3103 1142 2972 \n",
       "Q 928 2841 928 2578 \n",
       "Q 928 2378 1081 2264 \n",
       "Q 1234 2150 1697 2047 \n",
       "L 1894 2003 \n",
       "Q 2506 1872 2764 1633 \n",
       "Q 3022 1394 3022 966 \n",
       "Q 3022 478 2636 193 \n",
       "Q 2250 -91 1575 -91 \n",
       "Q 1294 -91 989 -36 \n",
       "Q 684 19 347 128 \n",
       "L 347 722 \n",
       "Q 666 556 975 473 \n",
       "Q 1284 391 1588 391 \n",
       "Q 1994 391 2212 530 \n",
       "Q 2431 669 2431 922 \n",
       "Q 2431 1156 2273 1281 \n",
       "Q 2116 1406 1581 1522 \n",
       "L 1381 1569 \n",
       "Q 847 1681 609 1914 \n",
       "Q 372 2147 372 2553 \n",
       "Q 372 3047 722 3315 \n",
       "Q 1072 3584 1716 3584 \n",
       "Q 2034 3584 2315 3537 \n",
       "Q 2597 3491 2834 3397 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-4b\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-65\" x=\"60.576172\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-79\" x=\"122.099609\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-73\" x=\"181.279297\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_2\">\n",
       "    <g id=\"ytick_1\">\n",
       "     <g id=\"line2d_3\">\n",
       "      <defs>\n",
       "       <path id=\"me80e3b7c5f\" d=\"M 0 0 \n",
       "L -3.5 0 \n",
       "\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use xlink:href=\"#me80e3b7c5f\" x=\"34.240625\" y=\"43.2\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_4\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(20.878125 46.999219)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_2\">\n",
       "     <g id=\"line2d_4\">\n",
       "      <g>\n",
       "       <use xlink:href=\"#me80e3b7c5f\" x=\"34.240625\" y=\"54.36\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_5\">\n",
       "      <!-- 1 -->\n",
       "      <g transform=\"translate(20.878125 58.159219)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-31\" d=\"M 794 531 \n",
       "L 1825 531 \n",
       "L 1825 4091 \n",
       "L 703 3866 \n",
       "L 703 4441 \n",
       "L 1819 4666 \n",
       "L 2450 4666 \n",
       "L 2450 531 \n",
       "L 3481 531 \n",
       "L 3481 0 \n",
       "L 794 0 \n",
       "L 794 531 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_6\">\n",
       "     <!-- Queries -->\n",
       "     <g transform=\"translate(14.798437 68.087031)rotate(-90)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path id=\"DejaVuSans-51\" d=\"M 2522 4238 \n",
       "Q 1834 4238 1429 3725 \n",
       "Q 1025 3213 1025 2328 \n",
       "Q 1025 1447 1429 934 \n",
       "Q 1834 422 2522 422 \n",
       "Q 3209 422 3611 934 \n",
       "Q 4013 1447 4013 2328 \n",
       "Q 4013 3213 3611 3725 \n",
       "Q 3209 4238 2522 4238 \n",
       "z\n",
       "M 3406 84 \n",
       "L 4238 -825 \n",
       "L 3475 -825 \n",
       "L 2784 -78 \n",
       "Q 2681 -84 2626 -87 \n",
       "Q 2572 -91 2522 -91 \n",
       "Q 1538 -91 948 567 \n",
       "Q 359 1225 359 2328 \n",
       "Q 359 3434 948 4092 \n",
       "Q 1538 4750 2522 4750 \n",
       "Q 3503 4750 4090 4092 \n",
       "Q 4678 3434 4678 2328 \n",
       "Q 4678 1516 4351 937 \n",
       "Q 4025 359 3406 84 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-75\" d=\"M 544 1381 \n",
       "L 544 3500 \n",
       "L 1119 3500 \n",
       "L 1119 1403 \n",
       "Q 1119 906 1312 657 \n",
       "Q 1506 409 1894 409 \n",
       "Q 2359 409 2629 706 \n",
       "Q 2900 1003 2900 1516 \n",
       "L 2900 3500 \n",
       "L 3475 3500 \n",
       "L 3475 0 \n",
       "L 2900 0 \n",
       "L 2900 538 \n",
       "Q 2691 219 2414 64 \n",
       "Q 2138 -91 1772 -91 \n",
       "Q 1169 -91 856 284 \n",
       "Q 544 659 544 1381 \n",
       "z\n",
       "M 1991 3584 \n",
       "L 1991 3584 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-72\" d=\"M 2631 2963 \n",
       "Q 2534 3019 2420 3045 \n",
       "Q 2306 3072 2169 3072 \n",
       "Q 1681 3072 1420 2755 \n",
       "Q 1159 2438 1159 1844 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2956 \n",
       "Q 1341 3275 1631 3429 \n",
       "Q 1922 3584 2338 3584 \n",
       "Q 2397 3584 2469 3576 \n",
       "Q 2541 3569 2628 3553 \n",
       "L 2631 2963 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-69\" d=\"M 603 3500 \n",
       "L 1178 3500 \n",
       "L 1178 0 \n",
       "L 603 0 \n",
       "L 603 3500 \n",
       "z\n",
       "M 603 4863 \n",
       "L 1178 4863 \n",
       "L 1178 4134 \n",
       "L 603 4134 \n",
       "L 603 4863 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-51\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-75\" x=\"78.710938\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-65\" x=\"142.089844\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-72\" x=\"203.613281\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-69\" x=\"244.726562\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-65\" x=\"272.509766\"/>\n",
       "      <use xlink:href=\"#DejaVuSans-73\" x=\"334.033203\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"patch_3\">\n",
       "    <path d=\"M 34.240625 59.94 \n",
       "L 34.240625 37.62 \n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_4\">\n",
       "    <path d=\"M 145.840625 59.94 \n",
       "L 145.840625 37.62 \n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_5\">\n",
       "    <path d=\"M 34.240625 59.94 \n",
       "L 145.840625 59.94 \n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_6\">\n",
       "    <path d=\"M 34.240625 37.62 \n",
       "L 145.840625 37.62 \n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "  </g>\n",
       "  <g id=\"axes_2\">\n",
       "   <g id=\"patch_7\">\n",
       "    <path d=\"M 152.815625 90.36 \n",
       "L 156.973625 90.36 \n",
       "L 156.973625 7.2 \n",
       "L 152.815625 7.2 \n",
       "z\n",
       "\" style=\"fill: #ffffff\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_8\">\n",
       "    <path clip-path=\"url(#p6a25fffa4c)\" style=\"fill: #ffffff; stroke: #ffffff; stroke-width: 0.01; stroke-linejoin: miter\"/>\n",
       "   </g>\n",
       "   <image xlink:href=\"data:image/png;base64,\n",
       "iVBORw0KGgoAAAANSUhEUgAAAAYAAAB0CAYAAACmJkOCAAAAyElEQVR4nM2WSwrEMAxDXcj9zzqbblp/egG/gEII02WEJMt2Qq+6f2XNN6yyO7dhGQQQo1qLFY9ij30AB5Rz6B7uIqNQKkiKAUwuAyRV8e6SmjTxgAcy5An+qTkD+vrQNVgw15sI5qWXi909kLwCpTCgDPgB83dfuXpVFDAD3vakR18H3KEqWJKJVCBAyTEgSukBw2WP9thsJP0zOAEoFcjoz22EUQ5kyFVhwH5BZx4MkBQ2Ee7mhAEjX2E88sxRCluyYg4MquoD0XoQUGMmzEcAAAAASUVORK5CYII=\" id=\"imagee62e610223\" transform=\"scale(1 -1)translate(0 -83.52)\" x=\"152.64\" y=\"-6.48\" width=\"4.32\" height=\"83.52\"/>\n",
       "   <g id=\"matplotlib.axis_3\">\n",
       "    <g id=\"ytick_3\">\n",
       "     <g id=\"line2d_5\">\n",
       "      <defs>\n",
       "       <path id=\"md6437c271d\" d=\"M 0 0 \n",
       "L 3.5 0 \n",
       "\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use xlink:href=\"#md6437c271d\" x=\"156.973625\" y=\"90.36\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_7\">\n",
       "      <!-- 0.0 -->\n",
       "      <g transform=\"translate(163.973625 94.159219)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-2e\" d=\"M 684 794 \n",
       "L 1344 794 \n",
       "L 1344 0 \n",
       "L 684 0 \n",
       "L 684 794 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_4\">\n",
       "     <g id=\"line2d_6\">\n",
       "      <g>\n",
       "       <use xlink:href=\"#md6437c271d\" x=\"156.973625\" y=\"57.096\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_8\">\n",
       "      <!-- 0.2 -->\n",
       "      <g transform=\"translate(163.973625 60.895219)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-32\" d=\"M 1228 531 \n",
       "L 3431 531 \n",
       "L 3431 0 \n",
       "L 469 0 \n",
       "L 469 531 \n",
       "Q 828 903 1448 1529 \n",
       "Q 2069 2156 2228 2338 \n",
       "Q 2531 2678 2651 2914 \n",
       "Q 2772 3150 2772 3378 \n",
       "Q 2772 3750 2511 3984 \n",
       "Q 2250 4219 1831 4219 \n",
       "Q 1534 4219 1204 4116 \n",
       "Q 875 4013 500 3803 \n",
       "L 500 4441 \n",
       "Q 881 4594 1212 4672 \n",
       "Q 1544 4750 1819 4750 \n",
       "Q 2544 4750 2975 4387 \n",
       "Q 3406 4025 3406 3419 \n",
       "Q 3406 3131 3298 2873 \n",
       "Q 3191 2616 2906 2266 \n",
       "Q 2828 2175 2409 1742 \n",
       "Q 1991 1309 1228 531 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-32\" x=\"95.410156\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_5\">\n",
       "     <g id=\"line2d_7\">\n",
       "      <g>\n",
       "       <use xlink:href=\"#md6437c271d\" x=\"156.973625\" y=\"23.832\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_9\">\n",
       "      <!-- 0.4 -->\n",
       "      <g transform=\"translate(163.973625 27.631219)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path id=\"DejaVuSans-34\" d=\"M 2419 4116 \n",
       "L 825 1625 \n",
       "L 2419 1625 \n",
       "L 2419 4116 \n",
       "z\n",
       "M 2253 4666 \n",
       "L 3047 4666 \n",
       "L 3047 1625 \n",
       "L 3713 1625 \n",
       "L 3713 1100 \n",
       "L 3047 1100 \n",
       "L 3047 0 \n",
       "L 2419 0 \n",
       "L 2419 1100 \n",
       "L 313 1100 \n",
       "L 313 1709 \n",
       "L 2253 4666 \n",
       "z\n",
       "\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n",
       "       <use xlink:href=\"#DejaVuSans-34\" x=\"95.410156\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"LineCollection_1\"/>\n",
       "   <g id=\"patch_9\">\n",
       "    <path d=\"M 152.815625 90.36 \n",
       "L 154.894625 90.36 \n",
       "L 156.973625 90.36 \n",
       "L 156.973625 7.2 \n",
       "L 154.894625 7.2 \n",
       "L 152.815625 7.2 \n",
       "L 152.815625 90.36 \n",
       "z\n",
       "\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n",
       "   </g>\n",
       "  </g>\n",
       " </g>\n",
       " <defs>\n",
       "  <clipPath id=\"p772bf68c7c\">\n",
       "   <rect x=\"34.240625\" y=\"37.62\" width=\"111.6\" height=\"22.32\"/>\n",
       "  </clipPath>\n",
       "  <clipPath id=\"p6a25fffa4c\">\n",
       "   <rect x=\"152.815625\" y=\"7.2\" width=\"4.158\" height=\"83.16\"/>\n",
       "  </clipPath>\n",
       " </defs>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<Figure size 250x250 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 均匀的注意力权重\n",
    "d2l.show_heatmaps(attention.attention_weights.reshape((1,1,2,10)),\n",
    "                 xlabel='Keys', ylabel='Queries')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "2041.23px",
    "left": "45.9792px",
    "top": "56px",
    "width": "301.875px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}