ckpt_insight.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa64f4fb-fc62-429f-959f-316c13feafcd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "08966a59-7352-4665-95ed-fb28fd5ce9d6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ckpt_home = '/root/autodl-tmp/sfllm-32/main'\n",
    "!mkdir -p {ckpt_home}\n",
    "ckpt_rep_path1 = f'{ckpt_home}/zero_pp_rank_0_mp_rank_00_optim_states.pt'\n",
    "ckpt_rep_path2 = f'{ckpt_home}/zero_pp_rank_1_mp_rank_00_optim_states.pt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "76c6c0d2-66c0-418b-bd90-6e9daf81b361",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'module': OrderedDict([('rope_emb.phase',\n",
       "               tensor([[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00, 0.0000e+00,\n",
       "                        0.0000e+00],\n",
       "                       [1.0000e+00, 8.3154e-01, 6.9189e-01,  ..., 1.7381e-04, 1.4460e-04,\n",
       "                        1.2022e-04],\n",
       "                       [2.0000e+00, 1.6631e+00, 1.3838e+00,  ..., 3.4761e-04, 2.8920e-04,\n",
       "                        2.4045e-04],\n",
       "                       ...,\n",
       "                       [1.0210e+03, 8.4900e+02, 7.0650e+02,  ..., 1.7749e-01, 1.4758e-01,\n",
       "                        1.2274e-01],\n",
       "                       [1.0220e+03, 8.5000e+02, 7.0700e+02,  ..., 1.7761e-01, 1.4771e-01,\n",
       "                        1.2286e-01],\n",
       "                       [1.0230e+03, 8.5100e+02, 7.0750e+02,  ..., 1.7773e-01, 1.4783e-01,\n",
       "                        1.2299e-01]], dtype=torch.float16)),\n",
       "              ('emb.weight',\n",
       "               tensor([[-0.0995,  0.2800,  0.0915,  ...,  0.5576,  1.0977,  0.3770],\n",
       "                       [ 1.4385,  1.7871, -1.1836,  ..., -0.2036, -1.3232, -0.0825],\n",
       "                       [ 0.8228, -0.9536, -0.5073,  ..., -0.1281,  0.9331,  2.0312],\n",
       "                       ...,\n",
       "                       [ 0.6167,  0.1750,  1.0518,  ...,  1.0078,  1.1406, -0.6108],\n",
       "                       [ 0.2233, -2.4863, -0.8384,  ..., -0.2500,  0.1692, -1.6982],\n",
       "                       [-1.1650, -1.6494,  1.9277,  ..., -0.2395,  1.0967,  1.1826]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.attn.proj.weight',\n",
       "               tensor([[ 0.0082, -0.0099,  0.0049,  ...,  0.0171,  0.0163, -0.0023],\n",
       "                       [-0.0182, -0.0153,  0.0134,  ...,  0.0152,  0.0109, -0.0023],\n",
       "                       [-0.0015, -0.0113, -0.0033,  ..., -0.0091,  0.0161,  0.0034],\n",
       "                       ...,\n",
       "                       [ 0.0171,  0.0128,  0.0088,  ..., -0.0084,  0.0016, -0.0128],\n",
       "                       [-0.0046,  0.0057, -0.0025,  ...,  0.0122,  0.0139, -0.0113],\n",
       "                       [-0.0077,  0.0022,  0.0112,  ..., -0.0178,  0.0141,  0.0218]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.attn.proj.bias',\n",
       "               tensor([-0.0097,  0.0150,  0.0096,  ..., -0.0160, -0.0211,  0.0197],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.attn.ff.weight',\n",
       "               tensor([[ 0.0050,  0.0026,  0.0017,  ...,  0.0041,  0.0126,  0.0022],\n",
       "                       [-0.0100, -0.0063, -0.0014,  ..., -0.0116, -0.0003,  0.0018],\n",
       "                       [ 0.0084,  0.0117,  0.0091,  ..., -0.0093,  0.0067, -0.0043],\n",
       "                       ...,\n",
       "                       [ 0.0037,  0.0057, -0.0037,  ..., -0.0104,  0.0053,  0.0097],\n",
       "                       [ 0.0035, -0.0070, -0.0022,  ..., -0.0107,  0.0102,  0.0155],\n",
       "                       [ 0.0146,  0.0061,  0.0111,  ..., -0.0059, -0.0004,  0.0032]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.attn.ff.bias',\n",
       "               tensor([-0.0007,  0.0171,  0.0039,  ...,  0.0128,  0.0084,  0.0125],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.ln1.w',\n",
       "               tensor([0.9756, 0.9883, 0.9790,  ..., 0.9771, 0.9727, 0.9766],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.ln1.b',\n",
       "               tensor([-0.0027, -0.0019,  0.0036,  ...,  0.0018,  0.0044,  0.0025],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.ln2.w',\n",
       "               tensor([0.9927, 0.9927, 0.9893,  ..., 0.9937, 0.9897, 1.0000],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.ln2.b',\n",
       "               tensor([ 0.0018,  0.0110, -0.0106,  ...,  0.0131,  0.0175,  0.0082],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.mlp.ff1.weight',\n",
       "               tensor([[ 7.3395e-03,  2.7103e-03,  1.8158e-02,  ...,  1.1162e-02,\n",
       "                         6.7863e-03,  7.4921e-03],\n",
       "                       [-1.6586e-02,  1.7487e-02,  4.6616e-03,  ..., -7.5226e-03,\n",
       "                        -1.0719e-02, -1.8066e-02],\n",
       "                       [-1.3232e-05, -1.9531e-02, -9.0866e-03,  ...,  1.5764e-03,\n",
       "                        -2.5349e-03,  2.5063e-03],\n",
       "                       ...,\n",
       "                       [ 4.1580e-03, -1.5617e-02, -6.3121e-05,  ..., -4.1351e-03,\n",
       "                        -1.5915e-02,  1.3227e-03],\n",
       "                       [ 3.1013e-03,  1.4420e-02, -8.9798e-03,  ..., -6.8550e-03,\n",
       "                        -7.1478e-04, -3.2539e-03],\n",
       "                       [ 5.8770e-05, -4.3602e-03, -7.7667e-03,  ..., -1.0849e-02,\n",
       "                         1.5869e-02, -1.6994e-03]], dtype=torch.float16)),\n",
       "              ('blocks.0.mlp.ff1.bias',\n",
       "               tensor([-0.0106,  0.0022, -0.0039,  ..., -0.0123, -0.0086,  0.0114],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.mlp.ff2.weight',\n",
       "               tensor([[ 0.0017, -0.0041,  0.0050,  ...,  0.0048,  0.0021, -0.0004],\n",
       "                       [ 0.0004, -0.0058,  0.0051,  ...,  0.0075, -0.0064,  0.0061],\n",
       "                       [ 0.0035,  0.0068,  0.0008,  ..., -0.0050,  0.0098, -0.0062],\n",
       "                       ...,\n",
       "                       [ 0.0094, -0.0008,  0.0047,  ...,  0.0012, -0.0104, -0.0015],\n",
       "                       [ 0.0064, -0.0077,  0.0035,  ...,  0.0096,  0.0026, -0.0071],\n",
       "                       [ 0.0069, -0.0074, -0.0032,  ..., -0.0060, -0.0045,  0.0077]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.0.mlp.ff2.bias',\n",
       "               tensor([-0.0038,  0.0033,  0.0036,  ...,  0.0038,  0.0028,  0.0027],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.attn.proj.weight',\n",
       "               tensor([[ 0.0211,  0.0071, -0.0074,  ...,  0.0227, -0.0151,  0.0042],\n",
       "                       [-0.0116, -0.0135,  0.0124,  ...,  0.0136, -0.0106,  0.0091],\n",
       "                       [ 0.0159,  0.0152,  0.0101,  ..., -0.0082,  0.0033,  0.0130],\n",
       "                       ...,\n",
       "                       [ 0.0102,  0.0061, -0.0102,  ..., -0.0076,  0.0101, -0.0115],\n",
       "                       [ 0.0121,  0.0060,  0.0143,  ..., -0.0106, -0.0131,  0.0105],\n",
       "                       [-0.0043, -0.0040,  0.0104,  ..., -0.0058, -0.0113,  0.0010]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.attn.proj.bias',\n",
       "               tensor([-0.0073, -0.0024,  0.0064,  ...,  0.0093,  0.0169,  0.0068],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.attn.ff.weight',\n",
       "               tensor([[ 0.0041, -0.0168,  0.0148,  ...,  0.0115, -0.0108, -0.0168],\n",
       "                       [-0.0042, -0.0008,  0.0070,  ...,  0.0010,  0.0205, -0.0148],\n",
       "                       [ 0.0021, -0.0132, -0.0045,  ...,  0.0189, -0.0102, -0.0012],\n",
       "                       ...,\n",
       "                       [-0.0053,  0.0084,  0.0131,  ...,  0.0144, -0.0100,  0.0108],\n",
       "                       [-0.0055, -0.0153, -0.0032,  ..., -0.0034,  0.0077,  0.0131],\n",
       "                       [-0.0111,  0.0086,  0.0050,  ...,  0.0151,  0.0124, -0.0164]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.attn.ff.bias',\n",
       "               tensor([-0.0005, -0.0016,  0.0167,  ...,  0.0195, -0.0091, -0.0086],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.ln1.w',\n",
       "               tensor([0.9790, 0.9868, 0.9814,  ..., 0.9844, 0.9746, 0.9790],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.ln1.b',\n",
       "               tensor([-0.0010, -0.0044, -0.0011,  ..., -0.0056, -0.0055, -0.0055],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.ln2.w',\n",
       "               tensor([1.0068, 0.9995, 1.0059,  ..., 1.0098, 0.9971, 1.0127],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.ln2.b',\n",
       "               tensor([0.0089, 0.0066, 0.0051,  ..., 0.0138, 0.0013, 0.0083],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.mlp.ff1.weight',\n",
       "               tensor([[-0.0130,  0.0133,  0.0095,  ..., -0.0051,  0.0065, -0.0136],\n",
       "                       [-0.0178,  0.0051,  0.0076,  ..., -0.0189, -0.0037, -0.0061],\n",
       "                       [-0.0200, -0.0073,  0.0118,  ...,  0.0103,  0.0127, -0.0166],\n",
       "                       ...,\n",
       "                       [ 0.0033, -0.0069,  0.0111,  ..., -0.0024, -0.0060,  0.0006],\n",
       "                       [-0.0177,  0.0050, -0.0103,  ..., -0.0043, -0.0074,  0.0128],\n",
       "                       [-0.0004,  0.0026, -0.0175,  ...,  0.0053,  0.0055,  0.0061]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.mlp.ff1.bias',\n",
       "               tensor([-0.0110,  0.0112, -0.0047,  ..., -0.0085,  0.0061, -0.0110],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.mlp.ff2.weight',\n",
       "               tensor([[-0.0004,  0.0109,  0.0082,  ...,  0.0039,  0.0088, -0.0050],\n",
       "                       [-0.0092, -0.0045,  0.0014,  ..., -0.0094, -0.0086, -0.0063],\n",
       "                       [-0.0054,  0.0026, -0.0063,  ..., -0.0137, -0.0063,  0.0021],\n",
       "                       ...,\n",
       "                       [ 0.0065,  0.0083, -0.0054,  ...,  0.0004, -0.0046,  0.0022],\n",
       "                       [-0.0035,  0.0140,  0.0019,  ...,  0.0078, -0.0057,  0.0021],\n",
       "                       [-0.0064, -0.0026, -0.0059,  ...,  0.0002, -0.0106,  0.0092]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.1.mlp.ff2.bias',\n",
       "               tensor([-0.0002,  0.0018,  0.0064,  ..., -0.0090, -0.0013,  0.0010],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.attn.proj.weight',\n",
       "               tensor([[-0.0129, -0.0050,  0.0124,  ...,  0.0019, -0.0181,  0.0144],\n",
       "                       [-0.0121, -0.0150, -0.0093,  ..., -0.0019, -0.0117, -0.0055],\n",
       "                       [-0.0074, -0.0015, -0.0037,  ..., -0.0064, -0.0026,  0.0107],\n",
       "                       ...,\n",
       "                       [ 0.0073,  0.0127,  0.0045,  ..., -0.0187,  0.0179, -0.0073],\n",
       "                       [-0.0020, -0.0023, -0.0026,  ..., -0.0140,  0.0089,  0.0184],\n",
       "                       [-0.0149, -0.0114, -0.0134,  ..., -0.0150,  0.0165,  0.0114]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.attn.proj.bias',\n",
       "               tensor([ 0.0079, -0.0150,  0.0090,  ..., -0.0019,  0.0112, -0.0049],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.attn.ff.weight',\n",
       "               tensor([[ 0.0115,  0.0111,  0.0027,  ...,  0.0070,  0.0070,  0.0026],\n",
       "                       [-0.0025,  0.0108,  0.0044,  ..., -0.0068,  0.0111,  0.0130],\n",
       "                       [-0.0129, -0.0051, -0.0065,  ..., -0.0012,  0.0030,  0.0059],\n",
       "                       ...,\n",
       "                       [ 0.0001,  0.0058, -0.0025,  ...,  0.0027, -0.0147, -0.0035],\n",
       "                       [-0.0040, -0.0037,  0.0108,  ...,  0.0048,  0.0090,  0.0029],\n",
       "                       [ 0.0158, -0.0202, -0.0079,  ..., -0.0009,  0.0010,  0.0065]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.attn.ff.bias',\n",
       "               tensor([-0.0120, -0.0084,  0.0177,  ...,  0.0089,  0.0129, -0.0085],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.ln1.w',\n",
       "               tensor([0.9829, 0.9883, 0.9902,  ..., 0.9912, 0.9824, 0.9893],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.ln1.b',\n",
       "               tensor([-0.0020, -0.0035, -0.0010,  ..., -0.0036, -0.0010, -0.0054],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.ln2.w',\n",
       "               tensor([1.0039, 1.0049, 1.0098,  ..., 1.0107, 1.0000, 1.0166],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.ln2.b',\n",
       "               tensor([ 0.0027,  0.0054,  0.0063,  ...,  0.0044, -0.0030,  0.0066],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.mlp.ff1.weight',\n",
       "               tensor([[ 0.0123,  0.0086,  0.0033,  ...,  0.0073,  0.0050, -0.0097],\n",
       "                       [-0.0094,  0.0132, -0.0127,  ..., -0.0085, -0.0031, -0.0034],\n",
       "                       [ 0.0043,  0.0187,  0.0049,  ..., -0.0089, -0.0118,  0.0067],\n",
       "                       ...,\n",
       "                       [-0.0061, -0.0064, -0.0183,  ...,  0.0122,  0.0177, -0.0138],\n",
       "                       [ 0.0171, -0.0001,  0.0093,  ...,  0.0166,  0.0066, -0.0062],\n",
       "                       [-0.0136,  0.0048,  0.0123,  ..., -0.0089, -0.0110,  0.0098]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.mlp.ff1.bias',\n",
       "               tensor([-0.0102,  0.0013,  0.0056,  ...,  0.0064,  0.0131, -0.0069],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.mlp.ff2.weight',\n",
       "               tensor([[ 0.0036,  0.0012,  0.0089,  ..., -0.0018,  0.0019, -0.0074],\n",
       "                       [-0.0004,  0.0009, -0.0054,  ...,  0.0009, -0.0018,  0.0092],\n",
       "                       [-0.0060, -0.0030,  0.0059,  ..., -0.0039,  0.0010, -0.0124],\n",
       "                       ...,\n",
       "                       [-0.0003, -0.0004,  0.0026,  ..., -0.0074, -0.0035, -0.0054],\n",
       "                       [-0.0040,  0.0045,  0.0037,  ..., -0.0062, -0.0074,  0.0050],\n",
       "                       [ 0.0009, -0.0003, -0.0034,  ...,  0.0091, -0.0051, -0.0002]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.2.mlp.ff2.bias',\n",
       "               tensor([-0.0024, -0.0003,  0.0047,  ..., -0.0089,  0.0041,  0.0037],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.attn.proj.weight',\n",
       "               tensor([[-0.0129, -0.0079,  0.0117,  ...,  0.0014,  0.0063,  0.0008],\n",
       "                       [-0.0207,  0.0009,  0.0087,  ...,  0.0135, -0.0229, -0.0032],\n",
       "                       [ 0.0113,  0.0176,  0.0083,  ..., -0.0160,  0.0036,  0.0111],\n",
       "                       ...,\n",
       "                       [ 0.0162,  0.0020, -0.0105,  ...,  0.0095, -0.0092, -0.0010],\n",
       "                       [-0.0096, -0.0094,  0.0046,  ..., -0.0048,  0.0176, -0.0177],\n",
       "                       [-0.0016, -0.0108, -0.0170,  ...,  0.0025, -0.0130,  0.0082]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.attn.proj.bias',\n",
       "               tensor([ 0.0069, -0.0031,  0.0076,  ..., -0.0039,  0.0011, -0.0167],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.attn.ff.weight',\n",
       "               tensor([[-0.0060,  0.0003, -0.0046,  ...,  0.0220,  0.0115,  0.0120],\n",
       "                       [-0.0065, -0.0056,  0.0125,  ..., -0.0120, -0.0214, -0.0078],\n",
       "                       [-0.0174, -0.0012,  0.0034,  ..., -0.0049, -0.0038, -0.0022],\n",
       "                       ...,\n",
       "                       [-0.0043,  0.0042, -0.0060,  ..., -0.0041, -0.0017,  0.0011],\n",
       "                       [-0.0072, -0.0044,  0.0125,  ..., -0.0079, -0.0054,  0.0048],\n",
       "                       [ 0.0015, -0.0037, -0.0102,  ..., -0.0003, -0.0062,  0.0002]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.attn.ff.bias',\n",
       "               tensor([ 0.0105, -0.0044,  0.0106,  ...,  0.0129,  0.0005, -0.0144],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.ln1.w',\n",
       "               tensor([0.9878, 0.9883, 0.9873,  ..., 0.9854, 0.9844, 0.9873],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.ln1.b',\n",
       "               tensor([-0.0006, -0.0024, -0.0003,  ..., -0.0036,  0.0019, -0.0053],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.ln2.w',\n",
       "               tensor([1.0039, 1.0039, 1.0068,  ..., 1.0068, 1.0107, 1.0137],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.ln2.b',\n",
       "               tensor([ 0.0020,  0.0038,  0.0023,  ...,  0.0010, -0.0022,  0.0038],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.mlp.ff1.weight',\n",
       "               tensor([[-0.0069,  0.0052,  0.0051,  ...,  0.0054, -0.0074, -0.0110],\n",
       "                       [-0.0135, -0.0052,  0.0078,  ..., -0.0045,  0.0046, -0.0190],\n",
       "                       [ 0.0034,  0.0041,  0.0203,  ..., -0.0005,  0.0093,  0.0079],\n",
       "                       ...,\n",
       "                       [ 0.0144,  0.0100,  0.0130,  ...,  0.0078,  0.0190,  0.0105],\n",
       "                       [ 0.0046, -0.0042,  0.0101,  ...,  0.0011,  0.0085, -0.0176],\n",
       "                       [-0.0032,  0.0070,  0.0008,  ...,  0.0100,  0.0070, -0.0059]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.mlp.ff1.bias',\n",
       "               tensor([-0.0138,  0.0055,  0.0103,  ...,  0.0141,  0.0062,  0.0033],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.mlp.ff2.weight',\n",
       "               tensor([[-0.0061,  0.0046, -0.0034,  ..., -0.0023, -0.0106,  0.0077],\n",
       "                       [-0.0046, -0.0015,  0.0030,  ..., -0.0090, -0.0051, -0.0082],\n",
       "                       [ 0.0071,  0.0005, -0.0070,  ..., -0.0004,  0.0067,  0.0066],\n",
       "                       ...,\n",
       "                       [-0.0042,  0.0012, -0.0066,  ..., -0.0110, -0.0088, -0.0066],\n",
       "                       [ 0.0028,  0.0029,  0.0061,  ..., -0.0039,  0.0023, -0.0047],\n",
       "                       [ 0.0034, -0.0006,  0.0042,  ...,  0.0034,  0.0019,  0.0124]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.3.mlp.ff2.bias',\n",
       "               tensor([-0.0035, -0.0059,  0.0061,  ..., -0.0028,  0.0025, -0.0061],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.attn.proj.weight',\n",
       "               tensor([[-1.0109e-02,  9.8495e-03,  1.4114e-02,  ..., -6.9008e-03,\n",
       "                         1.8740e-03, -1.6663e-02],\n",
       "                       [-7.7744e-03, -1.3901e-02, -7.4744e-05,  ..., -9.3918e-03,\n",
       "                         7.6561e-03, -2.3880e-02],\n",
       "                       [-8.3237e-03,  3.0880e-03, -9.9030e-03,  ..., -4.4632e-03,\n",
       "                         2.8172e-03, -2.4933e-02],\n",
       "                       ...,\n",
       "                       [-3.7117e-03, -9.9182e-03, -1.9112e-03,  ..., -3.0022e-03,\n",
       "                         1.1505e-02, -2.8191e-03],\n",
       "                       [-1.3008e-02,  5.1041e-03, -6.6223e-03,  ..., -1.4687e-02,\n",
       "                        -1.9169e-03, -1.4854e-02],\n",
       "                       [ 3.9749e-03,  1.3170e-03,  9.8495e-03,  ...,  9.2239e-03,\n",
       "                        -5.3520e-03,  1.9178e-03]], dtype=torch.float16)),\n",
       "              ('blocks.4.attn.proj.bias',\n",
       "               tensor([-0.0148,  0.0027,  0.0009,  ..., -0.0107,  0.0116,  0.0096],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.attn.ff.weight',\n",
       "               tensor([[ 0.0035, -0.0190,  0.0095,  ..., -0.0048,  0.0066,  0.0148],\n",
       "                       [ 0.0178, -0.0059, -0.0069,  ...,  0.0184, -0.0088, -0.0067],\n",
       "                       [-0.0175, -0.0097, -0.0084,  ...,  0.0086,  0.0052,  0.0093],\n",
       "                       ...,\n",
       "                       [ 0.0040, -0.0081, -0.0058,  ...,  0.0079,  0.0003, -0.0095],\n",
       "                       [-0.0106,  0.0098,  0.0140,  ...,  0.0089,  0.0084, -0.0112],\n",
       "                       [-0.0069, -0.0074, -0.0182,  ...,  0.0071, -0.0023,  0.0022]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.attn.ff.bias',\n",
       "               tensor([-0.0163,  0.0104,  0.0048,  ...,  0.0094, -0.0179, -0.0039],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.ln1.w',\n",
       "               tensor([0.9907, 0.9849, 0.9917,  ..., 0.9937, 0.9834, 0.9917],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.ln1.b',\n",
       "               tensor([ 0.0005, -0.0042,  0.0002,  ...,  0.0005,  0.0011, -0.0019],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.ln2.w',\n",
       "               tensor([1.0137, 1.0156, 1.0117,  ..., 1.0088, 1.0117, 1.0088],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.ln2.b',\n",
       "               tensor([-0.0011, -0.0023, -0.0007,  ..., -0.0001, -0.0022, -0.0007],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.mlp.ff1.weight',\n",
       "               tensor([[ 0.0056,  0.0157, -0.0009,  ..., -0.0012, -0.0154, -0.0086],\n",
       "                       [ 0.0121,  0.0173, -0.0046,  ...,  0.0081,  0.0034,  0.0183],\n",
       "                       [-0.0061, -0.0180,  0.0068,  ...,  0.0054,  0.0023,  0.0084],\n",
       "                       ...,\n",
       "                       [ 0.0115, -0.0161,  0.0048,  ..., -0.0140,  0.0080,  0.0180],\n",
       "                       [-0.0112,  0.0094,  0.0036,  ...,  0.0167,  0.0165,  0.0088],\n",
       "                       [ 0.0068,  0.0081,  0.0009,  ..., -0.0235,  0.0207, -0.0086]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.mlp.ff1.bias',\n",
       "               tensor([-0.0145, -0.0139,  0.0062,  ...,  0.0056, -0.0010,  0.0112],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.4.mlp.ff2.weight',\n",
       "               tensor([[ 7.3738e-03,  9.5701e-04, -4.1504e-03,  ..., -6.2218e-03,\n",
       "                        -2.9984e-03,  1.9681e-04],\n",
       "                       [-2.6779e-03, -1.4181e-03,  1.6336e-03,  ..., -5.6725e-03,\n",
       "                        -4.4136e-03, -1.1435e-03],\n",
       "                       [ 6.5956e-03, -3.7980e-04, -6.7520e-03,  ..., -2.8381e-03,\n",
       "                         1.7834e-03,  4.7112e-03],\n",
       "                       ...,\n",
       "                       [-6.3591e-03,  1.2383e-02, -1.0841e-02,  ...,  8.1329e-03,\n",
       "                        -3.3073e-03, -2.0542e-03],\n",
       "                       [-3.4161e-03, -5.0020e-04,  1.0544e-02,  ..., -9.4910e-03,\n",
       "                        -2.5711e-03, -1.1536e-02],\n",
       "                       [ 6.3972e-03, -5.4121e-05, -8.7547e-04,  ..., -6.4964e-03,\n",
       "                        -8.9569e-03,  1.6098e-03]], dtype=torch.float16)),\n",
       "              ('blocks.4.mlp.ff2.bias',\n",
       "               tensor([-0.0056,  0.0046,  0.0043,  ..., -0.0018, -0.0001,  0.0045],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.attn.proj.weight',\n",
       "               tensor([[ 0.0165, -0.0189, -0.0089,  ..., -0.0069,  0.0003,  0.0102],\n",
       "                       [-0.0061, -0.0224, -0.0120,  ...,  0.0045, -0.0112,  0.0066],\n",
       "                       [-0.0012,  0.0154, -0.0243,  ...,  0.0092, -0.0075,  0.0227],\n",
       "                       ...,\n",
       "                       [-0.0063, -0.0121,  0.0103,  ..., -0.0143, -0.0127,  0.0153],\n",
       "                       [-0.0022,  0.0174, -0.0129,  ...,  0.0119,  0.0064,  0.0098],\n",
       "                       [ 0.0008,  0.0098,  0.0011,  ...,  0.0002,  0.0147,  0.0162]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.attn.proj.bias',\n",
       "               tensor([-0.0064, -0.0104,  0.0184,  ...,  0.0148, -0.0028,  0.0102],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.attn.ff.weight',\n",
       "               tensor([[-1.6861e-02,  1.4580e-02,  1.6220e-02,  ...,  3.7632e-03,\n",
       "                         5.8746e-03, -4.6501e-03],\n",
       "                       [-2.1935e-03,  1.1230e-02,  1.2405e-02,  ...,  2.7955e-05,\n",
       "                         8.2321e-03,  1.5945e-02],\n",
       "                       [ 1.2360e-03,  1.5312e-02, -1.7654e-02,  ...,  6.5193e-03,\n",
       "                         1.6708e-02,  4.5967e-03],\n",
       "                       ...,\n",
       "                       [ 4.6616e-03, -6.4507e-03, -5.7182e-03,  ...,  4.8141e-03,\n",
       "                         8.2932e-03, -9.7885e-03],\n",
       "                       [-1.1940e-02, -4.4250e-03, -4.6654e-03,  ..., -1.2970e-03,\n",
       "                         1.5404e-02,  5.1804e-03],\n",
       "                       [-5.9586e-03,  5.8022e-03, -1.1719e-02,  ...,  1.0796e-02,\n",
       "                        -9.7656e-03, -1.6144e-02]], dtype=torch.float16)),\n",
       "              ('blocks.5.attn.ff.bias',\n",
       "               tensor([ 0.0009, -0.0160, -0.0086,  ...,  0.0030, -0.0140,  0.0172],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.ln1.w',\n",
       "               tensor([0.9937, 0.9878, 0.9976,  ..., 0.9907, 0.9868, 0.9888],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.ln1.b',\n",
       "               tensor([-0.0015, -0.0032,  0.0029,  ...,  0.0004,  0.0011,  0.0002],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.ln2.w',\n",
       "               tensor([1.0146, 1.0166, 1.0225,  ..., 1.0176, 1.0146, 1.0166],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.ln2.b',\n",
       "               tensor([-0.0014, -0.0038, -0.0056,  ..., -0.0001, -0.0020, -0.0008],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.mlp.ff1.weight',\n",
       "               tensor([[ 0.0020,  0.0024,  0.0008,  ...,  0.0154, -0.0100, -0.0143],\n",
       "                       [-0.0042,  0.0118, -0.0081,  ..., -0.0181,  0.0092,  0.0108],\n",
       "                       [ 0.0065,  0.0046,  0.0145,  ..., -0.0195,  0.0011, -0.0008],\n",
       "                       ...,\n",
       "                       [ 0.0126,  0.0051,  0.0127,  ...,  0.0041,  0.0232, -0.0067],\n",
       "                       [ 0.0014,  0.0148, -0.0058,  ..., -0.0214,  0.0013, -0.0085],\n",
       "                       [-0.0002, -0.0136,  0.0122,  ..., -0.0089,  0.0197,  0.0076]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.mlp.ff1.bias',\n",
       "               tensor([-0.0158,  0.0147,  0.0038,  ..., -0.0017, -0.0047,  0.0060],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.5.mlp.ff2.weight',\n",
       "               tensor([[ 6.1417e-03,  3.3169e-03,  7.2718e-04,  ..., -8.9931e-04,\n",
       "                         8.8501e-03,  4.9734e-04],\n",
       "                       [-4.5547e-03, -6.4621e-03,  5.9986e-04,  ..., -1.9131e-03,\n",
       "                        -3.2063e-03,  6.3782e-03],\n",
       "                       [-1.5783e-03,  5.8670e-03,  3.6163e-03,  ..., -6.1760e-03,\n",
       "                        -7.3128e-03,  2.4414e-03],\n",
       "                       ...,\n",
       "                       [-3.7766e-03, -1.3542e-03, -1.4963e-03,  ...,  4.7624e-05,\n",
       "                        -6.5575e-03,  7.0763e-03],\n",
       "                       [-2.7962e-03, -5.4169e-04, -6.5155e-03,  ..., -1.0963e-02,\n",
       "                        -5.3902e-03,  4.8327e-04],\n",
       "                       [ 2.3537e-03, -1.9150e-03,  1.5593e-03,  ...,  2.6646e-03,\n",
       "                         2.9907e-03,  7.6752e-03]], dtype=torch.float16)),\n",
       "              ('blocks.5.mlp.ff2.bias',\n",
       "               tensor([7.0343e-03, 3.6850e-03, 1.7815e-03,  ..., 2.9621e-03, 4.6074e-05,\n",
       "                       8.5602e-03], dtype=torch.float16)),\n",
       "              ('blocks.6.attn.proj.weight',\n",
       "               tensor([[ 0.0078, -0.0019, -0.0090,  ...,  0.0065,  0.0110,  0.0159],\n",
       "                       [-0.0053,  0.0094,  0.0131,  ..., -0.0200,  0.0116,  0.0265],\n",
       "                       [-0.0082,  0.0138,  0.0145,  ...,  0.0214,  0.0098, -0.0118],\n",
       "                       ...,\n",
       "                       [ 0.0081, -0.0018,  0.0057,  ...,  0.0050,  0.0057,  0.0024],\n",
       "                       [ 0.0172,  0.0007, -0.0028,  ..., -0.0119,  0.0071,  0.0088],\n",
       "                       [ 0.0145, -0.0157, -0.0034,  ...,  0.0054,  0.0060,  0.0121]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.attn.proj.bias',\n",
       "               tensor([ 0.0164, -0.0069,  0.0153,  ..., -0.0054, -0.0157, -0.0043],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.attn.ff.weight',\n",
       "               tensor([[ 9.8648e-03,  1.0023e-03, -1.0979e-02,  ..., -1.4023e-02,\n",
       "                        -1.5144e-02,  1.2465e-03],\n",
       "                       [-7.5417e-03,  2.9068e-03, -7.4539e-03,  ...,  1.9670e-04,\n",
       "                         1.0445e-02, -7.5302e-03],\n",
       "                       [-1.0002e-02,  3.2406e-03,  8.5983e-03,  ...,  7.6065e-03,\n",
       "                         7.8049e-03,  9.4604e-03],\n",
       "                       ...,\n",
       "                       [ 5.6725e-03, -1.3496e-02, -1.5381e-02,  ...,  6.9916e-05,\n",
       "                        -1.0300e-02,  7.8659e-03],\n",
       "                       [-2.0351e-03, -2.1591e-03, -3.3989e-03,  ..., -1.5732e-02,\n",
       "                        -8.5526e-03, -1.3466e-02],\n",
       "                       [-6.5765e-03, -2.4681e-03,  9.4910e-03,  ..., -1.5434e-02,\n",
       "                        -1.6800e-02, -7.3395e-03]], dtype=torch.float16)),\n",
       "              ('blocks.6.attn.ff.bias',\n",
       "               tensor([-0.0121,  0.0132, -0.0031,  ..., -0.0058, -0.0005, -0.0163],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.ln1.w',\n",
       "               tensor([0.9819, 0.9849, 0.9854,  ..., 0.9893, 0.9795, 0.9917],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.ln1.b',\n",
       "               tensor([-1.3161e-03, -8.7261e-04, -1.4257e-03,  ..., -1.4067e-05,\n",
       "                       -1.5106e-03,  2.7790e-03], dtype=torch.float16)),\n",
       "              ('blocks.6.ln2.w',\n",
       "               tensor([1.0146, 1.0146, 1.0205,  ..., 1.0156, 1.0146, 1.0156],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.ln2.b',\n",
       "               tensor([ 0.0012, -0.0024, -0.0034,  ...,  0.0002, -0.0023,  0.0033],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.mlp.ff1.weight',\n",
       "               tensor([[-0.0200, -0.0180, -0.0156,  ..., -0.0061,  0.0058, -0.0166],\n",
       "                       [ 0.0140, -0.0099, -0.0162,  ..., -0.0086, -0.0091, -0.0101],\n",
       "                       [-0.0064, -0.0013, -0.0013,  ...,  0.0022,  0.0078, -0.0042],\n",
       "                       ...,\n",
       "                       [-0.0086,  0.0200,  0.0087,  ...,  0.0013,  0.0094, -0.0197],\n",
       "                       [ 0.0019, -0.0015,  0.0156,  ...,  0.0050, -0.0009,  0.0056],\n",
       "                       [-0.0150, -0.0027, -0.0035,  ...,  0.0043, -0.0123, -0.0158]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.mlp.ff1.bias',\n",
       "               tensor([ 0.0028, -0.0176,  0.0121,  ...,  0.0045,  0.0044, -0.0152],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.mlp.ff2.weight',\n",
       "               tensor([[ 0.0075, -0.0048, -0.0024,  ...,  0.0008, -0.0041,  0.0058],\n",
       "                       [ 0.0061,  0.0033, -0.0014,  ..., -0.0115,  0.0065,  0.0098],\n",
       "                       [ 0.0041,  0.0045,  0.0065,  ...,  0.0028,  0.0054, -0.0056],\n",
       "                       ...,\n",
       "                       [ 0.0073,  0.0085, -0.0047,  ...,  0.0049, -0.0041,  0.0043],\n",
       "                       [-0.0055,  0.0060, -0.0017,  ..., -0.0004,  0.0021, -0.0042],\n",
       "                       [-0.0026, -0.0060, -0.0053,  ...,  0.0052, -0.0002,  0.0027]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.6.mlp.ff2.bias',\n",
       "               tensor([-0.0050,  0.0078, -0.0075,  ...,  0.0031,  0.0077,  0.0027],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.attn.proj.weight',\n",
       "               tensor([[-0.0049,  0.0158, -0.0041,  ...,  0.0145, -0.0035,  0.0087],\n",
       "                       [-0.0182,  0.0014, -0.0085,  ...,  0.0055, -0.0106,  0.0021],\n",
       "                       [ 0.0006,  0.0061,  0.0093,  ..., -0.0023, -0.0104, -0.0071],\n",
       "                       ...,\n",
       "                       [-0.0140, -0.0010,  0.0094,  ..., -0.0078, -0.0146,  0.0011],\n",
       "                       [ 0.0038, -0.0126,  0.0094,  ...,  0.0049, -0.0104,  0.0030],\n",
       "                       [ 0.0015, -0.0071,  0.0145,  ...,  0.0039, -0.0130, -0.0147]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.attn.proj.bias',\n",
       "               tensor([ 0.0128, -0.0046, -0.0076,  ..., -0.0168, -0.0074,  0.0143],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.attn.ff.weight',\n",
       "               tensor([[-0.0006,  0.0151, -0.0186,  ...,  0.0031,  0.0153,  0.0111],\n",
       "                       [-0.0095, -0.0010,  0.0034,  ...,  0.0023,  0.0055,  0.0119],\n",
       "                       [ 0.0227, -0.0084, -0.0067,  ...,  0.0091, -0.0117,  0.0028],\n",
       "                       ...,\n",
       "                       [-0.0094,  0.0077,  0.0111,  ..., -0.0020, -0.0013,  0.0117],\n",
       "                       [-0.0042,  0.0078, -0.0125,  ...,  0.0092,  0.0039,  0.0082],\n",
       "                       [ 0.0149,  0.0086,  0.0172,  ..., -0.0007,  0.0193,  0.0088]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.attn.ff.bias',\n",
       "               tensor([ 0.0098, -0.0098, -0.0042,  ...,  0.0111, -0.0119, -0.0112],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.ln1.w',\n",
       "               tensor([0.9980, 0.9990, 1.0029,  ..., 0.9990, 0.9990, 0.9956],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.ln1.b',\n",
       "               tensor([ 0.0001, -0.0004,  0.0002,  ...,  0.0011, -0.0025,  0.0023],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.ln2.w',\n",
       "               tensor([1.0176, 1.0156, 1.0166,  ..., 1.0186, 1.0176, 1.0156],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.ln2.b',\n",
       "               tensor([ 0.0002,  0.0014,  0.0019,  ...,  0.0016, -0.0029, -0.0036],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.mlp.ff1.weight',\n",
       "               tensor([[ 0.0147,  0.0010, -0.0086,  ..., -0.0151,  0.0089,  0.0103],\n",
       "                       [-0.0148, -0.0064, -0.0073,  ...,  0.0049, -0.0103,  0.0109],\n",
       "                       [-0.0062,  0.0169, -0.0051,  ...,  0.0143,  0.0102, -0.0060],\n",
       "                       ...,\n",
       "                       [ 0.0011,  0.0157,  0.0174,  ..., -0.0208,  0.0111,  0.0083],\n",
       "                       [-0.0127, -0.0130,  0.0056,  ..., -0.0061,  0.0139, -0.0119],\n",
       "                       [ 0.0142, -0.0198, -0.0066,  ..., -0.0108, -0.0132, -0.0050]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.mlp.ff1.bias',\n",
       "               tensor([ 0.0047,  0.0068,  0.0046,  ..., -0.0126, -0.0062,  0.0042],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.mlp.ff2.weight',\n",
       "               tensor([[-0.0029, -0.0007, -0.0077,  ...,  0.0118,  0.0126, -0.0105],\n",
       "                       [-0.0110, -0.0006,  0.0060,  ..., -0.0023,  0.0013, -0.0031],\n",
       "                       [-0.0076, -0.0010,  0.0099,  ..., -0.0030,  0.0006,  0.0088],\n",
       "                       ...,\n",
       "                       [-0.0010,  0.0024, -0.0100,  ...,  0.0110, -0.0024, -0.0068],\n",
       "                       [-0.0109, -0.0092,  0.0004,  ..., -0.0039,  0.0026, -0.0013],\n",
       "                       [-0.0007, -0.0002,  0.0030,  ..., -0.0064, -0.0066, -0.0090]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.7.mlp.ff2.bias',\n",
       "               tensor([ 0.0034, -0.0045,  0.0031,  ...,  0.0077,  0.0071, -0.0057],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.attn.proj.weight',\n",
       "               tensor([[-0.0126, -0.0082,  0.0140,  ...,  0.0024,  0.0050, -0.0160],\n",
       "                       [ 0.0136,  0.0092, -0.0077,  ...,  0.0006, -0.0193,  0.0099],\n",
       "                       [-0.0017,  0.0071, -0.0062,  ...,  0.0141,  0.0022,  0.0044],\n",
       "                       ...,\n",
       "                       [-0.0160,  0.0019, -0.0026,  ..., -0.0086, -0.0047,  0.0075],\n",
       "                       [ 0.0080,  0.0183,  0.0181,  ...,  0.0080,  0.0013,  0.0072],\n",
       "                       [-0.0082, -0.0188, -0.0039,  ...,  0.0042, -0.0071, -0.0021]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.attn.proj.bias',\n",
       "               tensor([ 0.0095,  0.0032,  0.0089,  ...,  0.0027, -0.0069,  0.0077],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.attn.ff.weight',\n",
       "               tensor([[ 0.0126, -0.0081,  0.0046,  ..., -0.0062,  0.0074,  0.0095],\n",
       "                       [-0.0061, -0.0136, -0.0167,  ..., -0.0202,  0.0013, -0.0145],\n",
       "                       [ 0.0085,  0.0085,  0.0161,  ...,  0.0123, -0.0074, -0.0216],\n",
       "                       ...,\n",
       "                       [ 0.0034, -0.0074, -0.0057,  ..., -0.0076, -0.0134, -0.0010],\n",
       "                       [ 0.0021, -0.0120, -0.0085,  ...,  0.0032,  0.0096, -0.0082],\n",
       "                       [-0.0178,  0.0013, -0.0173,  ..., -0.0008,  0.0004, -0.0085]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.attn.ff.bias',\n",
       "               tensor([-0.0112,  0.0063, -0.0105,  ..., -0.0018,  0.0054, -0.0009],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.ln1.w',\n",
       "               tensor([0.9902, 0.9917, 0.9946,  ..., 1.0020, 0.9854, 0.9897],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.ln1.b',\n",
       "               tensor([-7.5340e-05,  1.7872e-03,  1.3340e-04,  ...,  3.1853e-03,\n",
       "                       -2.0046e-03,  3.3188e-03], dtype=torch.float16)),\n",
       "              ('blocks.8.ln2.w',\n",
       "               tensor([1.0107, 1.0049, 1.0146,  ..., 1.0107, 1.0098, 1.0088],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.ln2.b',\n",
       "               tensor([-0.0004, -0.0015, -0.0045,  ..., -0.0025, -0.0007, -0.0014],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.mlp.ff1.weight',\n",
       "               tensor([[-1.1818e-02,  2.1992e-03, -5.7983e-03,  ..., -1.3115e-02,\n",
       "                         4.4098e-03,  4.1084e-03],\n",
       "                       [ 2.0645e-02, -1.1778e-03,  1.2268e-02,  ...,  1.1192e-02,\n",
       "                         1.1559e-02,  2.2697e-03],\n",
       "                       [ 8.8096e-05, -1.6205e-02, -1.2794e-02,  ...,  6.0196e-03,\n",
       "                         2.0569e-02,  1.0750e-02],\n",
       "                       ...,\n",
       "                       [ 9.5139e-03, -7.1449e-03,  6.7635e-03,  ..., -1.6678e-02,\n",
       "                         1.2466e-02, -1.3285e-03],\n",
       "                       [-2.0554e-02, -3.9368e-03,  8.2321e-03,  ..., -1.2909e-02,\n",
       "                         4.8904e-03, -1.4191e-02],\n",
       "                       [-7.6180e-03,  8.0338e-03,  2.2011e-03,  ...,  1.6785e-02,\n",
       "                         1.1337e-02,  2.2087e-03]], dtype=torch.float16)),\n",
       "              ('blocks.8.mlp.ff1.bias',\n",
       "               tensor([-0.0056, -0.0068, -0.0004,  ..., -0.0091, -0.0019,  0.0078],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.mlp.ff2.weight',\n",
       "               tensor([[ 0.0030, -0.0023,  0.0047,  ...,  0.0102, -0.0042,  0.0148],\n",
       "                       [-0.0081,  0.0136, -0.0007,  ..., -0.0046, -0.0041,  0.0056],\n",
       "                       [ 0.0021, -0.0076,  0.0104,  ...,  0.0047,  0.0074,  0.0034],\n",
       "                       ...,\n",
       "                       [-0.0008, -0.0003, -0.0053,  ...,  0.0158, -0.0012, -0.0054],\n",
       "                       [-0.0097, -0.0080, -0.0012,  ..., -0.0082,  0.0040, -0.0050],\n",
       "                       [ 0.0052,  0.0081,  0.0052,  ..., -0.0112, -0.0053, -0.0108]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.8.mlp.ff2.bias',\n",
       "               tensor([-0.0048,  0.0060,  0.0084,  ...,  0.0088,  0.0038, -0.0068],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.attn.proj.weight',\n",
       "               tensor([[-0.0030, -0.0005,  0.0035,  ...,  0.0027, -0.0230,  0.0027],\n",
       "                       [ 0.0011,  0.0084,  0.0163,  ..., -0.0246,  0.0154, -0.0134],\n",
       "                       [ 0.0034,  0.0045, -0.0139,  ..., -0.0088, -0.0100,  0.0002],\n",
       "                       ...,\n",
       "                       [ 0.0162,  0.0072, -0.0117,  ..., -0.0200, -0.0079, -0.0084],\n",
       "                       [ 0.0052,  0.0188, -0.0140,  ..., -0.0013, -0.0143,  0.0132],\n",
       "                       [-0.0194,  0.0125, -0.0016,  ...,  0.0184, -0.0197, -0.0031]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.attn.proj.bias',\n",
       "               tensor([ 0.0020, -0.0164,  0.0094,  ..., -0.0100,  0.0151,  0.0132],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.attn.ff.weight',\n",
       "               tensor([[ 1.4938e-02, -8.9188e-03, -1.0742e-02,  ..., -1.7624e-02,\n",
       "                         2.0813e-02,  1.4267e-03],\n",
       "                       [-1.3405e-02,  4.0665e-03, -1.2184e-02,  ...,  1.8890e-02,\n",
       "                        -1.7212e-02,  1.4748e-02],\n",
       "                       [ 1.0460e-02, -7.6294e-03,  8.4043e-06,  ..., -9.1410e-04,\n",
       "                        -5.8022e-03,  8.1253e-03],\n",
       "                       ...,\n",
       "                       [ 1.2207e-02,  1.6251e-02, -1.5869e-02,  ...,  6.1684e-03,\n",
       "                        -1.2878e-02,  1.3435e-02],\n",
       "                       [-2.3956e-03,  4.3945e-03, -4.3106e-03,  ..., -4.7150e-03,\n",
       "                        -3.5400e-03,  7.2670e-03],\n",
       "                       [ 1.5976e-02, -8.1635e-03,  2.9030e-03,  ..., -1.2312e-03,\n",
       "                        -7.3280e-03, -1.2159e-03]], dtype=torch.float16)),\n",
       "              ('blocks.9.attn.ff.bias',\n",
       "               tensor([ 0.0181, -0.0150,  0.0078,  ..., -0.0004, -0.0155,  0.0167],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.ln1.w',\n",
       "               tensor([1.0010, 1.0166, 1.0195,  ..., 1.0127, 1.0137, 1.0049],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.ln1.b',\n",
       "               tensor([ 1.5616e-05, -6.7854e-04,  8.1301e-04,  ...,  1.3685e-03,\n",
       "                       -1.7042e-03,  3.1490e-03], dtype=torch.float16)),\n",
       "              ('blocks.9.ln2.w',\n",
       "               tensor([1.0088, 1.0010, 1.0029,  ..., 1.0088, 1.0029, 1.0020],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.ln2.b',\n",
       "               tensor([-0.0028, -0.0017, -0.0046,  ..., -0.0037,  0.0017, -0.0037],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.mlp.ff1.weight',\n",
       "               tensor([[ 0.0088, -0.0163, -0.0062,  ...,  0.0022,  0.0120,  0.0055],\n",
       "                       [ 0.0032,  0.0032, -0.0060,  ...,  0.0015, -0.0141,  0.0176],\n",
       "                       [ 0.0015, -0.0026, -0.0102,  ...,  0.0074, -0.0142, -0.0206],\n",
       "                       ...,\n",
       "                       [-0.0152,  0.0152, -0.0153,  ...,  0.0059,  0.0114, -0.0118],\n",
       "                       [ 0.0015,  0.0006, -0.0210,  ...,  0.0115, -0.0109, -0.0052],\n",
       "                       [ 0.0096, -0.0047,  0.0166,  ..., -0.0158,  0.0085, -0.0166]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.mlp.ff1.bias',\n",
       "               tensor([ 0.0170, -0.0166,  0.0025,  ...,  0.0082,  0.0123,  0.0123],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.9.mlp.ff2.weight',\n",
       "               tensor([[-1.9968e-05, -1.7509e-03, -2.4815e-03,  ...,  9.3689e-03,\n",
       "                         3.7460e-03,  6.9351e-03],\n",
       "                       [ 1.6518e-03, -2.5520e-03, -5.1270e-03,  ...,  1.1218e-04,\n",
       "                        -8.1587e-04, -7.1602e-03],\n",
       "                       [ 5.6038e-03,  8.8806e-03, -1.0052e-03,  ...,  1.0139e-02,\n",
       "                         2.9011e-03, -1.5612e-03],\n",
       "                       ...,\n",
       "                       [-2.0161e-03, -9.2411e-04,  1.5535e-03,  ..., -2.9030e-03,\n",
       "                         5.3444e-03, -2.5902e-03],\n",
       "                       [-7.2861e-03, -2.0468e-04,  5.3444e-03,  ...,  4.6539e-03,\n",
       "                         1.7824e-03, -9.2316e-03],\n",
       "                       [ 2.8687e-03, -7.1182e-03,  5.4893e-03,  ..., -3.6354e-03,\n",
       "                         3.6180e-05, -5.2528e-03]], dtype=torch.float16)),\n",
       "              ('blocks.9.mlp.ff2.bias',\n",
       "               tensor([-0.0034, -0.0081,  0.0041,  ...,  0.0069,  0.0078,  0.0086],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.attn.proj.weight',\n",
       "               tensor([[-1.3680e-02,  1.1459e-02,  2.9411e-03,  ...,  6.2866e-03,\n",
       "                         2.4948e-03,  8.6517e-03],\n",
       "                       [ 8.9111e-03, -6.5651e-03, -3.3607e-03,  ...,  1.2886e-02,\n",
       "                         1.4076e-02,  1.1894e-02],\n",
       "                       [-6.8588e-03,  2.8267e-03,  6.7329e-03,  ..., -4.9095e-03,\n",
       "                         2.4338e-03, -1.6174e-02],\n",
       "                       ...,\n",
       "                       [ 3.4523e-03,  1.3809e-02,  6.5384e-03,  ..., -2.4170e-02,\n",
       "                        -9.2850e-03,  8.2703e-03],\n",
       "                       [-1.8814e-02, -1.6251e-02, -8.1100e-03,  ..., -2.9030e-03,\n",
       "                        -1.6403e-02,  7.2632e-03],\n",
       "                       [-1.4587e-02, -1.2426e-03, -6.8932e-03,  ...,  1.7685e-02,\n",
       "                        -2.2769e-05,  1.5762e-02]], dtype=torch.float16)),\n",
       "              ('blocks.10.attn.proj.bias',\n",
       "               tensor([-0.0013, -0.0166,  0.0117,  ..., -0.0103, -0.0161, -0.0048],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.attn.ff.weight',\n",
       "               tensor([[ 0.0163, -0.0021,  0.0130,  ...,  0.0164,  0.0092,  0.0025],\n",
       "                       [-0.0080, -0.0121, -0.0175,  ..., -0.0068, -0.0062,  0.0069],\n",
       "                       [-0.0078, -0.0150,  0.0021,  ...,  0.0061, -0.0241, -0.0203],\n",
       "                       ...,\n",
       "                       [-0.0130, -0.0169, -0.0164,  ..., -0.0194,  0.0081, -0.0133],\n",
       "                       [-0.0011,  0.0012,  0.0017,  ...,  0.0151, -0.0062, -0.0139],\n",
       "                       [-0.0047,  0.0164,  0.0101,  ..., -0.0002, -0.0055, -0.0142]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.attn.ff.bias',\n",
       "               tensor([-0.0119,  0.0102, -0.0150,  ...,  0.0034,  0.0001, -0.0097],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.ln1.w',\n",
       "               tensor([1.0127, 1.0205, 1.0176,  ..., 1.0166, 1.0088, 1.0137],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.ln1.b',\n",
       "               tensor([ 0.0045, -0.0008,  0.0026,  ...,  0.0020, -0.0060,  0.0032],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.ln2.w',\n",
       "               tensor([1.0166, 1.0107, 1.0234,  ..., 1.0166, 1.0225, 1.0176],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.ln2.b',\n",
       "               tensor([ 0.0009,  0.0027, -0.0068,  ...,  0.0003,  0.0005,  0.0029],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.mlp.ff1.weight',\n",
       "               tensor([[-7.9651e-03,  1.5686e-02, -1.4839e-02,  ..., -1.0971e-02,\n",
       "                        -1.6235e-02,  1.1566e-02],\n",
       "                       [-1.1467e-02, -4.8370e-03,  9.0027e-03,  ...,  2.3163e-02,\n",
       "                        -1.5343e-02, -1.8673e-03],\n",
       "                       [ 1.5266e-02,  3.8376e-03,  1.9897e-02,  ...,  1.6754e-02,\n",
       "                         1.5640e-02,  1.5198e-02],\n",
       "                       ...,\n",
       "                       [ 1.5259e-02, -3.8509e-03, -7.7782e-03,  ...,  5.2757e-03,\n",
       "                         1.1894e-02, -1.3298e-02],\n",
       "                       [ 1.2886e-02, -2.2697e-04, -4.1389e-03,  ..., -1.0284e-02,\n",
       "                         7.9269e-03,  1.1772e-02],\n",
       "                       [-8.3008e-03, -2.9488e-03,  6.2561e-03,  ...,  4.5896e-05,\n",
       "                         1.6083e-02, -5.9395e-03]], dtype=torch.float16)),\n",
       "              ('blocks.10.mlp.ff1.bias',\n",
       "               tensor([ 0.0068,  0.0083,  0.0184,  ...,  0.0076, -0.0069,  0.0168],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.10.mlp.ff2.weight',\n",
       "               tensor([[ 4.4785e-03, -6.4583e-03,  2.5520e-03,  ...,  1.8492e-03,\n",
       "                         3.0112e-04,  3.4404e-04],\n",
       "                       [-1.4486e-03,  4.4167e-05, -4.2915e-03,  ..., -4.7340e-03,\n",
       "                         6.8741e-03,  1.2589e-03],\n",
       "                       [-9.2316e-04, -1.8206e-03,  5.4703e-03,  ...,  4.1122e-03,\n",
       "                        -6.0501e-03,  6.1188e-03],\n",
       "                       ...,\n",
       "                       [ 1.4648e-03,  3.2024e-03,  2.8763e-03,  ...,  4.7379e-03,\n",
       "                        -2.6894e-03,  3.0174e-03],\n",
       "                       [ 2.7218e-03,  5.3644e-04, -1.0941e-02,  ...,  9.5701e-04,\n",
       "                        -3.8891e-03, -9.7656e-03],\n",
       "                       [-9.3689e-03,  1.2388e-03, -6.9923e-03,  ..., -2.8839e-03,\n",
       "                        -2.3341e-04, -8.9340e-03]], dtype=torch.float16)),\n",
       "              ('blocks.10.mlp.ff2.bias',\n",
       "               tensor([-0.0015, -0.0021,  0.0017,  ...,  0.0010, -0.0018, -0.0009],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.attn.proj.weight',\n",
       "               tensor([[-4.2725e-03,  4.0889e-04,  1.4639e-03,  ..., -1.2833e-02,\n",
       "                         9.3994e-03, -1.1696e-02],\n",
       "                       [ 8.0109e-04,  1.2711e-02,  3.1681e-03,  ..., -7.3433e-03,\n",
       "                        -6.1264e-03,  1.3298e-02],\n",
       "                       [ 6.0129e-04,  8.8501e-03, -7.3195e-04,  ...,  4.7646e-03,\n",
       "                        -6.9733e-03,  1.2369e-03],\n",
       "                       ...,\n",
       "                       [ 1.3092e-02,  1.2703e-02,  8.2474e-03,  ..., -5.2376e-03,\n",
       "                        -4.4556e-03, -8.8120e-03],\n",
       "                       [ 2.8172e-03,  6.5193e-03,  1.0147e-02,  ...,  1.1978e-03,\n",
       "                        -1.8204e-02,  1.0178e-02],\n",
       "                       [ 8.5831e-03, -1.0452e-03, -4.4823e-03,  ..., -8.0872e-03,\n",
       "                        -7.6342e-04, -4.6551e-05]], dtype=torch.float16)),\n",
       "              ('blocks.11.attn.proj.bias',\n",
       "               tensor([ 0.0134, -0.0013, -0.0065,  ..., -0.0159, -0.0114,  0.0087],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.attn.ff.weight',\n",
       "               tensor([[-0.0134, -0.0148, -0.0048,  ..., -0.0051, -0.0163,  0.0034],\n",
       "                       [-0.0164, -0.0132,  0.0015,  ..., -0.0041, -0.0036, -0.0176],\n",
       "                       [ 0.0170,  0.0087,  0.0047,  ...,  0.0075,  0.0090,  0.0027],\n",
       "                       ...,\n",
       "                       [ 0.0023,  0.0057, -0.0030,  ..., -0.0055,  0.0033, -0.0089],\n",
       "                       [ 0.0163,  0.0023, -0.0079,  ..., -0.0089, -0.0146,  0.0108],\n",
       "                       [-0.0213, -0.0103,  0.0145,  ..., -0.0107, -0.0155, -0.0033]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.attn.ff.bias',\n",
       "               tensor([ 0.0034, -0.0159, -0.0053,  ..., -0.0093, -0.0043,  0.0037],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.ln1.w',\n",
       "               tensor([0.9971, 1.0166, 1.0117,  ..., 1.0127, 1.0078, 1.0039],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.ln1.b',\n",
       "               tensor([ 2.6722e-03, -8.5592e-05,  1.0319e-03,  ...,  1.3590e-03,\n",
       "                       -2.2507e-03,  1.6670e-03], dtype=torch.float16)),\n",
       "              ('blocks.11.ln2.w',\n",
       "               tensor([1.0186, 1.0127, 1.0332,  ..., 1.0234, 1.0186, 1.0176],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.ln2.b',\n",
       "               tensor([ 0.0014, -0.0024, -0.0018,  ...,  0.0005, -0.0004,  0.0054],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.mlp.ff1.weight',\n",
       "               tensor([[-0.0131,  0.0010,  0.0164,  ..., -0.0144,  0.0014, -0.0109],\n",
       "                       [-0.0168,  0.0043, -0.0135,  ..., -0.0009, -0.0133, -0.0045],\n",
       "                       [-0.0016, -0.0028,  0.0135,  ...,  0.0178, -0.0045, -0.0021],\n",
       "                       ...,\n",
       "                       [-0.0115, -0.0074,  0.0192,  ...,  0.0052,  0.0133, -0.0148],\n",
       "                       [-0.0164, -0.0111,  0.0012,  ..., -0.0088, -0.0188, -0.0091],\n",
       "                       [ 0.0078, -0.0131, -0.0131,  ...,  0.0097,  0.0173, -0.0151]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.mlp.ff1.bias',\n",
       "               tensor([-0.0133, -0.0085, -0.0029,  ...,  0.0160,  0.0130,  0.0072],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.11.mlp.ff2.weight',\n",
       "               tensor([[ 8.5020e-04, -2.0103e-03, -7.7477e-03,  ..., -2.6741e-03,\n",
       "                        -7.5722e-03, -4.8523e-03],\n",
       "                       [-1.4296e-03,  3.9387e-04, -5.4169e-04,  ..., -6.1760e-03,\n",
       "                         1.3298e-02, -8.8120e-03],\n",
       "                       [-7.2420e-05,  7.1383e-04, -3.6144e-03,  ..., -1.1742e-02,\n",
       "                         1.1997e-03,  3.2177e-03],\n",
       "                       ...,\n",
       "                       [ 5.5885e-03, -2.1515e-03, -2.8477e-03,  ...,  7.2365e-03,\n",
       "                         5.9586e-03,  2.1210e-03],\n",
       "                       [ 9.4376e-03,  7.1259e-03,  3.7613e-03,  ..., -3.3855e-03,\n",
       "                         3.3875e-03, -3.4027e-03],\n",
       "                       [-1.4334e-03,  1.0361e-02, -3.4618e-03,  ...,  7.3280e-03,\n",
       "                         6.4278e-03, -4.0698e-04]], dtype=torch.float16)),\n",
       "              ('blocks.11.mlp.ff2.bias',\n",
       "               tensor([-0.0041,  0.0022,  0.0060,  ..., -0.0065, -0.0087, -0.0009],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.attn.proj.weight',\n",
       "               tensor([[ 0.0058,  0.0172,  0.0175,  ...,  0.0031, -0.0008, -0.0193],\n",
       "                       [-0.0015, -0.0136,  0.0130,  ...,  0.0029,  0.0156,  0.0103],\n",
       "                       [ 0.0024,  0.0047,  0.0117,  ...,  0.0114,  0.0036,  0.0096],\n",
       "                       ...,\n",
       "                       [-0.0104, -0.0006,  0.0034,  ...,  0.0094,  0.0224, -0.0086],\n",
       "                       [ 0.0056, -0.0040,  0.0035,  ..., -0.0156, -0.0002, -0.0013],\n",
       "                       [-0.0105, -0.0070,  0.0041,  ..., -0.0057,  0.0079, -0.0089]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.attn.proj.bias',\n",
       "               tensor([-0.0073, -0.0172, -0.0019,  ...,  0.0058,  0.0155,  0.0020],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.attn.ff.weight',\n",
       "               tensor([[-0.0026,  0.0106,  0.0150,  ...,  0.0029, -0.0073, -0.0009],\n",
       "                       [ 0.0028,  0.0065,  0.0169,  ..., -0.0024,  0.0149,  0.0124],\n",
       "                       [ 0.0103,  0.0128,  0.0065,  ...,  0.0126, -0.0084,  0.0146],\n",
       "                       ...,\n",
       "                       [-0.0030,  0.0127, -0.0019,  ..., -0.0167,  0.0146, -0.0172],\n",
       "                       [ 0.0028, -0.0049,  0.0233,  ...,  0.0004, -0.0062, -0.0173],\n",
       "                       [-0.0125, -0.0119,  0.0217,  ...,  0.0205, -0.0099, -0.0050]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.attn.ff.bias',\n",
       "               tensor([-0.0023, -0.0138, -0.0132,  ...,  0.0182, -0.0189, -0.0005],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.ln1.w',\n",
       "               tensor([0.9932, 0.9966, 1.0000,  ..., 0.9902, 1.0010, 0.9941],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.ln1.b',\n",
       "               tensor([-0.0003,  0.0011,  0.0002,  ...,  0.0026, -0.0016,  0.0014],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.ln2.w',\n",
       "               tensor([1.0186, 1.0186, 1.0303,  ..., 1.0186, 1.0234, 1.0166],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.ln2.b',\n",
       "               tensor([ 0.0046,  0.0059, -0.0042,  ..., -0.0037, -0.0029,  0.0006],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.mlp.ff1.weight',\n",
       "               tensor([[ 0.0039,  0.0095, -0.0026,  ...,  0.0179, -0.0147, -0.0165],\n",
       "                       [ 0.0036,  0.0202, -0.0097,  ...,  0.0070,  0.0023, -0.0190],\n",
       "                       [ 0.0016,  0.0067,  0.0005,  ...,  0.0029, -0.0025,  0.0057],\n",
       "                       ...,\n",
       "                       [ 0.0053, -0.0050,  0.0083,  ...,  0.0037, -0.0139, -0.0112],\n",
       "                       [-0.0115, -0.0018,  0.0028,  ...,  0.0153,  0.0093,  0.0156],\n",
       "                       [ 0.0080, -0.0025, -0.0172,  ..., -0.0159,  0.0205,  0.0126]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.mlp.ff1.bias',\n",
       "               tensor([-0.0105, -0.0010, -0.0001,  ...,  0.0130, -0.0121,  0.0063],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.12.mlp.ff2.weight',\n",
       "               tensor([[-7.5455e-03, -4.0054e-05, -1.1284e-02,  ...,  3.4499e-04,\n",
       "                         5.7297e-03,  7.1907e-03],\n",
       "                       [-1.0292e-02, -7.2327e-03, -1.8940e-03,  ...,  6.1607e-03,\n",
       "                        -1.5335e-03,  2.7275e-03],\n",
       "                       [ 7.6370e-03, -3.9787e-03,  4.8523e-03,  ..., -3.1986e-03,\n",
       "                         9.4299e-03, -2.9221e-03],\n",
       "                       ...,\n",
       "                       [ 6.3515e-03, -6.6147e-03,  3.4180e-03,  ...,  9.4452e-03,\n",
       "                        -5.1003e-03,  1.0094e-02],\n",
       "                       [ 6.3972e-03,  6.1226e-03,  4.2992e-03,  ..., -2.1534e-03,\n",
       "                        -6.9656e-03, -7.4272e-03],\n",
       "                       [-6.2180e-03,  7.3586e-03,  3.3722e-03,  ..., -9.7046e-03,\n",
       "                         2.6112e-03, -7.9498e-03]], dtype=torch.float16)),\n",
       "              ('blocks.12.mlp.ff2.bias',\n",
       "               tensor([ 0.0076, -0.0040,  0.0056,  ...,  0.0059,  0.0062, -0.0023],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.attn.proj.weight',\n",
       "               tensor([[ 0.0157, -0.0044,  0.0098,  ..., -0.0145,  0.0170,  0.0110],\n",
       "                       [ 0.0097, -0.0077, -0.0120,  ..., -0.0124,  0.0061,  0.0096],\n",
       "                       [-0.0008, -0.0001, -0.0026,  ...,  0.0126, -0.0192,  0.0047],\n",
       "                       ...,\n",
       "                       [ 0.0203,  0.0008, -0.0006,  ..., -0.0050,  0.0172, -0.0012],\n",
       "                       [-0.0088, -0.0081, -0.0190,  ...,  0.0162, -0.0140, -0.0029],\n",
       "                       [ 0.0012,  0.0061,  0.0186,  ..., -0.0193, -0.0009, -0.0139]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.attn.proj.bias',\n",
       "               tensor([ 0.0038, -0.0056, -0.0013,  ...,  0.0164,  0.0123,  0.0114],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.attn.ff.weight',\n",
       "               tensor([[-0.0110, -0.0076, -0.0033,  ...,  0.0196,  0.0109, -0.0013],\n",
       "                       [ 0.0063,  0.0060, -0.0076,  ...,  0.0110,  0.0026, -0.0052],\n",
       "                       [-0.0087, -0.0184,  0.0122,  ..., -0.0078, -0.0034,  0.0156],\n",
       "                       ...,\n",
       "                       [-0.0167,  0.0067,  0.0127,  ..., -0.0168,  0.0071,  0.0007],\n",
       "                       [-0.0075, -0.0127, -0.0141,  ...,  0.0082, -0.0096,  0.0122],\n",
       "                       [-0.0011,  0.0020,  0.0003,  ..., -0.0096,  0.0137,  0.0141]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.attn.ff.bias',\n",
       "               tensor([ 0.0045, -0.0006,  0.0022,  ...,  0.0040, -0.0061, -0.0034],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.ln1.w',\n",
       "               tensor([0.9941, 1.0049, 1.0098,  ..., 1.0010, 1.0029, 0.9951],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.ln1.b',\n",
       "               tensor([-1.4954e-03, -4.0531e-05,  9.2030e-04,  ...,  2.2781e-04,\n",
       "                        7.9203e-04, -1.4572e-03], dtype=torch.float16)),\n",
       "              ('blocks.13.ln2.w',\n",
       "               tensor([1.0186, 1.0195, 1.0322,  ..., 1.0225, 1.0215, 1.0146],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.ln2.b',\n",
       "               tensor([ 0.0011,  0.0016, -0.0034,  ...,  0.0014, -0.0036,  0.0054],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.mlp.ff1.weight',\n",
       "               tensor([[ 0.0083,  0.0130,  0.0111,  ...,  0.0082,  0.0266,  0.0046],\n",
       "                       [ 0.0096, -0.0131, -0.0251,  ..., -0.0122, -0.0089, -0.0023],\n",
       "                       [ 0.0115,  0.0163, -0.0028,  ..., -0.0048, -0.0050,  0.0039],\n",
       "                       ...,\n",
       "                       [ 0.0181, -0.0167, -0.0033,  ..., -0.0161, -0.0176,  0.0013],\n",
       "                       [-0.0002,  0.0004,  0.0139,  ...,  0.0137, -0.0139, -0.0018],\n",
       "                       [-0.0193, -0.0031,  0.0027,  ..., -0.0122, -0.0024,  0.0125]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.mlp.ff1.bias',\n",
       "               tensor([-0.0064, -0.0151,  0.0086,  ..., -0.0112, -0.0149, -0.0088],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.mlp.ff2.weight',\n",
       "               tensor([[-0.0053, -0.0004,  0.0029,  ...,  0.0053,  0.0029, -0.0023],\n",
       "                       [-0.0051,  0.0040, -0.0081,  ...,  0.0074,  0.0046,  0.0007],\n",
       "                       [-0.0023,  0.0057,  0.0113,  ...,  0.0043, -0.0006, -0.0008],\n",
       "                       ...,\n",
       "                       [-0.0058, -0.0027,  0.0013,  ...,  0.0034, -0.0024,  0.0004],\n",
       "                       [-0.0025,  0.0119, -0.0109,  ..., -0.0009,  0.0039,  0.0104],\n",
       "                       [ 0.0042, -0.0045, -0.0050,  ...,  0.0067,  0.0004,  0.0044]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.13.mlp.ff2.bias',\n",
       "               tensor([ 0.0016,  0.0015,  0.0032,  ..., -0.0024,  0.0008, -0.0058],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.14.attn.proj.weight',\n",
       "               tensor([[-0.0145,  0.0100, -0.0151,  ..., -0.0090,  0.0084, -0.0120],\n",
       "                       [-0.0141,  0.0094, -0.0202,  ..., -0.0134, -0.0048, -0.0050],\n",
       "                       [-0.0163,  0.0125, -0.0073,  ..., -0.0089,  0.0017,  0.0027],\n",
       "                       ...,\n",
       "                       [ 0.0169,  0.0052, -0.0024,  ...,  0.0006,  0.0045, -0.0046],\n",
       "                       [-0.0041,  0.0105, -0.0246,  ..., -0.0158,  0.0029, -0.0108],\n",
       "                       [-0.0086,  0.0166, -0.0107,  ...,  0.0169, -0.0074,  0.0104]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.14.attn.proj.bias',\n",
       "               tensor([ 0.0036, -0.0034, -0.0072,  ...,  0.0032,  0.0073,  0.0039],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.14.attn.ff.weight',\n",
       "               tensor([[-0.0125, -0.0007,  0.0123,  ...,  0.0183,  0.0070, -0.0145],\n",
       "                       [ 0.0145, -0.0099,  0.0076,  ...,  0.0027, -0.0142, -0.0051],\n",
       "                       [ 0.0166,  0.0011, -0.0122,  ..., -0.0159, -0.0054,  0.0184],\n",
       "                       ...,\n",
       "                       [-0.0126,  0.0122,  0.0256,  ..., -0.0079, -0.0143,  0.0028],\n",
       "                       [ 0.0120,  0.0114,  0.0039,  ...,  0.0056, -0.0134, -0.0029],\n",
       "                       [-0.0041, -0.0073,  0.0163,  ..., -0.0041,  0.0089, -0.0188]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.14.attn.ff.bias',\n",
       "               tensor([-1.2688e-02, -3.7651e-03,  3.2616e-03,  ..., -1.0071e-02,\n",
       "                       -2.0623e-05, -7.0343e-03], dtype=torch.float16)),\n",
       "              ('blocks.14.ln1.w',\n",
       "               tensor([0.9932, 1.0010, 1.0010,  ..., 0.9941, 1.0000, 0.9976],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.14.ln1.b',\n",
       "               tensor([-2.7981e-03,  1.7989e-04, -7.6890e-05,  ...,  1.2696e-04,\n",
       "                        1.5383e-03, -1.9932e-03], dtype=torch.float16)),\n",
       "              ('blocks.14.ln2.w',\n",
       "               tensor([1.0127, 1.0127, 1.0205,  ..., 1.0186, 1.0146, 1.0166],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.14.ln2.b',\n",
       "               tensor([-0.0017, -0.0017, -0.0027,  ..., -0.0003,  0.0044,  0.0034],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.14.mlp.ff1.weight',\n",
       "               tensor([[ 0.0061,  0.0034, -0.0052,  ..., -0.0005,  0.0132,  0.0151],\n",
       "                       [-0.0251,  0.0080, -0.0072,  ...,  0.0140,  0.0035,  0.0016],\n",
       "                       [-0.0126, -0.0062,  0.0025,  ...,  0.0081, -0.0033, -0.0080],\n",
       "                       ...,\n",
       "                       [-0.0039,  0.0140, -0.0158,  ..., -0.0118, -0.0138, -0.0079],\n",
       "                       [ 0.0162, -0.0033,  0.0002,  ..., -0.0110,  0.0214,  0.0089],\n",
       "                       [-0.0116,  0.0029, -0.0125,  ...,  0.0080, -0.0034, -0.0074]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.14.mlp.ff1.bias',\n",
       "               tensor([ 5.0974e-04, -7.2517e-03,  1.1658e-02,  ..., -6.9313e-03,\n",
       "                       -1.0689e-02, -1.2577e-05], dtype=torch.float16)),\n",
       "              ('blocks.14.mlp.ff2.weight',\n",
       "               tensor([[ 4.9934e-03,  3.6526e-03,  2.8667e-03,  ..., -1.0147e-02,\n",
       "                        -1.3571e-03,  3.1185e-03],\n",
       "                       [ 3.2616e-03,  6.1531e-03,  6.7806e-04,  ...,  2.8634e-04,\n",
       "                        -4.8447e-03, -6.7902e-03],\n",
       "                       [-5.1422e-03,  8.7261e-04,  4.7264e-03,  ...,  7.2784e-03,\n",
       "                         8.3447e-06,  9.2392e-03],\n",
       "                       ...,\n",
       "                       [ 6.8855e-03, -2.5120e-03,  1.8520e-03,  ...,  1.6699e-03,\n",
       "                        -4.0359e-03,  7.1564e-03],\n",
       "                       [-3.4695e-03,  1.3542e-03, -4.5090e-03,  ..., -5.2261e-03,\n",
       "                         2.1458e-03,  5.1231e-03],\n",
       "                       [-2.1362e-03,  5.9242e-03,  7.3013e-03,  ..., -4.9162e-04,\n",
       "                         2.6741e-03,  8.5640e-04]], dtype=torch.float16)),\n",
       "              ('blocks.14.mlp.ff2.bias',\n",
       "               tensor([ 0.0098, -0.0071,  0.0052,  ...,  0.0079,  0.0042,  0.0012],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.attn.proj.weight',\n",
       "               tensor([[ 0.0098, -0.0117,  0.0059,  ...,  0.0128, -0.0068,  0.0078],\n",
       "                       [-0.0140,  0.0115,  0.0088,  ...,  0.0148, -0.0024,  0.0033],\n",
       "                       [ 0.0125, -0.0153,  0.0182,  ..., -0.0005, -0.0066,  0.0006],\n",
       "                       ...,\n",
       "                       [-0.0010,  0.0092,  0.0026,  ..., -0.0117, -0.0022, -0.0174],\n",
       "                       [ 0.0128, -0.0123, -0.0129,  ...,  0.0050,  0.0036, -0.0072],\n",
       "                       [-0.0111,  0.0023,  0.0131,  ...,  0.0155, -0.0006, -0.0032]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.attn.proj.bias',\n",
       "               tensor([ 0.0082,  0.0047, -0.0034,  ...,  0.0008,  0.0164,  0.0002],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.attn.ff.weight',\n",
       "               tensor([[ 0.0055, -0.0021, -0.0038,  ..., -0.0155, -0.0110,  0.0020],\n",
       "                       [ 0.0103, -0.0049,  0.0014,  ...,  0.0057,  0.0047, -0.0152],\n",
       "                       [-0.0031,  0.0107,  0.0009,  ...,  0.0077,  0.0024, -0.0049],\n",
       "                       ...,\n",
       "                       [ 0.0067, -0.0048,  0.0125,  ...,  0.0108, -0.0064, -0.0036],\n",
       "                       [-0.0033, -0.0085, -0.0055,  ..., -0.0182, -0.0064, -0.0034],\n",
       "                       [-0.0086, -0.0186,  0.0075,  ..., -0.0008, -0.0015,  0.0149]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.attn.ff.bias',\n",
       "               tensor([-0.0059,  0.0098, -0.0039,  ..., -0.0032, -0.0093, -0.0116],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.ln1.w',\n",
       "               tensor([0.9917, 1.0010, 0.9995,  ..., 0.9976, 1.0020, 1.0029],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.ln1.b',\n",
       "               tensor([-0.0040, -0.0026,  0.0022,  ...,  0.0007,  0.0011, -0.0012],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.ln2.w',\n",
       "               tensor([1.0137, 1.0156, 1.0176,  ..., 1.0127, 1.0176, 1.0127],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.ln2.b',\n",
       "               tensor([ 0.0006,  0.0004, -0.0066,  ...,  0.0015,  0.0020,  0.0023],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.mlp.ff1.weight',\n",
       "               tensor([[ 0.0106,  0.0150, -0.0185,  ...,  0.0147,  0.0107, -0.0208],\n",
       "                       [ 0.0112,  0.0168,  0.0087,  ..., -0.0039, -0.0037,  0.0054],\n",
       "                       [-0.0038, -0.0195, -0.0138,  ...,  0.0093, -0.0064,  0.0044],\n",
       "                       ...,\n",
       "                       [ 0.0069,  0.0101, -0.0171,  ...,  0.0043,  0.0111,  0.0010],\n",
       "                       [ 0.0042,  0.0049,  0.0039,  ..., -0.0083,  0.0170, -0.0020],\n",
       "                       [ 0.0051,  0.0132,  0.0096,  ..., -0.0022, -0.0038, -0.0076]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.mlp.ff1.bias',\n",
       "               tensor([ 0.0072,  0.0052, -0.0033,  ...,  0.0107,  0.0146, -0.0019],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.mlp.ff2.weight',\n",
       "               tensor([[-0.0117, -0.0022,  0.0051,  ...,  0.0051,  0.0055,  0.0095],\n",
       "                       [-0.0021,  0.0057, -0.0003,  ...,  0.0034, -0.0074, -0.0056],\n",
       "                       [ 0.0056,  0.0060,  0.0114,  ..., -0.0012, -0.0006,  0.0017],\n",
       "                       ...,\n",
       "                       [ 0.0042,  0.0015, -0.0010,  ..., -0.0040,  0.0100, -0.0048],\n",
       "                       [ 0.0007, -0.0103,  0.0072,  ..., -0.0055, -0.0012,  0.0034],\n",
       "                       [ 0.0069,  0.0083, -0.0009,  ...,  0.0034,  0.0027, -0.0039]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.15.mlp.ff2.bias',\n",
       "               tensor([-0.0068, -0.0013,  0.0014,  ...,  0.0007, -0.0019,  0.0062],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.attn.proj.weight',\n",
       "               tensor([[ 0.0112,  0.0060,  0.0217,  ...,  0.0020, -0.0129,  0.0076],\n",
       "                       [ 0.0056,  0.0189,  0.0109,  ...,  0.0152, -0.0144, -0.0235],\n",
       "                       [-0.0013, -0.0065, -0.0062,  ..., -0.0207, -0.0152,  0.0049],\n",
       "                       ...,\n",
       "                       [-0.0129,  0.0104,  0.0072,  ...,  0.0064, -0.0037,  0.0038],\n",
       "                       [ 0.0137, -0.0154,  0.0110,  ..., -0.0043,  0.0036,  0.0130],\n",
       "                       [ 0.0191, -0.0061, -0.0184,  ..., -0.0098,  0.0049, -0.0106]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.attn.proj.bias',\n",
       "               tensor([ 0.0010,  0.0148, -0.0051,  ..., -0.0018,  0.0007, -0.0034],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.attn.ff.weight',\n",
       "               tensor([[-7.9269e-03,  1.3817e-02,  1.4824e-02,  ...,  1.9503e-03,\n",
       "                        -2.3136e-03, -6.7902e-03],\n",
       "                       [ 3.9673e-04,  1.1932e-02, -4.1580e-04,  ..., -2.1782e-03,\n",
       "                        -1.3916e-02,  1.2054e-03],\n",
       "                       [ 1.7670e-02, -8.8577e-03, -1.2497e-02,  ...,  1.8711e-03,\n",
       "                         7.3662e-03,  1.3588e-02],\n",
       "                       ...,\n",
       "                       [-1.2421e-02, -1.0384e-02, -8.6823e-03,  ..., -1.5419e-02,\n",
       "                         8.8959e-03, -4.3182e-03],\n",
       "                       [ 4.7722e-03,  9.6741e-03, -2.4605e-03,  ...,  1.3184e-02,\n",
       "                        -1.0826e-02,  5.7602e-03],\n",
       "                       [ 2.9316e-03, -3.2485e-05, -2.1076e-04,  ..., -3.2082e-03,\n",
       "                        -4.5967e-03, -4.0398e-03]], dtype=torch.float16)),\n",
       "              ('blocks.16.attn.ff.bias',\n",
       "               tensor([ 0.0056, -0.0167, -0.0095,  ...,  0.0199,  0.0015,  0.0053],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.ln1.w',\n",
       "               tensor([0.9922, 0.9966, 0.9976,  ..., 0.9985, 0.9922, 0.9878],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.ln1.b',\n",
       "               tensor([-0.0038,  0.0008,  0.0009,  ...,  0.0015, -0.0001, -0.0007],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.ln2.w',\n",
       "               tensor([1.0088, 1.0186, 1.0215,  ..., 1.0137, 1.0098, 1.0098],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.ln2.b',\n",
       "               tensor([-0.0020, -0.0016, -0.0059,  ...,  0.0027,  0.0004,  0.0018],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.mlp.ff1.weight',\n",
       "               tensor([[-0.0053, -0.0163, -0.0048,  ...,  0.0065,  0.0051, -0.0144],\n",
       "                       [ 0.0111,  0.0157,  0.0137,  ...,  0.0096, -0.0007,  0.0129],\n",
       "                       [ 0.0101,  0.0064, -0.0094,  ...,  0.0041, -0.0085,  0.0145],\n",
       "                       ...,\n",
       "                       [ 0.0015, -0.0052,  0.0183,  ..., -0.0033,  0.0121, -0.0202],\n",
       "                       [ 0.0040, -0.0114,  0.0111,  ..., -0.0177, -0.0091, -0.0027],\n",
       "                       [ 0.0007,  0.0122, -0.0125,  ...,  0.0027,  0.0008, -0.0052]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.mlp.ff1.bias',\n",
       "               tensor([ 0.0088, -0.0057, -0.0129,  ...,  0.0057, -0.0133, -0.0129],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.mlp.ff2.weight',\n",
       "               tensor([[-0.0034,  0.0041, -0.0070,  ...,  0.0007,  0.0047,  0.0044],\n",
       "                       [ 0.0086, -0.0020,  0.0053,  ...,  0.0054, -0.0026,  0.0018],\n",
       "                       [-0.0045, -0.0075, -0.0004,  ..., -0.0009, -0.0094,  0.0030],\n",
       "                       ...,\n",
       "                       [ 0.0044, -0.0137,  0.0025,  ..., -0.0043,  0.0070,  0.0007],\n",
       "                       [ 0.0013, -0.0036,  0.0101,  ..., -0.0119,  0.0049, -0.0072],\n",
       "                       [ 0.0031,  0.0069,  0.0086,  ...,  0.0093,  0.0005, -0.0081]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.16.mlp.ff2.bias',\n",
       "               tensor([-0.0054,  0.0047, -0.0042,  ...,  0.0055,  0.0022,  0.0087],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.attn.proj.weight',\n",
       "               tensor([[ 0.0080,  0.0049, -0.0174,  ...,  0.0116,  0.0041,  0.0197],\n",
       "                       [-0.0218, -0.0074, -0.0097,  ...,  0.0123,  0.0042,  0.0137],\n",
       "                       [ 0.0102, -0.0038, -0.0145,  ...,  0.0066,  0.0100,  0.0083],\n",
       "                       ...,\n",
       "                       [-0.0185,  0.0012,  0.0081,  ...,  0.0041, -0.0082,  0.0109],\n",
       "                       [ 0.0127,  0.0148,  0.0089,  ..., -0.0149, -0.0097, -0.0117],\n",
       "                       [ 0.0007,  0.0167, -0.0115,  ..., -0.0020, -0.0099, -0.0023]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.attn.proj.bias',\n",
       "               tensor([-0.0048,  0.0071, -0.0134,  ..., -0.0024,  0.0063,  0.0111],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.attn.ff.weight',\n",
       "               tensor([[ 0.0082,  0.0094, -0.0094,  ...,  0.0139,  0.0130, -0.0102],\n",
       "                       [ 0.0023,  0.0094,  0.0112,  ...,  0.0082,  0.0030, -0.0120],\n",
       "                       [-0.0149,  0.0125,  0.0139,  ..., -0.0062,  0.0052, -0.0096],\n",
       "                       ...,\n",
       "                       [ 0.0060,  0.0104, -0.0106,  ...,  0.0158,  0.0160,  0.0118],\n",
       "                       [-0.0075, -0.0051, -0.0054,  ...,  0.0065, -0.0118, -0.0093],\n",
       "                       [ 0.0116, -0.0012,  0.0157,  ..., -0.0021, -0.0054, -0.0101]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.attn.ff.bias',\n",
       "               tensor([-0.0131,  0.0143,  0.0021,  ...,  0.0039, -0.0184, -0.0115],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.ln1.w',\n",
       "               tensor([0.9941, 1.0000, 1.0000,  ..., 0.9917, 0.9946, 0.9995],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.ln1.b',\n",
       "               tensor([-0.0029, -0.0001,  0.0001,  ...,  0.0023, -0.0007, -0.0010],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.ln2.w',\n",
       "               tensor([1.0068, 1.0098, 1.0156,  ..., 1.0146, 1.0078, 1.0088],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.ln2.b',\n",
       "               tensor([-0.0033, -0.0028, -0.0045,  ...,  0.0032,  0.0011,  0.0002],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.mlp.ff1.weight',\n",
       "               tensor([[-0.0104,  0.0183, -0.0038,  ..., -0.0041, -0.0098,  0.0214],\n",
       "                       [-0.0237, -0.0127, -0.0093,  ...,  0.0037, -0.0138, -0.0023],\n",
       "                       [-0.0187, -0.0143,  0.0150,  ...,  0.0143,  0.0071,  0.0135],\n",
       "                       ...,\n",
       "                       [-0.0070,  0.0146, -0.0148,  ...,  0.0036, -0.0088, -0.0021],\n",
       "                       [-0.0162,  0.0043,  0.0152,  ..., -0.0107,  0.0130, -0.0081],\n",
       "                       [ 0.0114,  0.0145,  0.0133,  ..., -0.0142, -0.0007, -0.0129]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.mlp.ff1.bias',\n",
       "               tensor([-0.0047,  0.0088, -0.0023,  ...,  0.0150, -0.0111, -0.0108],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.mlp.ff2.weight',\n",
       "               tensor([[-0.0012,  0.0102,  0.0005,  ..., -0.0003,  0.0012,  0.0026],\n",
       "                       [ 0.0042, -0.0052,  0.0005,  ..., -0.0028, -0.0027,  0.0033],\n",
       "                       [ 0.0040, -0.0062,  0.0007,  ...,  0.0058, -0.0049, -0.0097],\n",
       "                       ...,\n",
       "                       [ 0.0085, -0.0018, -0.0063,  ..., -0.0105, -0.0034,  0.0021],\n",
       "                       [-0.0033,  0.0044, -0.0047,  ...,  0.0017,  0.0041,  0.0067],\n",
       "                       [-0.0057,  0.0008, -0.0057,  ...,  0.0085,  0.0100, -0.0033]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.17.mlp.ff2.bias',\n",
       "               tensor([ 0.0017, -0.0035, -0.0075,  ...,  0.0035, -0.0017,  0.0037],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.attn.proj.weight',\n",
       "               tensor([[ 0.0045,  0.0163,  0.0159,  ..., -0.0141,  0.0004,  0.0188],\n",
       "                       [-0.0235,  0.0062, -0.0092,  ..., -0.0185, -0.0060,  0.0190],\n",
       "                       [-0.0121,  0.0172, -0.0095,  ..., -0.0117, -0.0019, -0.0066],\n",
       "                       ...,\n",
       "                       [-0.0109, -0.0143,  0.0158,  ...,  0.0024, -0.0075, -0.0044],\n",
       "                       [ 0.0141, -0.0008,  0.0053,  ..., -0.0026,  0.0206, -0.0154],\n",
       "                       [-0.0129, -0.0118, -0.0169,  ..., -0.0085, -0.0009, -0.0102]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.attn.proj.bias',\n",
       "               tensor([-0.0113,  0.0164, -0.0028,  ..., -0.0093,  0.0005,  0.0079],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.attn.ff.weight',\n",
       "               tensor([[-0.0136, -0.0109, -0.0171,  ...,  0.0124,  0.0033,  0.0023],\n",
       "                       [ 0.0109, -0.0211, -0.0035,  ..., -0.0017,  0.0018, -0.0159],\n",
       "                       [-0.0069, -0.0152, -0.0066,  ..., -0.0119,  0.0058,  0.0085],\n",
       "                       ...,\n",
       "                       [-0.0162,  0.0215,  0.0065,  ...,  0.0112,  0.0017,  0.0112],\n",
       "                       [ 0.0159, -0.0068,  0.0132,  ...,  0.0180,  0.0037, -0.0020],\n",
       "                       [-0.0015, -0.0035,  0.0122,  ...,  0.0067,  0.0161, -0.0107]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.attn.ff.bias',\n",
       "               tensor([ 0.0172, -0.0039,  0.0010,  ..., -0.0107,  0.0021,  0.0056],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.ln1.w',\n",
       "               tensor([0.9922, 0.9951, 0.9971,  ..., 0.9961, 0.9897, 0.9971],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.ln1.b',\n",
       "               tensor([-0.0038, -0.0023,  0.0015,  ...,  0.0039,  0.0013, -0.0014],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.ln2.w',\n",
       "               tensor([1.0078, 1.0146, 1.0176,  ..., 1.0176, 1.0088, 1.0127],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.ln2.b',\n",
       "               tensor([-0.0019, -0.0005, -0.0021,  ...,  0.0016,  0.0032,  0.0014],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.mlp.ff1.weight',\n",
       "               tensor([[-0.0145,  0.0095,  0.0063,  ..., -0.0052,  0.0092,  0.0124],\n",
       "                       [-0.0053, -0.0077, -0.0128,  ..., -0.0034, -0.0064,  0.0039],\n",
       "                       [ 0.0036, -0.0120, -0.0154,  ..., -0.0033,  0.0068, -0.0130],\n",
       "                       ...,\n",
       "                       [ 0.0087, -0.0123,  0.0040,  ...,  0.0086,  0.0015,  0.0127],\n",
       "                       [-0.0015,  0.0096, -0.0096,  ...,  0.0116,  0.0196,  0.0141],\n",
       "                       [-0.0067, -0.0174, -0.0068,  ...,  0.0104,  0.0074, -0.0147]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.mlp.ff1.bias',\n",
       "               tensor([ 0.0074,  0.0104, -0.0037,  ...,  0.0006,  0.0053,  0.0150],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.mlp.ff2.weight',\n",
       "               tensor([[-0.0004,  0.0065,  0.0072,  ..., -0.0039, -0.0015,  0.0035],\n",
       "                       [ 0.0040, -0.0047,  0.0013,  ..., -0.0026,  0.0028, -0.0055],\n",
       "                       [ 0.0025,  0.0124,  0.0061,  ...,  0.0005,  0.0102,  0.0077],\n",
       "                       ...,\n",
       "                       [-0.0054, -0.0039,  0.0093,  ...,  0.0006, -0.0012, -0.0079],\n",
       "                       [-0.0004, -0.0030,  0.0008,  ..., -0.0042,  0.0021, -0.0047],\n",
       "                       [ 0.0024,  0.0039,  0.0024,  ..., -0.0014, -0.0004,  0.0065]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.18.mlp.ff2.bias',\n",
       "               tensor([ 0.0031,  0.0001, -0.0063,  ...,  0.0063, -0.0031, -0.0041],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.attn.proj.weight',\n",
       "               tensor([[-0.0216, -0.0075,  0.0225,  ..., -0.0012,  0.0032,  0.0109],\n",
       "                       [-0.0012, -0.0139,  0.0106,  ...,  0.0005, -0.0129,  0.0061],\n",
       "                       [-0.0105, -0.0046,  0.0173,  ...,  0.0018,  0.0031,  0.0032],\n",
       "                       ...,\n",
       "                       [ 0.0075,  0.0159,  0.0017,  ..., -0.0027,  0.0140, -0.0041],\n",
       "                       [ 0.0139,  0.0138, -0.0136,  ...,  0.0108,  0.0212, -0.0150],\n",
       "                       [-0.0075, -0.0158,  0.0094,  ...,  0.0149,  0.0088,  0.0056]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.attn.proj.bias',\n",
       "               tensor([-0.0014,  0.0166, -0.0095,  ...,  0.0105, -0.0021, -0.0105],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.attn.ff.weight',\n",
       "               tensor([[ 0.0068,  0.0017, -0.0095,  ...,  0.0036,  0.0084, -0.0048],\n",
       "                       [-0.0116,  0.0164,  0.0081,  ..., -0.0022, -0.0121,  0.0125],\n",
       "                       [ 0.0051,  0.0035, -0.0017,  ..., -0.0019, -0.0021,  0.0150],\n",
       "                       ...,\n",
       "                       [ 0.0017, -0.0019, -0.0164,  ...,  0.0155,  0.0069,  0.0199],\n",
       "                       [-0.0084, -0.0138, -0.0240,  ...,  0.0118, -0.0080, -0.0086],\n",
       "                       [ 0.0058, -0.0002, -0.0129,  ...,  0.0068,  0.0087, -0.0138]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.attn.ff.bias',\n",
       "               tensor([-0.0061, -0.0006,  0.0003,  ...,  0.0148,  0.0076, -0.0169],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.ln1.w',\n",
       "               tensor([0.9922, 0.9917, 0.9907,  ..., 0.9917, 0.9912, 0.9941],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.ln1.b',\n",
       "               tensor([-0.0037, -0.0025,  0.0010,  ...,  0.0039, -0.0001, -0.0010],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.ln2.w',\n",
       "               tensor([1.0029, 1.0117, 1.0049,  ..., 1.0146, 1.0107, 1.0068],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.ln2.b',\n",
       "               tensor([-0.0046, -0.0025, -0.0013,  ...,  0.0033,  0.0035, -0.0005],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.mlp.ff1.weight',\n",
       "               tensor([[ 0.0139, -0.0047,  0.0003,  ...,  0.0064,  0.0047, -0.0181],\n",
       "                       [-0.0111, -0.0032, -0.0192,  ..., -0.0117,  0.0072,  0.0154],\n",
       "                       [ 0.0170, -0.0123,  0.0005,  ...,  0.0046, -0.0044,  0.0042],\n",
       "                       ...,\n",
       "                       [-0.0063,  0.0008,  0.0208,  ...,  0.0008, -0.0151,  0.0161],\n",
       "                       [-0.0101,  0.0069, -0.0075,  ...,  0.0149, -0.0153, -0.0016],\n",
       "                       [ 0.0056, -0.0144,  0.0117,  ...,  0.0176, -0.0064, -0.0164]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.mlp.ff1.bias',\n",
       "               tensor([-0.0079, -0.0093,  0.0030,  ...,  0.0171, -0.0153, -0.0027],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.mlp.ff2.weight',\n",
       "               tensor([[ 0.0018, -0.0095, -0.0004,  ..., -0.0043, -0.0007,  0.0044],\n",
       "                       [ 0.0039, -0.0036, -0.0022,  ..., -0.0093,  0.0023,  0.0055],\n",
       "                       [-0.0015,  0.0060, -0.0089,  ..., -0.0058,  0.0094, -0.0083],\n",
       "                       ...,\n",
       "                       [ 0.0064,  0.0073,  0.0041,  ..., -0.0033,  0.0035, -0.0066],\n",
       "                       [-0.0053, -0.0059,  0.0023,  ...,  0.0078,  0.0145,  0.0073],\n",
       "                       [ 0.0107,  0.0035,  0.0059,  ...,  0.0087, -0.0026, -0.0020]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.19.mlp.ff2.bias',\n",
       "               tensor([-0.0027, -0.0056,  0.0045,  ...,  0.0085, -0.0062,  0.0042],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.attn.proj.weight',\n",
       "               tensor([[ 0.0161, -0.0020,  0.0179,  ..., -0.0070, -0.0115, -0.0185],\n",
       "                       [ 0.0118,  0.0085,  0.0057,  ...,  0.0204, -0.0111, -0.0159],\n",
       "                       [-0.0047, -0.0016,  0.0074,  ..., -0.0149, -0.0201, -0.0122],\n",
       "                       ...,\n",
       "                       [-0.0161, -0.0067, -0.0049,  ...,  0.0224,  0.0164,  0.0078],\n",
       "                       [ 0.0051,  0.0102, -0.0201,  ..., -0.0011,  0.0172,  0.0157],\n",
       "                       [-0.0018,  0.0159, -0.0114,  ...,  0.0144, -0.0090,  0.0113]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.attn.proj.bias',\n",
       "               tensor([ 0.0023, -0.0144,  0.0038,  ..., -0.0173,  0.0109,  0.0099],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.attn.ff.weight',\n",
       "               tensor([[ 2.1553e-03,  4.9057e-03,  9.7198e-03,  ..., -3.4828e-03,\n",
       "                         4.8180e-03, -4.6692e-03],\n",
       "                       [-1.1147e-02,  2.3697e-02, -1.5442e-02,  ..., -2.2774e-03,\n",
       "                        -5.6496e-03, -3.1204e-03],\n",
       "                       [-1.0483e-02, -8.3237e-03, -3.4676e-03,  ..., -5.1041e-03,\n",
       "                        -4.0092e-03, -9.0027e-03],\n",
       "                       ...,\n",
       "                       [ 3.7727e-03,  1.2825e-02,  2.4414e-03,  ..., -1.6068e-02,\n",
       "                         8.3447e-07,  1.1246e-02],\n",
       "                       [-2.0004e-02, -7.4654e-03,  1.2398e-02,  ..., -3.6964e-03,\n",
       "                         7.5531e-03,  1.0574e-02],\n",
       "                       [ 1.4900e-02, -1.6769e-02, -7.0801e-03,  ..., -1.9293e-03,\n",
       "                        -1.2505e-02,  1.0138e-03]], dtype=torch.float16)),\n",
       "              ('blocks.20.attn.ff.bias',\n",
       "               tensor([-0.0145,  0.0003,  0.0084,  ..., -0.0118, -0.0180, -0.0147],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.ln1.w',\n",
       "               tensor([0.9907, 0.9990, 0.9980,  ..., 0.9971, 0.9966, 0.9873],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.ln1.b',\n",
       "               tensor([-4.4403e-03, -6.4230e-04,  7.9107e-04,  ...,  4.7607e-03,\n",
       "                       -9.7156e-06, -2.6155e-04], dtype=torch.float16)),\n",
       "              ('blocks.20.ln2.w',\n",
       "               tensor([0.9995, 1.0088, 1.0127,  ..., 1.0098, 1.0059, 1.0020],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.ln2.b',\n",
       "               tensor([-0.0037, -0.0023, -0.0041,  ...,  0.0035,  0.0022, -0.0003],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.mlp.ff1.weight',\n",
       "               tensor([[ 0.0069, -0.0053, -0.0197,  ..., -0.0101,  0.0065,  0.0165],\n",
       "                       [-0.0073,  0.0212,  0.0188,  ..., -0.0003,  0.0091,  0.0049],\n",
       "                       [-0.0089,  0.0027, -0.0090,  ...,  0.0055,  0.0040, -0.0063],\n",
       "                       ...,\n",
       "                       [-0.0054,  0.0003,  0.0107,  ...,  0.0037,  0.0127, -0.0162],\n",
       "                       [-0.0189, -0.0153,  0.0131,  ..., -0.0205,  0.0070,  0.0085],\n",
       "                       [ 0.0084, -0.0070,  0.0148,  ...,  0.0035, -0.0106,  0.0004]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.mlp.ff1.bias',\n",
       "               tensor([ 0.0121,  0.0107, -0.0097,  ..., -0.0150,  0.0140,  0.0050],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.mlp.ff2.weight',\n",
       "               tensor([[-0.0010, -0.0055, -0.0022,  ..., -0.0041,  0.0045, -0.0022],\n",
       "                       [ 0.0004, -0.0051,  0.0056,  ...,  0.0014,  0.0058,  0.0002],\n",
       "                       [ 0.0041, -0.0041, -0.0018,  ...,  0.0022, -0.0085, -0.0151],\n",
       "                       ...,\n",
       "                       [ 0.0080, -0.0037,  0.0012,  ...,  0.0038,  0.0007,  0.0064],\n",
       "                       [-0.0093,  0.0037,  0.0070,  ...,  0.0052,  0.0051,  0.0014],\n",
       "                       [ 0.0008, -0.0006, -0.0034,  ...,  0.0009,  0.0020, -0.0013]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.20.mlp.ff2.bias',\n",
       "               tensor([ 0.0051, -0.0053, -0.0084,  ..., -0.0066, -0.0082, -0.0003],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.attn.proj.weight',\n",
       "               tensor([[-0.0110, -0.0127, -0.0167,  ..., -0.0084, -0.0067,  0.0114],\n",
       "                       [-0.0023, -0.0144,  0.0151,  ...,  0.0123, -0.0172,  0.0211],\n",
       "                       [ 0.0143, -0.0219, -0.0010,  ..., -0.0102, -0.0005,  0.0184],\n",
       "                       ...,\n",
       "                       [-0.0028,  0.0109, -0.0162,  ..., -0.0146,  0.0042, -0.0094],\n",
       "                       [-0.0072,  0.0218, -0.0193,  ...,  0.0035,  0.0143, -0.0133],\n",
       "                       [-0.0041,  0.0082, -0.0075,  ..., -0.0141,  0.0076, -0.0079]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.attn.proj.bias',\n",
       "               tensor([ 0.0020,  0.0053,  0.0104,  ..., -0.0032,  0.0032, -0.0067],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.attn.ff.weight',\n",
       "               tensor([[ 0.0098, -0.0013,  0.0186,  ...,  0.0069,  0.0095,  0.0213],\n",
       "                       [ 0.0129, -0.0090,  0.0051,  ..., -0.0017, -0.0079,  0.0022],\n",
       "                       [-0.0183,  0.0033,  0.0069,  ..., -0.0026,  0.0113,  0.0177],\n",
       "                       ...,\n",
       "                       [-0.0148, -0.0037,  0.0047,  ..., -0.0141,  0.0107,  0.0005],\n",
       "                       [-0.0109,  0.0075,  0.0009,  ...,  0.0050,  0.0137,  0.0108],\n",
       "                       [-0.0208, -0.0167,  0.0040,  ...,  0.0003,  0.0019, -0.0022]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.attn.ff.bias',\n",
       "               tensor([ 0.0154,  0.0177, -0.0082,  ..., -0.0160, -0.0158,  0.0006],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.ln1.w',\n",
       "               tensor([0.9912, 0.9946, 0.9995,  ..., 0.9917, 0.9956, 1.0000],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.ln1.b',\n",
       "               tensor([-0.0039,  0.0003, -0.0005,  ...,  0.0043, -0.0009,  0.0011],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.ln2.w',\n",
       "               tensor([1.0049, 1.0068, 1.0088,  ..., 1.0117, 1.0127, 1.0029],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.ln2.b',\n",
       "               tensor([-5.0697e-03, -1.6413e-03, -2.5158e-03,  ...,  3.2306e-05,\n",
       "                        6.9380e-04, -1.2264e-03], dtype=torch.float16)),\n",
       "              ('blocks.21.mlp.ff1.weight',\n",
       "               tensor([[ 0.0075, -0.0096,  0.0083,  ...,  0.0077,  0.0007,  0.0157],\n",
       "                       [ 0.0009, -0.0138,  0.0197,  ..., -0.0071,  0.0218,  0.0060],\n",
       "                       [-0.0139,  0.0016, -0.0058,  ...,  0.0076, -0.0098, -0.0187],\n",
       "                       ...,\n",
       "                       [-0.0093, -0.0066,  0.0061,  ...,  0.0012,  0.0180,  0.0113],\n",
       "                       [-0.0166,  0.0152, -0.0024,  ...,  0.0057,  0.0155, -0.0036],\n",
       "                       [-0.0057,  0.0206,  0.0140,  ...,  0.0009,  0.0133,  0.0063]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.mlp.ff1.bias',\n",
       "               tensor([-0.0059, -0.0031,  0.0047,  ..., -0.0092,  0.0016,  0.0006],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.21.mlp.ff2.weight',\n",
       "               tensor([[-4.0131e-03,  2.0733e-03, -3.1796e-03,  ...,  7.5989e-03,\n",
       "                        -2.4757e-03,  5.2376e-03],\n",
       "                       [-4.3831e-03,  4.0665e-03, -1.1520e-02,  ...,  4.3631e-05,\n",
       "                        -1.0841e-02,  3.7518e-03],\n",
       "                       [-6.0272e-03, -3.7632e-03, -9.7809e-03,  ..., -6.2180e-03,\n",
       "                         4.9858e-03,  2.5635e-03],\n",
       "                       ...,\n",
       "                       [-1.1230e-02, -8.5373e-03,  2.9926e-03,  ..., -5.1422e-03,\n",
       "                        -4.7302e-03,  6.4774e-03],\n",
       "                       [ 7.7934e-03, -7.9727e-03,  2.2469e-03,  ...,  6.0501e-03,\n",
       "                        -6.5231e-03, -4.7493e-03],\n",
       "                       [ 4.5319e-03, -3.6583e-03,  8.9722e-03,  ..., -6.6490e-03,\n",
       "                         4.6883e-03,  2.6016e-03]], dtype=torch.float16)),\n",
       "              ('blocks.21.mlp.ff2.bias',\n",
       "               tensor([-0.0043, -0.0030, -0.0052,  ..., -0.0041,  0.0017,  0.0022],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.attn.proj.weight',\n",
       "               tensor([[-0.0076,  0.0055, -0.0163,  ..., -0.0096, -0.0015, -0.0215],\n",
       "                       [-0.0161,  0.0088,  0.0112,  ..., -0.0118,  0.0110,  0.0036],\n",
       "                       [ 0.0101,  0.0020,  0.0172,  ...,  0.0055,  0.0046, -0.0047],\n",
       "                       ...,\n",
       "                       [ 0.0180,  0.0046,  0.0008,  ...,  0.0124, -0.0177, -0.0038],\n",
       "                       [-0.0017,  0.0039, -0.0047,  ...,  0.0116, -0.0043,  0.0058],\n",
       "                       [-0.0139, -0.0110,  0.0120,  ..., -0.0091, -0.0089,  0.0076]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.attn.proj.bias',\n",
       "               tensor([ 0.0175,  0.0176,  0.0189,  ...,  0.0073, -0.0107,  0.0001],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.attn.ff.weight',\n",
       "               tensor([[-0.0154, -0.0069, -0.0010,  ..., -0.0136, -0.0126,  0.0021],\n",
       "                       [ 0.0160, -0.0131, -0.0023,  ..., -0.0120,  0.0177,  0.0060],\n",
       "                       [-0.0053,  0.0027,  0.0131,  ...,  0.0164, -0.0067,  0.0057],\n",
       "                       ...,\n",
       "                       [-0.0153,  0.0013,  0.0101,  ..., -0.0060, -0.0032,  0.0041],\n",
       "                       [-0.0023,  0.0019, -0.0088,  ...,  0.0153, -0.0123, -0.0114],\n",
       "                       [-0.0005,  0.0002, -0.0064,  ...,  0.0009,  0.0184,  0.0088]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.attn.ff.bias',\n",
       "               tensor([-0.0077,  0.0160,  0.0151,  ...,  0.0018,  0.0010, -0.0022],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.ln1.w',\n",
       "               tensor([0.9912, 0.9927, 0.9966,  ..., 0.9897, 0.9878, 0.9951],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.ln1.b',\n",
       "               tensor([-0.0046,  0.0009,  0.0016,  ...,  0.0044,  0.0009,  0.0006],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.ln2.w',\n",
       "               tensor([0.9971, 1.0137, 1.0078,  ..., 1.0117, 1.0078, 1.0039],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.ln2.b',\n",
       "               tensor([-0.0044, -0.0045, -0.0020,  ...,  0.0015,  0.0029,  0.0009],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.mlp.ff1.weight',\n",
       "               tensor([[-0.0058,  0.0167, -0.0005,  ..., -0.0147,  0.0003,  0.0036],\n",
       "                       [ 0.0159, -0.0028, -0.0161,  ..., -0.0152, -0.0128, -0.0173],\n",
       "                       [-0.0055,  0.0112, -0.0093,  ..., -0.0066, -0.0099,  0.0022],\n",
       "                       ...,\n",
       "                       [-0.0098, -0.0065, -0.0024,  ...,  0.0175,  0.0031, -0.0119],\n",
       "                       [-0.0020,  0.0101, -0.0074,  ...,  0.0119,  0.0108, -0.0016],\n",
       "                       [-0.0106,  0.0161, -0.0156,  ...,  0.0022, -0.0137,  0.0092]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.mlp.ff1.bias',\n",
       "               tensor([ 0.0031,  0.0063, -0.0075,  ..., -0.0180, -0.0082,  0.0159],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.mlp.ff2.weight',\n",
       "               tensor([[-0.0087,  0.0043, -0.0004,  ...,  0.0086,  0.0039,  0.0024],\n",
       "                       [ 0.0052, -0.0051, -0.0047,  ...,  0.0028,  0.0085, -0.0092],\n",
       "                       [-0.0041,  0.0086, -0.0048,  ...,  0.0010, -0.0067,  0.0019],\n",
       "                       ...,\n",
       "                       [ 0.0026,  0.0100,  0.0010,  ..., -0.0037,  0.0011, -0.0006],\n",
       "                       [-0.0023,  0.0015,  0.0050,  ...,  0.0028,  0.0045,  0.0102],\n",
       "                       [-0.0034, -0.0011, -0.0011,  ...,  0.0033,  0.0015, -0.0045]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.22.mlp.ff2.bias',\n",
       "               tensor([ 0.0027, -0.0005, -0.0076,  ...,  0.0101, -0.0098,  0.0042],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.attn.proj.weight',\n",
       "               tensor([[-0.0120, -0.0085, -0.0073,  ...,  0.0045, -0.0052, -0.0097],\n",
       "                       [-0.0098,  0.0044, -0.0063,  ...,  0.0161, -0.0084,  0.0008],\n",
       "                       [-0.0176, -0.0003, -0.0212,  ..., -0.0048,  0.0029,  0.0118],\n",
       "                       ...,\n",
       "                       [ 0.0156,  0.0139, -0.0130,  ...,  0.0134, -0.0105,  0.0194],\n",
       "                       [ 0.0078,  0.0159, -0.0105,  ..., -0.0062, -0.0166, -0.0099],\n",
       "                       [ 0.0086,  0.0015, -0.0094,  ...,  0.0062, -0.0133, -0.0152]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.attn.proj.bias',\n",
       "               tensor([ 0.0096,  0.0114, -0.0111,  ...,  0.0162, -0.0012, -0.0097],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.attn.ff.weight',\n",
       "               tensor([[ 0.0038, -0.0067, -0.0161,  ..., -0.0081, -0.0026, -0.0048],\n",
       "                       [-0.0121,  0.0087,  0.0189,  ...,  0.0001,  0.0055,  0.0087],\n",
       "                       [-0.0051,  0.0147, -0.0117,  ...,  0.0023, -0.0083, -0.0110],\n",
       "                       ...,\n",
       "                       [ 0.0087,  0.0058, -0.0012,  ...,  0.0181,  0.0064,  0.0123],\n",
       "                       [ 0.0101, -0.0143,  0.0047,  ...,  0.0097,  0.0112,  0.0172],\n",
       "                       [ 0.0073,  0.0107, -0.0090,  ...,  0.0060, -0.0048,  0.0154]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.attn.ff.bias',\n",
       "               tensor([ 5.8711e-05,  1.2779e-02, -5.6534e-03,  ...,  4.2305e-03,\n",
       "                        3.1605e-03,  3.8261e-03], dtype=torch.float16)),\n",
       "              ('blocks.23.ln1.w',\n",
       "               tensor([0.9907, 0.9976, 0.9883,  ..., 0.9917, 0.9941, 0.9941],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.ln1.b',\n",
       "               tensor([-0.0045, -0.0024,  0.0001,  ...,  0.0056,  0.0005,  0.0013],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.ln2.w',\n",
       "               tensor([1.0000, 1.0049, 1.0049,  ..., 1.0137, 1.0020, 0.9971],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.ln2.b',\n",
       "               tensor([-0.0028, -0.0010, -0.0029,  ...,  0.0021,  0.0044,  0.0002],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.mlp.ff1.weight',\n",
       "               tensor([[ 0.0132, -0.0066,  0.0113,  ..., -0.0002,  0.0031,  0.0092],\n",
       "                       [-0.0011, -0.0148,  0.0065,  ...,  0.0146, -0.0096,  0.0060],\n",
       "                       [-0.0017, -0.0156, -0.0168,  ..., -0.0123, -0.0011, -0.0162],\n",
       "                       ...,\n",
       "                       [-0.0153,  0.0176, -0.0112,  ...,  0.0160,  0.0154,  0.0048],\n",
       "                       [ 0.0117, -0.0088,  0.0148,  ..., -0.0022, -0.0121, -0.0215],\n",
       "                       [ 0.0109, -0.0094, -0.0030,  ..., -0.0072,  0.0170,  0.0005]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.mlp.ff1.bias',\n",
       "               tensor([ 0.0137, -0.0132, -0.0117,  ...,  0.0062, -0.0153, -0.0101],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.mlp.ff2.weight',\n",
       "               tensor([[ 0.0026,  0.0121,  0.0052,  ..., -0.0043, -0.0003, -0.0019],\n",
       "                       [ 0.0017, -0.0052,  0.0108,  ...,  0.0074, -0.0015,  0.0007],\n",
       "                       [-0.0003,  0.0053,  0.0087,  ..., -0.0040, -0.0069, -0.0030],\n",
       "                       ...,\n",
       "                       [ 0.0036, -0.0025, -0.0043,  ...,  0.0113, -0.0051, -0.0036],\n",
       "                       [-0.0022, -0.0088, -0.0079,  ...,  0.0004,  0.0094, -0.0092],\n",
       "                       [-0.0110, -0.0156,  0.0059,  ..., -0.0016,  0.0007, -0.0037]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.23.mlp.ff2.bias',\n",
       "               tensor([ 1.1883e-03, -8.1921e-04,  3.3073e-03,  ...,  2.9240e-03,\n",
       "                       -7.4883e-03, -1.8239e-05], dtype=torch.float16)),\n",
       "              ('blocks.24.attn.proj.weight',\n",
       "               tensor([[-0.0089, -0.0093,  0.0192,  ...,  0.0135,  0.0082, -0.0010],\n",
       "                       [-0.0020,  0.0148,  0.0116,  ..., -0.0023,  0.0017,  0.0091],\n",
       "                       [ 0.0024,  0.0054,  0.0227,  ..., -0.0054,  0.0025, -0.0088],\n",
       "                       ...,\n",
       "                       [-0.0121, -0.0025, -0.0050,  ...,  0.0081,  0.0015, -0.0127],\n",
       "                       [ 0.0079,  0.0124,  0.0117,  ..., -0.0043, -0.0184,  0.0191],\n",
       "                       [ 0.0138, -0.0107, -0.0222,  ...,  0.0081,  0.0121, -0.0023]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.attn.proj.bias',\n",
       "               tensor([ 0.0168,  0.0015, -0.0124,  ..., -0.0132,  0.0155, -0.0057],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.attn.ff.weight',\n",
       "               tensor([[-3.8147e-03,  1.1253e-02,  1.8570e-02,  ..., -3.1509e-03,\n",
       "                        -1.0216e-02,  1.1047e-02],\n",
       "                       [ 1.0635e-02,  1.6846e-02, -4.5929e-03,  ..., -1.6495e-02,\n",
       "                        -7.8812e-03, -8.3733e-04],\n",
       "                       [ 1.3268e-02,  1.7487e-02, -1.6724e-02,  ...,  1.7071e-04,\n",
       "                        -6.1684e-03, -1.3245e-02],\n",
       "                       ...,\n",
       "                       [-8.2321e-03,  1.0933e-02,  5.1422e-03,  ..., -1.4000e-02,\n",
       "                         1.8707e-02,  1.1696e-02],\n",
       "                       [ 4.3201e-04, -6.0883e-03,  1.1078e-02,  ...,  2.5928e-05,\n",
       "                        -1.3756e-02, -8.5602e-03],\n",
       "                       [-2.6360e-03, -5.7831e-03, -1.3313e-03,  ...,  1.5121e-02,\n",
       "                         6.5193e-03, -5.6000e-03]], dtype=torch.float16)),\n",
       "              ('blocks.24.attn.ff.bias',\n",
       "               tensor([-0.0122,  0.0091, -0.0171,  ..., -0.0015,  0.0088,  0.0044],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.ln1.w',\n",
       "               tensor([0.9888, 0.9878, 0.9819,  ..., 0.9932, 0.9878, 0.9868],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.ln1.b',\n",
       "               tensor([-0.0058, -0.0026,  0.0009,  ...,  0.0053,  0.0002,  0.0009],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.ln2.w',\n",
       "               tensor([1.0020, 1.0020, 1.0029,  ..., 1.0068, 1.0059, 1.0059],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.ln2.b',\n",
       "               tensor([-0.0043, -0.0043, -0.0002,  ..., -0.0006,  0.0033, -0.0021],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.mlp.ff1.weight',\n",
       "               tensor([[-0.0004, -0.0062, -0.0134,  ...,  0.0107,  0.0196, -0.0112],\n",
       "                       [ 0.0081, -0.0066,  0.0017,  ..., -0.0049,  0.0173, -0.0070],\n",
       "                       [-0.0155, -0.0118,  0.0008,  ...,  0.0044,  0.0220,  0.0163],\n",
       "                       ...,\n",
       "                       [ 0.0008,  0.0223, -0.0195,  ..., -0.0094,  0.0116, -0.0069],\n",
       "                       [-0.0181, -0.0201,  0.0044,  ...,  0.0111,  0.0019,  0.0113],\n",
       "                       [ 0.0155, -0.0036, -0.0004,  ...,  0.0052, -0.0106, -0.0120]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.mlp.ff1.bias',\n",
       "               tensor([ 0.0092, -0.0102,  0.0155,  ..., -0.0096,  0.0114,  0.0095],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.24.mlp.ff2.weight',\n",
       "               tensor([[-2.0256e-03, -5.8022e-03, -4.2610e-03,  ..., -8.1482e-03,\n",
       "                        -2.2316e-04, -1.9627e-03],\n",
       "                       [-6.0921e-03, -1.9417e-03,  8.8882e-03,  ..., -1.2604e-02,\n",
       "                         5.8250e-03,  1.5631e-03],\n",
       "                       [ 2.0504e-03,  7.5912e-03,  6.1226e-03,  ...,  7.0534e-03,\n",
       "                         8.1940e-03, -4.3631e-04],\n",
       "                       ...,\n",
       "                       [ 5.4131e-03, -7.8678e-05, -1.1568e-03,  ...,  6.4735e-03,\n",
       "                        -4.1237e-03,  7.8125e-03],\n",
       "                       [-6.1302e-03, -9.6054e-03, -1.1040e-02,  ...,  3.3760e-03,\n",
       "                        -2.4796e-03, -1.9989e-03],\n",
       "                       [-9.3699e-04, -5.3825e-03,  2.6665e-03,  ...,  5.3978e-04,\n",
       "                         2.4071e-03,  6.7596e-03]], dtype=torch.float16)),\n",
       "              ('blocks.24.mlp.ff2.bias',\n",
       "               tensor([-0.0012,  0.0007, -0.0043,  ...,  0.0022,  0.0011, -0.0081],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.attn.proj.weight',\n",
       "               tensor([[-1.2566e-02,  2.0172e-02, -3.2654e-03,  ..., -1.0918e-02,\n",
       "                        -6.9237e-03,  4.3030e-03],\n",
       "                       [-9.1171e-03, -3.7174e-03,  1.2085e-02,  ...,  1.7452e-03,\n",
       "                        -1.1730e-03, -4.7326e-05],\n",
       "                       [-1.0300e-02, -2.6188e-03,  1.7502e-02,  ..., -5.5466e-03,\n",
       "                        -6.2561e-03,  1.2878e-02],\n",
       "                       ...,\n",
       "                       [-3.9062e-03, -1.6809e-05,  1.5366e-02,  ..., -1.8539e-02,\n",
       "                        -6.1226e-03,  1.2779e-02],\n",
       "                       [-1.6068e-02,  2.1652e-02,  1.4412e-02,  ...,  1.6617e-02,\n",
       "                        -4.8332e-03, -8.7547e-04],\n",
       "                       [ 5.6190e-03, -1.2970e-02, -8.8043e-03,  ...,  1.0422e-02,\n",
       "                        -1.7681e-03, -5.4970e-03]], dtype=torch.float16)),\n",
       "              ('blocks.25.attn.proj.bias',\n",
       "               tensor([-0.0125,  0.0177, -0.0051,  ..., -0.0130, -0.0064,  0.0173],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.attn.ff.weight',\n",
       "               tensor([[-0.0054, -0.0106, -0.0079,  ...,  0.0040,  0.0178,  0.0075],\n",
       "                       [ 0.0173,  0.0109,  0.0176,  ..., -0.0113, -0.0084, -0.0088],\n",
       "                       [-0.0066,  0.0197, -0.0060,  ..., -0.0009,  0.0041, -0.0059],\n",
       "                       ...,\n",
       "                       [-0.0140,  0.0071, -0.0139,  ..., -0.0085, -0.0125, -0.0008],\n",
       "                       [-0.0081, -0.0026, -0.0082,  ..., -0.0010,  0.0021, -0.0083],\n",
       "                       [ 0.0115, -0.0040, -0.0005,  ..., -0.0106,  0.0134,  0.0200]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.attn.ff.bias',\n",
       "               tensor([ 0.0019,  0.0172, -0.0087,  ..., -0.0051,  0.0025, -0.0027],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.ln1.w',\n",
       "               tensor([0.9893, 0.9897, 0.9927,  ..., 0.9917, 0.9873, 0.9912],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.ln1.b',\n",
       "               tensor([-5.1880e-03, -7.0333e-06,  1.0719e-03,  ...,  3.6240e-03,\n",
       "                        1.2779e-03,  6.4039e-04], dtype=torch.float16)),\n",
       "              ('blocks.25.ln2.w',\n",
       "               tensor([1.0010, 1.0146, 1.0068,  ..., 1.0059, 1.0059, 0.9990],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.ln2.b',\n",
       "               tensor([-0.0003, -0.0069,  0.0017,  ..., -0.0008,  0.0061,  0.0007],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.mlp.ff1.weight',\n",
       "               tensor([[-0.0072,  0.0026, -0.0135,  ...,  0.0108,  0.0046,  0.0083],\n",
       "                       [ 0.0110,  0.0117,  0.0147,  ..., -0.0081, -0.0113, -0.0014],\n",
       "                       [-0.0130, -0.0074,  0.0061,  ...,  0.0131, -0.0071,  0.0162],\n",
       "                       ...,\n",
       "                       [-0.0050,  0.0152, -0.0031,  ...,  0.0134,  0.0177,  0.0154],\n",
       "                       [ 0.0200,  0.0009,  0.0022,  ..., -0.0166,  0.0121, -0.0100],\n",
       "                       [ 0.0004,  0.0075,  0.0043,  ..., -0.0211,  0.0165,  0.0013]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.mlp.ff1.bias',\n",
       "               tensor([-0.0032,  0.0025, -0.0110,  ..., -0.0159, -0.0008, -0.0173],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.mlp.ff2.weight',\n",
       "               tensor([[-0.0024, -0.0110, -0.0078,  ..., -0.0072, -0.0014, -0.0047],\n",
       "                       [ 0.0028, -0.0018, -0.0020,  ..., -0.0026, -0.0012, -0.0106],\n",
       "                       [ 0.0023, -0.0078,  0.0030,  ...,  0.0117,  0.0018, -0.0024],\n",
       "                       ...,\n",
       "                       [-0.0021, -0.0040, -0.0067,  ..., -0.0036,  0.0041,  0.0105],\n",
       "                       [-0.0044, -0.0011,  0.0070,  ...,  0.0014, -0.0101, -0.0048],\n",
       "                       [-0.0091,  0.0081,  0.0048,  ...,  0.0037, -0.0015,  0.0014]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.25.mlp.ff2.bias',\n",
       "               tensor([-0.0015, -0.0054, -0.0011,  ...,  0.0099, -0.0058,  0.0014],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.attn.proj.weight',\n",
       "               tensor([[-0.0168,  0.0035,  0.0204,  ...,  0.0169, -0.0135, -0.0086],\n",
       "                       [-0.0140,  0.0015,  0.0012,  ...,  0.0223,  0.0227, -0.0047],\n",
       "                       [ 0.0166, -0.0084, -0.0133,  ...,  0.0155, -0.0059,  0.0051],\n",
       "                       ...,\n",
       "                       [-0.0102,  0.0087, -0.0009,  ..., -0.0040, -0.0038,  0.0162],\n",
       "                       [-0.0106,  0.0092,  0.0029,  ...,  0.0180,  0.0173,  0.0134],\n",
       "                       [-0.0017,  0.0019,  0.0084,  ..., -0.0167,  0.0012,  0.0072]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.attn.proj.bias',\n",
       "               tensor([-0.0055, -0.0030,  0.0025,  ..., -0.0009,  0.0114,  0.0002],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.attn.ff.weight',\n",
       "               tensor([[-0.0148,  0.0031,  0.0074,  ..., -0.0018, -0.0201,  0.0003],\n",
       "                       [-0.0174,  0.0122,  0.0101,  ...,  0.0059, -0.0025,  0.0049],\n",
       "                       [-0.0108, -0.0030, -0.0063,  ..., -0.0116,  0.0062, -0.0187],\n",
       "                       ...,\n",
       "                       [-0.0021, -0.0140, -0.0192,  ...,  0.0067,  0.0038, -0.0100],\n",
       "                       [ 0.0073,  0.0028,  0.0057,  ..., -0.0087, -0.0041, -0.0024],\n",
       "                       [ 0.0153,  0.0012, -0.0180,  ..., -0.0002, -0.0093,  0.0146]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.attn.ff.bias',\n",
       "               tensor([-0.0131,  0.0101, -0.0061,  ...,  0.0102, -0.0097,  0.0084],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.ln1.w',\n",
       "               tensor([0.9907, 0.9893, 0.9922,  ..., 0.9922, 0.9912, 0.9956],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.ln1.b',\n",
       "               tensor([-0.0045, -0.0026,  0.0026,  ...,  0.0026,  0.0014,  0.0008],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.ln2.w',\n",
       "               tensor([1.0020, 0.9990, 1.0117,  ..., 1.0068, 1.0088, 1.0049],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.ln2.b',\n",
       "               tensor([-0.0023, -0.0029,  0.0039,  ..., -0.0044,  0.0020,  0.0019],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.mlp.ff1.weight',\n",
       "               tensor([[-0.0051,  0.0167, -0.0103,  ...,  0.0168,  0.0217,  0.0161],\n",
       "                       [ 0.0061, -0.0050, -0.0077,  ..., -0.0009, -0.0167, -0.0020],\n",
       "                       [-0.0105,  0.0113, -0.0043,  ..., -0.0168,  0.0112, -0.0071],\n",
       "                       ...,\n",
       "                       [ 0.0031,  0.0019, -0.0026,  ...,  0.0025,  0.0116,  0.0013],\n",
       "                       [ 0.0144,  0.0108,  0.0078,  ..., -0.0006,  0.0030, -0.0132],\n",
       "                       [ 0.0095, -0.0194, -0.0091,  ..., -0.0045, -0.0079, -0.0015]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.mlp.ff1.bias',\n",
       "               tensor([ 0.0010, -0.0004,  0.0161,  ...,  0.0102,  0.0023, -0.0081],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.26.mlp.ff2.weight',\n",
       "               tensor([[ 1.0204e-03,  3.1128e-03,  3.5515e-03,  ..., -1.1307e-02,\n",
       "                         7.1831e-03,  3.5458e-03],\n",
       "                       [-8.7404e-04, -4.6844e-03, -6.1150e-03,  ..., -2.6166e-05,\n",
       "                        -6.2447e-03,  6.2065e-03],\n",
       "                       [ 9.3689e-03,  1.4214e-02, -2.0428e-03,  ...,  7.7667e-03,\n",
       "                        -1.6108e-03, -1.0157e-03],\n",
       "                       ...,\n",
       "                       [-9.7046e-03,  7.0305e-03,  2.6073e-03,  ..., -5.4092e-03,\n",
       "                         4.4022e-03, -5.4092e-03],\n",
       "                       [-1.0635e-02,  5.5122e-03, -8.7357e-03,  ..., -1.5860e-03,\n",
       "                         1.1005e-03, -6.4707e-04],\n",
       "                       [-9.7847e-04, -2.9049e-03, -6.6223e-03,  ...,  6.8760e-04,\n",
       "                         2.3384e-03,  4.7035e-03]], dtype=torch.float16)),\n",
       "              ('blocks.26.mlp.ff2.bias',\n",
       "               tensor([ 0.0074, -0.0076, -0.0014,  ..., -0.0066,  0.0054,  0.0030],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.attn.proj.weight',\n",
       "               tensor([[-0.0054, -0.0015,  0.0004,  ..., -0.0018, -0.0170,  0.0020],\n",
       "                       [-0.0170, -0.0085,  0.0055,  ..., -0.0154,  0.0061, -0.0205],\n",
       "                       [ 0.0215,  0.0025, -0.0049,  ..., -0.0034, -0.0135,  0.0097],\n",
       "                       ...,\n",
       "                       [ 0.0007,  0.0009, -0.0091,  ..., -0.0007,  0.0066,  0.0247],\n",
       "                       [ 0.0006, -0.0182,  0.0159,  ..., -0.0186,  0.0137, -0.0013],\n",
       "                       [-0.0001, -0.0090,  0.0086,  ..., -0.0131, -0.0041, -0.0008]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.attn.proj.bias',\n",
       "               tensor([-0.0091,  0.0111, -0.0168,  ...,  0.0124, -0.0135,  0.0016],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.attn.ff.weight',\n",
       "               tensor([[-1.0483e-02, -8.3160e-03,  1.3268e-02,  ..., -6.7520e-03,\n",
       "                        -8.8577e-03,  1.2102e-03],\n",
       "                       [-1.7395e-02, -4.7493e-03,  1.5442e-02,  ..., -1.8768e-03,\n",
       "                         1.1826e-03,  8.7051e-03],\n",
       "                       [-1.0780e-02,  1.6983e-02,  3.3112e-03,  ..., -1.8299e-05,\n",
       "                        -3.4409e-03,  3.1452e-03],\n",
       "                       ...,\n",
       "                       [ 1.0086e-02,  1.5129e-02,  4.0436e-03,  ...,  1.0269e-02,\n",
       "                        -3.7613e-03, -9.3079e-03],\n",
       "                       [ 1.7548e-03,  1.1360e-02, -9.1171e-03,  ...,  9.3918e-03,\n",
       "                        -2.3590e-02, -3.2253e-03],\n",
       "                       [ 4.4441e-03,  1.2222e-02, -1.0315e-02,  ..., -1.2960e-03,\n",
       "                        -6.8092e-03, -1.0338e-02]], dtype=torch.float16)),\n",
       "              ('blocks.27.attn.ff.bias',\n",
       "               tensor([ 0.0060, -0.0068, -0.0157,  ..., -0.0004,  0.0091, -0.0167],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.ln1.w',\n",
       "               tensor([0.9897, 0.9844, 0.9878,  ..., 0.9873, 0.9868, 0.9893],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.ln1.b',\n",
       "               tensor([-0.0035, -0.0032,  0.0023,  ...,  0.0043,  0.0051,  0.0005],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.ln2.w',\n",
       "               tensor([1.0000, 1.0088, 1.0107,  ..., 1.0029, 1.0020, 0.9995],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.ln2.b',\n",
       "               tensor([ 1.5821e-03, -3.1519e-04, -5.2452e-04,  ..., -3.9339e-05,\n",
       "                        4.0321e-03, -3.8075e-04], dtype=torch.float16)),\n",
       "              ('blocks.27.mlp.ff1.weight',\n",
       "               tensor([[ 0.0073, -0.0183, -0.0072,  ...,  0.0116, -0.0033,  0.0140],\n",
       "                       [-0.0087,  0.0169, -0.0071,  ..., -0.0136, -0.0122, -0.0275],\n",
       "                       [ 0.0074, -0.0211,  0.0018,  ..., -0.0167, -0.0023, -0.0094],\n",
       "                       ...,\n",
       "                       [-0.0184,  0.0110,  0.0012,  ...,  0.0140,  0.0158, -0.0174],\n",
       "                       [-0.0176,  0.0028, -0.0153,  ...,  0.0026, -0.0089, -0.0050],\n",
       "                       [-0.0086,  0.0045,  0.0133,  ...,  0.0034,  0.0184,  0.0117]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.mlp.ff1.bias',\n",
       "               tensor([-0.0126,  0.0047,  0.0088,  ..., -0.0032, -0.0018,  0.0092],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.mlp.ff2.weight',\n",
       "               tensor([[ 0.0037,  0.0034, -0.0094,  ..., -0.0003, -0.0030,  0.0033],\n",
       "                       [ 0.0026,  0.0018,  0.0167,  ...,  0.0021,  0.0064, -0.0046],\n",
       "                       [ 0.0085, -0.0105, -0.0027,  ...,  0.0019,  0.0055, -0.0038],\n",
       "                       ...,\n",
       "                       [-0.0063,  0.0004, -0.0002,  ..., -0.0051, -0.0085,  0.0012],\n",
       "                       [-0.0004, -0.0075, -0.0004,  ...,  0.0051,  0.0029,  0.0029],\n",
       "                       [-0.0024,  0.0074, -0.0015,  ...,  0.0078,  0.0020, -0.0022]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.27.mlp.ff2.bias',\n",
       "               tensor([ 0.0030,  0.0007,  0.0024,  ..., -0.0050, -0.0085,  0.0065],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.attn.proj.weight',\n",
       "               tensor([[ 0.0120, -0.0108, -0.0001,  ..., -0.0099,  0.0076,  0.0207],\n",
       "                       [-0.0082,  0.0002, -0.0110,  ...,  0.0068, -0.0126,  0.0093],\n",
       "                       [ 0.0168,  0.0036,  0.0091,  ...,  0.0140,  0.0047,  0.0125],\n",
       "                       ...,\n",
       "                       [ 0.0133,  0.0046, -0.0119,  ..., -0.0057, -0.0045, -0.0027],\n",
       "                       [ 0.0117,  0.0073,  0.0177,  ..., -0.0028,  0.0048,  0.0097],\n",
       "                       [-0.0030, -0.0009,  0.0126,  ..., -0.0196,  0.0078, -0.0039]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.attn.proj.bias',\n",
       "               tensor([ 0.0026, -0.0004, -0.0075,  ...,  0.0117,  0.0024,  0.0063],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.attn.ff.weight',\n",
       "               tensor([[ 0.0034,  0.0099, -0.0066,  ...,  0.0041, -0.0136, -0.0121],\n",
       "                       [-0.0142, -0.0070,  0.0028,  ...,  0.0049,  0.0159, -0.0015],\n",
       "                       [-0.0158,  0.0145, -0.0135,  ..., -0.0046,  0.0029, -0.0068],\n",
       "                       ...,\n",
       "                       [ 0.0152, -0.0120, -0.0064,  ...,  0.0033,  0.0114,  0.0138],\n",
       "                       [ 0.0116,  0.0104, -0.0024,  ...,  0.0033,  0.0059, -0.0100],\n",
       "                       [ 0.0025,  0.0003, -0.0041,  ..., -0.0005,  0.0057, -0.0099]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.attn.ff.bias',\n",
       "               tensor([ 0.0101, -0.0124, -0.0011,  ...,  0.0039,  0.0103, -0.0055],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.ln1.w',\n",
       "               tensor([0.9814, 0.9849, 0.9883,  ..., 0.9854, 0.9829, 0.9883],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.ln1.b',\n",
       "               tensor([-5.9471e-03, -1.0080e-03,  6.6566e-04,  ...,  2.6913e-03,\n",
       "                        3.9291e-03, -4.5598e-05], dtype=torch.float16)),\n",
       "              ('blocks.28.ln2.w',\n",
       "               tensor([1.0020, 1.0098, 1.0107,  ..., 1.0000, 1.0049, 1.0010],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.ln2.b',\n",
       "               tensor([ 0.0018,  0.0031,  0.0061,  ..., -0.0041,  0.0022, -0.0017],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.mlp.ff1.weight',\n",
       "               tensor([[ 0.0090,  0.0108, -0.0030,  ...,  0.0108, -0.0069, -0.0121],\n",
       "                       [ 0.0208,  0.0177, -0.0081,  ..., -0.0114,  0.0123, -0.0113],\n",
       "                       [ 0.0171,  0.0003, -0.0115,  ..., -0.0049, -0.0034,  0.0010],\n",
       "                       ...,\n",
       "                       [ 0.0017,  0.0024,  0.0102,  ..., -0.0192, -0.0058,  0.0011],\n",
       "                       [-0.0080, -0.0194,  0.0109,  ...,  0.0123, -0.0133,  0.0142],\n",
       "                       [-0.0209, -0.0113,  0.0164,  ...,  0.0034,  0.0137,  0.0119]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.mlp.ff1.bias',\n",
       "               tensor([-0.0029, -0.0188,  0.0176,  ..., -0.0107,  0.0160, -0.0181],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.28.mlp.ff2.weight',\n",
       "               tensor([[ 2.1114e-03, -6.9656e-03,  1.7233e-03,  ..., -7.7677e-04,\n",
       "                        -3.3264e-03,  3.0270e-03],\n",
       "                       [-1.1047e-02, -1.1330e-02,  8.0466e-05,  ..., -1.8101e-03,\n",
       "                         1.5594e-02, -4.3678e-03],\n",
       "                       [-1.5192e-03,  9.1095e-03,  8.3389e-03,  ...,  2.0313e-03,\n",
       "                        -6.1073e-03, -1.2756e-02],\n",
       "                       ...,\n",
       "                       [-1.9665e-03,  4.4250e-03,  1.4753e-03,  ..., -5.6801e-03,\n",
       "                         3.4351e-03,  1.0080e-03],\n",
       "                       [-1.9703e-03, -6.4430e-03,  2.6093e-03,  ...,  2.4719e-03,\n",
       "                         1.0323e-02, -9.0561e-03],\n",
       "                       [ 8.8501e-03,  1.1520e-02, -8.8348e-03,  ...,  3.5324e-03,\n",
       "                        -3.6259e-03,  1.3428e-03]], dtype=torch.float16)),\n",
       "              ('blocks.28.mlp.ff2.bias',\n",
       "               tensor([-0.0041, -0.0008,  0.0059,  ..., -0.0066, -0.0050, -0.0011],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.29.attn.proj.weight',\n",
       "               tensor([[ 0.0064,  0.0133,  0.0070,  ...,  0.0158, -0.0007,  0.0181],\n",
       "                       [-0.0004, -0.0149, -0.0044,  ..., -0.0227, -0.0046, -0.0005],\n",
       "                       [ 0.0043, -0.0052, -0.0097,  ...,  0.0023, -0.0051, -0.0156],\n",
       "                       ...,\n",
       "                       [-0.0077, -0.0117,  0.0050,  ...,  0.0143,  0.0117, -0.0152],\n",
       "                       [-0.0118,  0.0137, -0.0036,  ...,  0.0087,  0.0081, -0.0162],\n",
       "                       [ 0.0026, -0.0053,  0.0111,  ...,  0.0175,  0.0137,  0.0002]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.29.attn.proj.bias',\n",
       "               tensor([-5.4054e-03, -1.0292e-02, -1.5011e-03,  ..., -7.4744e-05,\n",
       "                        9.7885e-03,  1.9093e-03], dtype=torch.float16)),\n",
       "              ('blocks.29.attn.ff.weight',\n",
       "               tensor([[ 8.8739e-04,  1.0635e-02, -1.7941e-05,  ..., -6.3324e-03,\n",
       "                         8.0872e-03, -2.1820e-03],\n",
       "                       [-1.5955e-03, -1.1742e-02,  8.4381e-03,  ...,  1.5617e-02,\n",
       "                        -1.6907e-02, -9.0866e-03],\n",
       "                       [ 1.4244e-02, -4.9934e-03,  8.1482e-03,  ..., -1.0185e-02,\n",
       "                        -1.1696e-02,  1.7426e-02],\n",
       "                       ...,\n",
       "                       [ 4.2381e-03,  6.9275e-03, -1.2451e-02,  ...,  1.0582e-02,\n",
       "                         1.5274e-02,  1.2680e-02],\n",
       "                       [ 5.8632e-03,  6.3820e-03,  7.8812e-03,  ..., -1.1551e-02,\n",
       "                        -1.3412e-02,  3.3989e-03],\n",
       "                       [-5.9204e-03, -1.6586e-02,  1.7807e-02,  ...,  4.5853e-03,\n",
       "                         1.5930e-02,  2.5768e-03]], dtype=torch.float16)),\n",
       "              ('blocks.29.attn.ff.bias',\n",
       "               tensor([-0.0090, -0.0152,  0.0119,  ...,  0.0110, -0.0100, -0.0107],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.29.ln1.w',\n",
       "               tensor([0.9858, 0.9819, 0.9893,  ..., 0.9868, 0.9854, 0.9883],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.29.ln1.b',\n",
       "               tensor([-4.0474e-03,  5.2035e-05,  3.1853e-04,  ...,  4.2648e-03,\n",
       "                        4.9057e-03,  7.3576e-04], dtype=torch.float16)),\n",
       "              ('blocks.29.ln2.w',\n",
       "               tensor([0.9985, 1.0107, 1.0068,  ..., 1.0029, 0.9990, 1.0020],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.29.ln2.b',\n",
       "               tensor([ 0.0002,  0.0031, -0.0010,  ..., -0.0059,  0.0043, -0.0037],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.29.mlp.ff1.weight',\n",
       "               tensor([[-8.0204e-04,  1.8417e-02, -1.7262e-03,  ..., -4.3602e-03,\n",
       "                         1.4847e-02,  1.2642e-02],\n",
       "                       [-4.6501e-03, -5.6076e-03,  1.0315e-02,  ..., -1.0696e-02,\n",
       "                         1.7410e-02, -1.3382e-02],\n",
       "                       [ 8.8959e-03,  1.8950e-03,  5.4054e-03,  ...,  4.0855e-03,\n",
       "                        -2.9411e-03, -8.1635e-03],\n",
       "                       ...,\n",
       "                       [ 5.3940e-03, -4.0054e-03,  2.4414e-02,  ..., -7.4506e-05,\n",
       "                        -8.8501e-03, -1.0872e-02],\n",
       "                       [ 1.7212e-02, -1.1116e-02,  5.3825e-03,  ...,  8.1787e-03,\n",
       "                        -1.3672e-02,  2.3327e-03],\n",
       "                       [ 8.5983e-03, -1.2489e-02, -1.1282e-03,  ..., -1.0643e-02,\n",
       "                         3.3689e-04,  8.7204e-03]], dtype=torch.float16)),\n",
       "              ('blocks.29.mlp.ff1.bias',\n",
       "               tensor([-0.0021,  0.0036,  0.0052,  ...,  0.0076, -0.0005,  0.0097],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.29.mlp.ff2.weight',\n",
       "               tensor([[ 0.0068,  0.0016,  0.0039,  ...,  0.0075, -0.0037, -0.0053],\n",
       "                       [-0.0122, -0.0060, -0.0040,  ...,  0.0006,  0.0073,  0.0079],\n",
       "                       [ 0.0101,  0.0021,  0.0058,  ..., -0.0014, -0.0066, -0.0010],\n",
       "                       ...,\n",
       "                       [ 0.0036, -0.0039, -0.0089,  ...,  0.0022,  0.0038,  0.0035],\n",
       "                       [-0.0021, -0.0044,  0.0096,  ..., -0.0040,  0.0064,  0.0001],\n",
       "                       [-0.0100,  0.0087, -0.0022,  ...,  0.0016,  0.0034, -0.0051]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.29.mlp.ff2.bias',\n",
       "               tensor([ 0.0021,  0.0041, -0.0068,  ..., -0.0084,  0.0051,  0.0035],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.attn.proj.weight',\n",
       "               tensor([[ 3.0255e-04, -1.0765e-02, -2.2678e-03,  ...,  7.1287e-05,\n",
       "                        -6.1684e-03,  1.4656e-02],\n",
       "                       [-1.6891e-02,  1.5900e-02,  7.6256e-03,  ..., -7.6370e-03,\n",
       "                        -1.4915e-02, -1.5879e-03],\n",
       "                       [-1.9760e-02, -1.1131e-02,  1.3756e-02,  ...,  6.1684e-03,\n",
       "                        -1.8387e-02,  1.3458e-02],\n",
       "                       ...,\n",
       "                       [-1.0300e-02, -9.5062e-03,  5.9967e-03,  ..., -1.4114e-02,\n",
       "                         1.1101e-02, -1.5671e-02],\n",
       "                       [ 1.4664e-02,  1.1230e-02,  2.1561e-02,  ..., -1.1612e-02,\n",
       "                         3.4885e-03,  1.0353e-02],\n",
       "                       [-4.3488e-03, -1.4442e-02, -4.0245e-03,  ...,  4.0245e-03,\n",
       "                        -5.9166e-03,  1.0239e-02]], dtype=torch.float16)),\n",
       "              ('blocks.30.attn.proj.bias',\n",
       "               tensor([ 0.0108,  0.0004,  0.0169,  ..., -0.0034, -0.0067, -0.0061],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.attn.ff.weight',\n",
       "               tensor([[-2.4586e-03, -1.5038e-02,  1.6617e-02,  ..., -1.2802e-02,\n",
       "                         5.3787e-03,  1.5228e-02],\n",
       "                       [ 1.8509e-02,  1.4557e-02,  1.1490e-02,  ..., -7.4720e-04,\n",
       "                        -2.1400e-03,  1.3802e-02],\n",
       "                       [-1.8559e-03,  8.8501e-03, -5.9509e-03,  ..., -7.7171e-03,\n",
       "                        -5.6190e-03, -3.5324e-03],\n",
       "                       ...,\n",
       "                       [-4.6501e-03,  1.3687e-02, -4.7035e-03,  ...,  1.9569e-03,\n",
       "                         1.5173e-03, -8.2092e-03],\n",
       "                       [ 1.1429e-02,  4.2737e-05, -2.1255e-02,  ...,  1.5869e-03,\n",
       "                        -2.8934e-03, -6.2485e-03],\n",
       "                       [-6.8626e-03,  7.3624e-03, -1.6052e-02,  ..., -3.4313e-03,\n",
       "                         1.2833e-02, -1.6006e-02]], dtype=torch.float16)),\n",
       "              ('blocks.30.attn.ff.bias',\n",
       "               tensor([-0.0089, -0.0164, -0.0063,  ..., -0.0027,  0.0086,  0.0108],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.ln1.w',\n",
       "               tensor([0.9849, 0.9829, 0.9824,  ..., 0.9800, 0.9883, 0.9839],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.ln1.b',\n",
       "               tensor([-0.0045, -0.0012, -0.0002,  ...,  0.0036,  0.0047,  0.0007],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.ln2.w',\n",
       "               tensor([0.9980, 1.0059, 1.0098,  ..., 1.0049, 0.9995, 1.0020],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.ln2.b',\n",
       "               tensor([-0.0028,  0.0040, -0.0014,  ..., -0.0097,  0.0027, -0.0052],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.mlp.ff1.weight',\n",
       "               tensor([[-0.0184,  0.0007, -0.0170,  ..., -0.0017, -0.0072, -0.0116],\n",
       "                       [ 0.0070, -0.0016,  0.0064,  ..., -0.0112,  0.0032,  0.0129],\n",
       "                       [ 0.0100,  0.0105,  0.0168,  ..., -0.0120,  0.0179, -0.0080],\n",
       "                       ...,\n",
       "                       [-0.0159,  0.0164, -0.0241,  ..., -0.0202,  0.0191,  0.0138],\n",
       "                       [ 0.0075, -0.0154,  0.0051,  ..., -0.0038,  0.0078, -0.0075],\n",
       "                       [ 0.0120, -0.0171,  0.0057,  ...,  0.0151, -0.0149, -0.0128]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.mlp.ff1.bias',\n",
       "               tensor([-0.0120, -0.0091, -0.0100,  ...,  0.0005, -0.0067, -0.0073],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.30.mlp.ff2.weight',\n",
       "               tensor([[ 1.0849e-02,  3.6030e-03, -5.9776e-03,  ...,  7.4997e-03,\n",
       "                         5.4884e-04, -1.1721e-03],\n",
       "                       [-1.4095e-03,  2.9163e-03, -4.3793e-03,  ...,  8.8811e-05,\n",
       "                        -9.6054e-03,  1.2756e-02],\n",
       "                       [ 3.7994e-03, -9.7466e-04, -3.8967e-03,  ...,  6.9466e-03,\n",
       "                        -1.1551e-02,  1.1053e-03],\n",
       "                       ...,\n",
       "                       [-3.5405e-04,  6.0692e-03, -3.0499e-03,  ...,  1.1272e-03,\n",
       "                         4.6082e-03, -1.0208e-02],\n",
       "                       [ 1.1787e-02, -3.1815e-03,  9.3889e-04,  ..., -8.5526e-03,\n",
       "                        -4.8790e-03, -6.2294e-03],\n",
       "                       [-7.4768e-03, -5.7983e-03, -6.4392e-03,  ..., -8.5983e-03,\n",
       "                         2.2602e-03,  5.8289e-03]], dtype=torch.float16)),\n",
       "              ('blocks.30.mlp.ff2.bias',\n",
       "               tensor([ 0.0036, -0.0001,  0.0096,  ...,  0.0083, -0.0061,  0.0053],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.attn.proj.weight',\n",
       "               tensor([[-0.0071, -0.0167,  0.0019,  ..., -0.0167,  0.0151,  0.0071],\n",
       "                       [-0.0045, -0.0082, -0.0007,  ..., -0.0123, -0.0013, -0.0077],\n",
       "                       [-0.0031,  0.0176, -0.0065,  ...,  0.0050, -0.0055,  0.0113],\n",
       "                       ...,\n",
       "                       [ 0.0062,  0.0054, -0.0051,  ..., -0.0164,  0.0072,  0.0153],\n",
       "                       [ 0.0075, -0.0145, -0.0089,  ...,  0.0133,  0.0163, -0.0130],\n",
       "                       [ 0.0085, -0.0094, -0.0046,  ..., -0.0004, -0.0084, -0.0020]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.attn.proj.bias',\n",
       "               tensor([-0.0203,  0.0167,  0.0138,  ..., -0.0152, -0.0073,  0.0106],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.attn.ff.weight',\n",
       "               tensor([[-0.0121, -0.0140, -0.0072,  ...,  0.0051, -0.0096,  0.0123],\n",
       "                       [ 0.0095, -0.0032,  0.0091,  ...,  0.0140, -0.0052, -0.0127],\n",
       "                       [-0.0142,  0.0041, -0.0093,  ..., -0.0073,  0.0050, -0.0132],\n",
       "                       ...,\n",
       "                       [-0.0059,  0.0060,  0.0078,  ...,  0.0013,  0.0104,  0.0155],\n",
       "                       [-0.0121,  0.0131, -0.0054,  ..., -0.0017, -0.0096, -0.0104],\n",
       "                       [ 0.0117,  0.0165,  0.0139,  ..., -0.0008,  0.0084,  0.0122]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.attn.ff.bias',\n",
       "               tensor([ 0.0072, -0.0083,  0.0179,  ..., -0.0090, -0.0001, -0.0059],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.ln1.w',\n",
       "               tensor([0.9854, 0.9854, 0.9819,  ..., 0.9839, 0.9829, 0.9863],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.ln1.b',\n",
       "               tensor([-0.0040,  0.0007,  0.0004,  ...,  0.0020,  0.0053,  0.0003],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.ln2.w',\n",
       "               tensor([0.9897, 0.9966, 0.9932,  ..., 0.9902, 0.9912, 0.9917],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.ln2.b',\n",
       "               tensor([-0.0038,  0.0019, -0.0011,  ..., -0.0051,  0.0021, -0.0031],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.mlp.ff1.weight',\n",
       "               tensor([[ 0.0089, -0.0093, -0.0054,  ..., -0.0006, -0.0050, -0.0178],\n",
       "                       [ 0.0025,  0.0155,  0.0003,  ..., -0.0148, -0.0026, -0.0009],\n",
       "                       [-0.0002, -0.0007, -0.0006,  ...,  0.0115, -0.0114, -0.0077],\n",
       "                       ...,\n",
       "                       [-0.0114, -0.0121, -0.0025,  ...,  0.0117, -0.0107, -0.0107],\n",
       "                       [-0.0049, -0.0159,  0.0059,  ...,  0.0023,  0.0100,  0.0144],\n",
       "                       [-0.0088,  0.0074, -0.0111,  ..., -0.0147, -0.0160, -0.0137]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.mlp.ff1.bias',\n",
       "               tensor([ 0.0226,  0.0010,  0.0077,  ...,  0.0156, -0.0170, -0.0152],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.mlp.ff2.weight',\n",
       "               tensor([[-0.0052,  0.0014, -0.0016,  ..., -0.0072,  0.0003, -0.0083],\n",
       "                       [ 0.0021, -0.0086,  0.0003,  ...,  0.0091,  0.0006, -0.0035],\n",
       "                       [ 0.0064,  0.0034, -0.0006,  ..., -0.0036, -0.0014, -0.0025],\n",
       "                       ...,\n",
       "                       [ 0.0014,  0.0102, -0.0081,  ..., -0.0120,  0.0066, -0.0030],\n",
       "                       [-0.0007,  0.0085, -0.0035,  ..., -0.0056, -0.0070,  0.0027],\n",
       "                       [ 0.0078, -0.0071,  0.0077,  ...,  0.0112,  0.0016,  0.0049]],\n",
       "                      dtype=torch.float16)),\n",
       "              ('blocks.31.mlp.ff2.bias',\n",
       "               tensor([ 0.0082, -0.0049, -0.0008,  ..., -0.0038, -0.0003,  0.0069],\n",
       "                      dtype=torch.float16)),\n",
       "              ('ln.w',\n",
       "               tensor([1.0742, 1.0957, 1.1006,  ..., 1.0967, 1.0850, 1.0947],\n",
       "                      dtype=torch.float16)),\n",
       "              ('ln.b',\n",
       "               tensor([ 0.0260, -0.0132,  0.0081,  ...,  0.0026, -0.0245,  0.0175],\n",
       "                      dtype=torch.float16)),\n",
       "              ('decoder.weight',\n",
       "               tensor([[-0.0285, -0.0087, -0.0026,  ..., -0.0071,  0.0049, -0.0130],\n",
       "                       [-0.0316,  0.0158, -0.0020,  ...,  0.0024,  0.0191, -0.0114],\n",
       "                       [-0.0313,  0.0080,  0.0088,  ...,  0.0008,  0.0304,  0.0024],\n",
       "                       ...,\n",
       "                       [-0.0070,  0.0053,  0.0062,  ..., -0.0134,  0.0176, -0.0075],\n",
       "                       [-0.0109, -0.0064, -0.0061,  ...,  0.0172,  0.0152, -0.0208],\n",
       "                       [-0.0050,  0.0162, -0.0043,  ..., -0.0002,  0.0167, -0.0138]],\n",
       "                      dtype=torch.float16))]),\n",
       " 'buffer_names': [],\n",
       " 'optimizer': None,\n",
       " 'param_shapes': [OrderedDict([('emb.weight', torch.Size([32000, 3200])),\n",
       "               ('blocks.0.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.0.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.0.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.0.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.0.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.0.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.0.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.0.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.0.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.0.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.0.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.0.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.1.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.1.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.1.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.1.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.1.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.1.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.1.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.1.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.1.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.1.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.1.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.1.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.2.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.2.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.2.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.2.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.2.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.2.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.2.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.2.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.2.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.2.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.2.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.2.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.3.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.3.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.3.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.3.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.3.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.3.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.3.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.3.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.3.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.3.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.3.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.3.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.4.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.4.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.4.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.4.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.4.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.4.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.4.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.4.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.4.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.4.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.4.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.4.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.5.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.5.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.5.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.5.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.5.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.5.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.5.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.5.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.5.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.5.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.5.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.5.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.6.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.6.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.6.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.6.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.6.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.6.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.6.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.6.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.6.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.6.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.6.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.6.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.7.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.7.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.7.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.7.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.7.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.7.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.7.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.7.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.7.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.7.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.7.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.7.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.8.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.8.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.8.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.8.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.8.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.8.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.8.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.8.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.8.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.8.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.8.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.8.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.9.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.9.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.9.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.9.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.9.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.9.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.9.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.9.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.9.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.9.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.9.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.9.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.10.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.10.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.10.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.10.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.10.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.10.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.10.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.10.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.10.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.10.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.10.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.10.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.11.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.11.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.11.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.11.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.11.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.11.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.11.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.11.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.11.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.11.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.11.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.11.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.12.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.12.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.12.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.12.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.12.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.12.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.12.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.12.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.12.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.12.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.12.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.12.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.13.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.13.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.13.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.13.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.13.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.13.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.13.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.13.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.13.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.13.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.13.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.13.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.14.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.14.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.14.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.14.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.14.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.14.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.14.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.14.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.14.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.14.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.14.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.14.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.15.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.15.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.15.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.15.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.15.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.15.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.15.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.15.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.15.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.15.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.15.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.15.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.16.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.16.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.16.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.16.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.16.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.16.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.16.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.16.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.16.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.16.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.16.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.16.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.17.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.17.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.17.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.17.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.17.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.17.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.17.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.17.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.17.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.17.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.17.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.17.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.18.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.18.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.18.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.18.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.18.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.18.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.18.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.18.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.18.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.18.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.18.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.18.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.19.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.19.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.19.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.19.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.19.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.19.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.19.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.19.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.19.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.19.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.19.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.19.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.20.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.20.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.20.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.20.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.20.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.20.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.20.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.20.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.20.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.20.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.20.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.20.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.21.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.21.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.21.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.21.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.21.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.21.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.21.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.21.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.21.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.21.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.21.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.21.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.22.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.22.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.22.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.22.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.22.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.22.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.22.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.22.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.22.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.22.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.22.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.22.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.23.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.23.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.23.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.23.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.23.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.23.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.23.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.23.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.23.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.23.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.23.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.23.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.24.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.24.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.24.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.24.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.24.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.24.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.24.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.24.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.24.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.24.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.24.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.24.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.25.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.25.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.25.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.25.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.25.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.25.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.25.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.25.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.25.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.25.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.25.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.25.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.26.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.26.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.26.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.26.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.26.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.26.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.26.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.26.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.26.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.26.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.26.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.26.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.27.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.27.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.27.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.27.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.27.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.27.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.27.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.27.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.27.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.27.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.27.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.27.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.28.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.28.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.28.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.28.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.28.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.28.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.28.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.28.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.28.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.28.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.28.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.28.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.29.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.29.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.29.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.29.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.29.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.29.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.29.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.29.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.29.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.29.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.29.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.29.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.30.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.30.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.30.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.30.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.30.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.30.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.30.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.30.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.30.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.30.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.30.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.30.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('blocks.31.attn.proj.weight', torch.Size([9600, 3200])),\n",
       "               ('blocks.31.attn.proj.bias', torch.Size([9600])),\n",
       "               ('blocks.31.attn.ff.weight', torch.Size([3200, 3200])),\n",
       "               ('blocks.31.attn.ff.bias', torch.Size([3200])),\n",
       "               ('blocks.31.ln1.w', torch.Size([3200])),\n",
       "               ('blocks.31.ln1.b', torch.Size([3200])),\n",
       "               ('blocks.31.ln2.w', torch.Size([3200])),\n",
       "               ('blocks.31.ln2.b', torch.Size([3200])),\n",
       "               ('blocks.31.mlp.ff1.weight', torch.Size([12800, 3200])),\n",
       "               ('blocks.31.mlp.ff1.bias', torch.Size([12800])),\n",
       "               ('blocks.31.mlp.ff2.weight', torch.Size([3200, 12800])),\n",
       "               ('blocks.31.mlp.ff2.bias', torch.Size([3200])),\n",
       "               ('ln.w', torch.Size([3200])),\n",
       "               ('ln.b', torch.Size([3200])),\n",
       "               ('decoder.weight', torch.Size([32000, 3200]))])],\n",
       " 'frozen_param_shapes': None,\n",
       " 'shared_params': {},\n",
       " 'frozen_param_fragments': None,\n",
       " 'lr_scheduler': {'last_batch_iteration': 6106},\n",
       " 'data_sampler': None,\n",
       " 'random_ltd': None,\n",
       " 'sparse_tensor_module_names': set(),\n",
       " 'skipped_steps': 18,\n",
       " 'global_steps': 6125,\n",
       " 'global_samples': 2352000,\n",
       " 'dp_world_size': 2,\n",
       " 'mp_world_size': 1,\n",
       " 'ds_config': './config/pretrain.json',\n",
       " 'ds_version': '0.10.0'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_ckpt = torch.load('/root/autodl-tmp/sfllm-magic32/main/mp_rank_00_model_states.pt', map_location='cpu')\n",
    "model_ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1d4ec466-9a00-4da0-a5db-c69445d4c7f1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-08-10 01:07:21,529] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    }
   ],
   "source": [
    "ckpt1 = torch.load('/root/autodl-tmp/sfllm-magic32/main/zero_pp_rank_0_mp_rank_00_optim_states.pt', map_location='cpu')\n",
    "ckpt2 = torch.load('/root/autodl-tmp/sfllm-magic32/main/zero_pp_rank_1_mp_rank_00_optim_states.pt', map_location='cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c0ab847b-1345-4967-85c5-eca15b94c220",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for k in model_ckpt['module'].keys():\n",
    "    if 'drop' in k:\n",
    "        print(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d902f8d-d59a-4fff-9049-f7139f124835",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'optimizer_state_dict': {'loss_scaler': <deepspeed.runtime.fp16.loss_scaler.DynamicLossScaler at 0x7faaac5d1700>,\n",
       "  'dynamic_loss_scale': True,\n",
       "  'overflow': False,\n",
       "  'clip_grad': 1,\n",
       "  'base_optimizer_state': {'state': {0: {'step': 5391,\n",
       "     'exp_avg': tensor([ 1.0615e-17,  3.4319e-21,  8.2115e-19,  ...,  9.7487e-08,\n",
       "             -5.8921e-08, -3.2961e-07]),\n",
       "     'exp_avg_sq': tensor([2.6876e-19, 2.8094e-26, 1.6084e-21,  ..., 8.2704e-13, 9.3212e-13,\n",
       "             6.6332e-13])}},\n",
       "   'param_groups': [{'lr': 9.7e-06,\n",
       "     'bias_correction': True,\n",
       "     'betas': [0.9, 0.95],\n",
       "     'eps': 1e-08,\n",
       "     'weight_decay': 0.01,\n",
       "     'step': 0,\n",
       "     'params': [0]}]},\n",
       "  'single_partition_of_fp32_groups': [tensor([-0.0995,  0.2802,  0.0915,  ...,  0.0018, -0.0130,  0.0071],\n",
       "          requires_grad=True)],\n",
       "  'zero_stage': <ZeroStageEnum.optimizer_states: 1>,\n",
       "  'group_paddings': [0],\n",
       "  'partition_count': [2],\n",
       "  'ds_version': '0.10.0',\n",
       "  'param_slice_mappings': [OrderedDict([('emb.weight',\n",
       "                 fragment_address(numel=102400000, start=0)),\n",
       "                ('blocks.0.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=102400000)),\n",
       "                ('blocks.0.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=133120000)),\n",
       "                ('blocks.0.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=133129600)),\n",
       "                ('blocks.0.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=143369600)),\n",
       "                ('blocks.0.ln1.w',\n",
       "                 fragment_address(numel=3200, start=143372800)),\n",
       "                ('blocks.0.ln1.b',\n",
       "                 fragment_address(numel=3200, start=143376000)),\n",
       "                ('blocks.0.ln2.w',\n",
       "                 fragment_address(numel=3200, start=143379200)),\n",
       "                ('blocks.0.ln2.b',\n",
       "                 fragment_address(numel=3200, start=143382400)),\n",
       "                ('blocks.0.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=143385600)),\n",
       "                ('blocks.0.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=184345600)),\n",
       "                ('blocks.0.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=184358400)),\n",
       "                ('blocks.0.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=225318400)),\n",
       "                ('blocks.1.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=225321600)),\n",
       "                ('blocks.1.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=256041600)),\n",
       "                ('blocks.1.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=256051200)),\n",
       "                ('blocks.1.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=266291200)),\n",
       "                ('blocks.1.ln1.w',\n",
       "                 fragment_address(numel=3200, start=266294400)),\n",
       "                ('blocks.1.ln1.b',\n",
       "                 fragment_address(numel=3200, start=266297600)),\n",
       "                ('blocks.1.ln2.w',\n",
       "                 fragment_address(numel=3200, start=266300800)),\n",
       "                ('blocks.1.ln2.b',\n",
       "                 fragment_address(numel=3200, start=266304000)),\n",
       "                ('blocks.1.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=266307200)),\n",
       "                ('blocks.1.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=307267200)),\n",
       "                ('blocks.1.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=307280000)),\n",
       "                ('blocks.1.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=348240000)),\n",
       "                ('blocks.2.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=348243200)),\n",
       "                ('blocks.2.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=378963200)),\n",
       "                ('blocks.2.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=378972800)),\n",
       "                ('blocks.2.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=389212800)),\n",
       "                ('blocks.2.ln1.w',\n",
       "                 fragment_address(numel=3200, start=389216000)),\n",
       "                ('blocks.2.ln1.b',\n",
       "                 fragment_address(numel=3200, start=389219200)),\n",
       "                ('blocks.2.ln2.w',\n",
       "                 fragment_address(numel=3200, start=389222400)),\n",
       "                ('blocks.2.ln2.b',\n",
       "                 fragment_address(numel=3200, start=389225600)),\n",
       "                ('blocks.2.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=389228800)),\n",
       "                ('blocks.2.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=430188800)),\n",
       "                ('blocks.2.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=430201600)),\n",
       "                ('blocks.2.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=471161600)),\n",
       "                ('blocks.3.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=471164800)),\n",
       "                ('blocks.3.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=501884800)),\n",
       "                ('blocks.3.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=501894400)),\n",
       "                ('blocks.3.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=512134400)),\n",
       "                ('blocks.3.ln1.w',\n",
       "                 fragment_address(numel=3200, start=512137600)),\n",
       "                ('blocks.3.ln1.b',\n",
       "                 fragment_address(numel=3200, start=512140800)),\n",
       "                ('blocks.3.ln2.w',\n",
       "                 fragment_address(numel=3200, start=512144000)),\n",
       "                ('blocks.3.ln2.b',\n",
       "                 fragment_address(numel=3200, start=512147200)),\n",
       "                ('blocks.3.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=512150400)),\n",
       "                ('blocks.3.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=553110400)),\n",
       "                ('blocks.3.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=553123200)),\n",
       "                ('blocks.3.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=594083200)),\n",
       "                ('blocks.4.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=594086400)),\n",
       "                ('blocks.4.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=624806400)),\n",
       "                ('blocks.4.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=624816000)),\n",
       "                ('blocks.4.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=635056000)),\n",
       "                ('blocks.4.ln1.w',\n",
       "                 fragment_address(numel=3200, start=635059200)),\n",
       "                ('blocks.4.ln1.b',\n",
       "                 fragment_address(numel=3200, start=635062400)),\n",
       "                ('blocks.4.ln2.w',\n",
       "                 fragment_address(numel=3200, start=635065600)),\n",
       "                ('blocks.4.ln2.b',\n",
       "                 fragment_address(numel=3200, start=635068800)),\n",
       "                ('blocks.4.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=635072000)),\n",
       "                ('blocks.4.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=676032000)),\n",
       "                ('blocks.4.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=676044800)),\n",
       "                ('blocks.4.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=717004800)),\n",
       "                ('blocks.5.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=717008000)),\n",
       "                ('blocks.5.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=747728000)),\n",
       "                ('blocks.5.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=747737600)),\n",
       "                ('blocks.5.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=757977600)),\n",
       "                ('blocks.5.ln1.w',\n",
       "                 fragment_address(numel=3200, start=757980800)),\n",
       "                ('blocks.5.ln1.b',\n",
       "                 fragment_address(numel=3200, start=757984000)),\n",
       "                ('blocks.5.ln2.w',\n",
       "                 fragment_address(numel=3200, start=757987200)),\n",
       "                ('blocks.5.ln2.b',\n",
       "                 fragment_address(numel=3200, start=757990400)),\n",
       "                ('blocks.5.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=757993600)),\n",
       "                ('blocks.5.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=798953600)),\n",
       "                ('blocks.5.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=798966400)),\n",
       "                ('blocks.5.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=839926400)),\n",
       "                ('blocks.6.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=839929600)),\n",
       "                ('blocks.6.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=870649600)),\n",
       "                ('blocks.6.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=870659200)),\n",
       "                ('blocks.6.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=880899200)),\n",
       "                ('blocks.6.ln1.w',\n",
       "                 fragment_address(numel=3200, start=880902400)),\n",
       "                ('blocks.6.ln1.b',\n",
       "                 fragment_address(numel=3200, start=880905600)),\n",
       "                ('blocks.6.ln2.w',\n",
       "                 fragment_address(numel=3200, start=880908800)),\n",
       "                ('blocks.6.ln2.b',\n",
       "                 fragment_address(numel=3200, start=880912000)),\n",
       "                ('blocks.6.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=880915200)),\n",
       "                ('blocks.6.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=921875200)),\n",
       "                ('blocks.6.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=921888000)),\n",
       "                ('blocks.6.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=962848000)),\n",
       "                ('blocks.7.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=962851200)),\n",
       "                ('blocks.7.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=993571200)),\n",
       "                ('blocks.7.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=993580800)),\n",
       "                ('blocks.7.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1003820800)),\n",
       "                ('blocks.7.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1003824000)),\n",
       "                ('blocks.7.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1003827200)),\n",
       "                ('blocks.7.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1003830400)),\n",
       "                ('blocks.7.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1003833600)),\n",
       "                ('blocks.7.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1003836800)),\n",
       "                ('blocks.7.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=1044796800)),\n",
       "                ('blocks.7.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=1044809600)),\n",
       "                ('blocks.7.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=1085769600)),\n",
       "                ('blocks.8.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=1085772800)),\n",
       "                ('blocks.8.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=1116492800)),\n",
       "                ('blocks.8.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=1116502400)),\n",
       "                ('blocks.8.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1126742400)),\n",
       "                ('blocks.8.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1126745600)),\n",
       "                ('blocks.8.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1126748800)),\n",
       "                ('blocks.8.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1126752000)),\n",
       "                ('blocks.8.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1126755200)),\n",
       "                ('blocks.8.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1126758400)),\n",
       "                ('blocks.8.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=1167718400)),\n",
       "                ('blocks.8.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=1167731200)),\n",
       "                ('blocks.8.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=1208691200)),\n",
       "                ('blocks.9.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=1208694400)),\n",
       "                ('blocks.9.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=1239414400)),\n",
       "                ('blocks.9.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=1239424000)),\n",
       "                ('blocks.9.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1249664000)),\n",
       "                ('blocks.9.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1249667200)),\n",
       "                ('blocks.9.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1249670400)),\n",
       "                ('blocks.9.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1249673600)),\n",
       "                ('blocks.9.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1249676800)),\n",
       "                ('blocks.9.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1249680000)),\n",
       "                ('blocks.9.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=1290640000)),\n",
       "                ('blocks.9.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=1290652800)),\n",
       "                ('blocks.9.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=1331612800)),\n",
       "                ('blocks.10.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=1331616000)),\n",
       "                ('blocks.10.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=1362336000)),\n",
       "                ('blocks.10.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=1362345600)),\n",
       "                ('blocks.10.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1372585600)),\n",
       "                ('blocks.10.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1372588800)),\n",
       "                ('blocks.10.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1372592000)),\n",
       "                ('blocks.10.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1372595200)),\n",
       "                ('blocks.10.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1372598400)),\n",
       "                ('blocks.10.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1372601600)),\n",
       "                ('blocks.10.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=1413561600)),\n",
       "                ('blocks.10.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=1413574400)),\n",
       "                ('blocks.10.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=1454534400)),\n",
       "                ('blocks.11.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=1454537600)),\n",
       "                ('blocks.11.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=1485257600)),\n",
       "                ('blocks.11.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=1485267200)),\n",
       "                ('blocks.11.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1495507200)),\n",
       "                ('blocks.11.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1495510400)),\n",
       "                ('blocks.11.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1495513600)),\n",
       "                ('blocks.11.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1495516800)),\n",
       "                ('blocks.11.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1495520000)),\n",
       "                ('blocks.11.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1495523200)),\n",
       "                ('blocks.11.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=1536483200)),\n",
       "                ('blocks.11.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=1536496000)),\n",
       "                ('blocks.11.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=1577456000)),\n",
       "                ('blocks.12.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=1577459200)),\n",
       "                ('blocks.12.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=1608179200)),\n",
       "                ('blocks.12.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=1608188800)),\n",
       "                ('blocks.12.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1618428800)),\n",
       "                ('blocks.12.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1618432000)),\n",
       "                ('blocks.12.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1618435200)),\n",
       "                ('blocks.12.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1618438400)),\n",
       "                ('blocks.12.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1618441600)),\n",
       "                ('blocks.12.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1618444800)),\n",
       "                ('blocks.12.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=1659404800)),\n",
       "                ('blocks.12.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=1659417600)),\n",
       "                ('blocks.12.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=1700377600)),\n",
       "                ('blocks.13.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=1700380800)),\n",
       "                ('blocks.13.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=1731100800)),\n",
       "                ('blocks.13.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=1731110400)),\n",
       "                ('blocks.13.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1741350400)),\n",
       "                ('blocks.13.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1741353600)),\n",
       "                ('blocks.13.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1741356800)),\n",
       "                ('blocks.13.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1741360000)),\n",
       "                ('blocks.13.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1741363200)),\n",
       "                ('blocks.13.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1741366400)),\n",
       "                ('blocks.13.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=1782326400)),\n",
       "                ('blocks.13.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=1782339200)),\n",
       "                ('blocks.13.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=1823299200)),\n",
       "                ('blocks.14.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=1823302400)),\n",
       "                ('blocks.14.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=1854022400)),\n",
       "                ('blocks.14.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=1854032000)),\n",
       "                ('blocks.14.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1864272000)),\n",
       "                ('blocks.14.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1864275200)),\n",
       "                ('blocks.14.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1864278400)),\n",
       "                ('blocks.14.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1864281600)),\n",
       "                ('blocks.14.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1864284800)),\n",
       "                ('blocks.14.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1864288000)),\n",
       "                ('blocks.14.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=1905248000)),\n",
       "                ('blocks.14.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=1905260800)),\n",
       "                ('blocks.14.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=1946220800)),\n",
       "                ('blocks.15.attn.proj.weight',\n",
       "                 fragment_address(numel=30720000, start=1946224000)),\n",
       "                ('blocks.15.attn.proj.bias',\n",
       "                 fragment_address(numel=9600, start=1976944000)),\n",
       "                ('blocks.15.attn.ff.weight',\n",
       "                 fragment_address(numel=10240000, start=1976953600)),\n",
       "                ('blocks.15.attn.ff.bias',\n",
       "                 fragment_address(numel=3200, start=1987193600)),\n",
       "                ('blocks.15.ln1.w',\n",
       "                 fragment_address(numel=3200, start=1987196800)),\n",
       "                ('blocks.15.ln1.b',\n",
       "                 fragment_address(numel=3200, start=1987200000)),\n",
       "                ('blocks.15.ln2.w',\n",
       "                 fragment_address(numel=3200, start=1987203200)),\n",
       "                ('blocks.15.ln2.b',\n",
       "                 fragment_address(numel=3200, start=1987206400)),\n",
       "                ('blocks.15.mlp.ff1.weight',\n",
       "                 fragment_address(numel=40960000, start=1987209600)),\n",
       "                ('blocks.15.mlp.ff1.bias',\n",
       "                 fragment_address(numel=12800, start=2028169600)),\n",
       "                ('blocks.15.mlp.ff2.weight',\n",
       "                 fragment_address(numel=40960000, start=2028182400)),\n",
       "                ('blocks.15.mlp.ff2.bias',\n",
       "                 fragment_address(numel=3200, start=2069142400)),\n",
       "                ('blocks.16.attn.proj.weight',\n",
       "                 fragment_address(numel=3200, start=2069145600))])]},\n",
       " 'ds_config': './config/ds_cfg_pretrain2.json',\n",
       " 'ds_version': '0.10.0'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpt1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3c349b8d-e52f-4b5c-851b-d1cb9c6a8b19",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ckpt1['optimizer_state_dict']['base_optimizer_state']['state'][0]['step'] = 2600\n",
    "# ckpt2['optimizer_state_dict']['base_optimizer_state']['state'][0]['step'] = 2600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7d75ef6c-b347-4d59-90f9-024e3e538913",
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(ckpt1, ckpt_rep_path1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c46ba2ef-cbd2-4a1b-9774-1e395eee53c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(ckpt2, ckpt_rep_path2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6987fb47-a5b8-4692-99af-0a6ea0b0a6e8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}